## ----setup, include=FALSE-----------------------------------------------------
knitr::opts_chunk$set(fig.width = 50, fig.height = 50, warning = FALSE, message = FALSE, echo = FALSE, cache = FALSE, eval = FALSE)

dir_wili_2018 = '/IN CASE I WANT TO REBUILD THE VIGNETTE MODIFY AND POINT THIS DIR TO THE WILLI 2018 DIRECTORY/wili-2018'
dir_files = '/IN CASE I WANT TO REBUILD THE VIGNETTE MODIFY AND POINT THIS DIR TO THE HUMAN DECLARATION FILES/declaration_human_rights'         # the other 'dir_files' directory in line 663 is mentioned just in case I download the declaration files inside the 'wili-2018' directory


## ----echo = T-----------------------------------------------------------------
#  
#  fasttext_supported_languages = c('af', 'als', 'am', 'an', 'ar', 'arz', 'as', 'ast', 'av',
#                                   'az', 'azb', 'ba', 'bar', 'bcl', 'be', 'bg', 'bh', 'bn',
#                                   'bo', 'bpy', 'br', 'bs', 'bxr', 'ca', 'cbk', 'ce', 'ceb',
#                                   'ckb', 'co', 'cs', 'cv', 'cy', 'da', 'de', 'diq', 'dsb',
#                                   'dty', 'dv', 'el', 'eml', 'en', 'eo', 'es', 'et', 'eu',
#                                   'fa', 'fi', 'fr', 'frr', 'fy', 'ga', 'gd', 'gl', 'gn',
#                                   'gom', 'gu', 'gv', 'he', 'hi', 'hif', 'hr', 'hsb', 'ht',
#                                   'hu', 'hy', 'ia', 'id', 'ie', 'ilo', 'io', 'is', 'it',
#                                   'ja', 'jbo', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'krc',
#                                   'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lez', 'li', 'lmo',
#                                   'lo', 'lrc', 'lt', 'lv', 'mai', 'mg', 'mhr', 'min', 'mk',
#                                   'ml', 'mn', 'mr', 'mrj', 'ms', 'mt', 'mwl', 'my', 'myv',
#                                   'mzn', 'nah', 'nap', 'nds', 'ne', 'new', 'nl', 'nn', 'no',
#                                   'oc', 'or', 'os', 'pa', 'pam', 'pfl', 'pl', 'pms', 'pnb',
#                                   'ps', 'pt', 'qu', 'rm', 'ro', 'ru', 'rue', 'sa', 'sah',
#                                   'sc', 'scn', 'sco', 'sd', 'sh', 'si', 'sk', 'sl', 'so',
#                                   'sq', 'sr', 'su', 'sv', 'sw', 'ta', 'te', 'tg', 'th', 'tk',
#                                   'tl', 'tr', 'tt', 'tyv', 'ug', 'uk', 'ur', 'uz', 'vec',
#                                   'vep', 'vi', 'vls', 'vo', 'wa', 'war', 'wuu', 'xal', 'xmf',
#                                   'yi', 'yo', 'yue', 'zh')

## ----echo = T-----------------------------------------------------------------
#  
#  isocodes = ISOcodes::ISO_639_2
#  # head(isocodes)
#  
#  comp_cases = complete.cases(isocodes$Alpha_2)
#  isocodes_fasttext = isocodes[comp_cases, ]
#  # dim(isocodes_fasttext)
#  
#  idx_keep_fasttext = which(isocodes_fasttext$Alpha_2 %in% fasttext_supported_languages)
#  
#  isocodes_fasttext = isocodes_fasttext[idx_keep_fasttext, ]
#  isocodes_fasttext = data.table::data.table(isocodes_fasttext)
#  # isocodes_fasttext
#  
#  lower_nams = tolower(isocodes_fasttext$Name)
#  lower_nams = trimws(as.vector(unlist(lapply(strsplit(lower_nams, "[;, ]"), function(x) x[1]))), which = 'both')   # remove second or third naming of the country name
#  
#  isocodes_fasttext$Name_tolower = lower_nams
#  isocodes_fasttext

## ----echo = T-----------------------------------------------------------------
#  
#  print_accuracy = function(size_input_data,
#                            true_data,
#                            preds_data,
#                            method) {
#  
#    cat(glue::glue("Total Rows: {size_input_data}"), '\n')
#    rnd_2 = round(length(preds_data)/size_input_data, 4)
#    msg_2 ="Predicted Rows: {length(preds_data)}  ({rnd_2 * 100}% predicted)"
#    cat(glue::glue(msg_2), '\n')
#    cat(glue::glue("Missing Values:  {size_input_data - length(preds_data)}"), '\n')
#    rnd_4 = round(sum(true_data == preds_data) / length(preds_data), 4)
#    msg_4 = "Accuracy on 'Predicted Rows' using '{method}':  {rnd_4 * 100}%"
#    cat(glue::glue(msg_4), '\n')
#  }
#  
#  # fasttext language identification supported languages as described in https://fasttext.cc/docs/en/language-identification.html

## -----------------------------------------------------------------------------
#  
#  list.files(dir_wili_2018)
#  

## -----------------------------------------------------------------------------
#  
#  pth_x_test = file.path(dir_wili_2018, 'x_test.txt')
#  wili_test_x = data.table::fread(file = pth_x_test, sep = '\n', stringsAsFactors = F, header = F, nThread = parallel::detectCores())
#  
#  pth_y_test = file.path(dir_wili_2018, 'y_test.txt')
#  wili_test_y = data.table::fread(file = pth_y_test, sep = '\n', stringsAsFactors = F, header = F, nThread = parallel::detectCores())
#  nrow_init = nrow(wili_test_y)
#  
#  inters_labels = which(wili_test_y$V1 %in% isocodes_fasttext$Alpha_3_B)
#  
#  # subset both the 'x_test' and the 'y_test' data
#  
#  wili_test_x = wili_test_x[inters_labels, ]
#  wili_test_y = wili_test_y[inters_labels, ]
#  
#  cat(glue::glue("Initial observations:  {nrow_init}  Subset based on isocodes:  {nrow(wili_test_y)}   Number of languages based on subset:  {length(unique(wili_test_y$V1))}"), '\n')
#  
#  head(wili_test_y)
#  

## ----echo = T-----------------------------------------------------------------
#  
#  file_ftz = system.file("language_identification/lid.176.ftz", package = "fastText")
#  
#  dtbl_res_in = fastText::language_identification(input_obj = wili_test_x$V1,
#                                                  pre_trained_language_model_path = file_ftz,
#                                                  k = 1,
#                                                  th = 0.0,
#                                                  threads = 1,
#                                                  verbose = TRUE)

## ----echo = T-----------------------------------------------------------------
#  
#  dtbl_res_in$true_label = wili_test_y$V1
#  # dtbl_res_in
#  
#  isocodes_fasttext_subs = isocodes_fasttext[, c(1,3)]   # merge the predicted labels with the 3-letter isocodes
#  
#  merg_labels = merge(dtbl_res_in, isocodes_fasttext_subs, by.x = 'iso_lang_1', by.y = 'Alpha_2')
#  # as.vector(colSums(is.na(merg_labels)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = merg_labels$true_label,
#                 preds_data = merg_labels$Alpha_3_B,
#                 method = 'fastText (.ftz pre-trained model)')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  file_bin = file.path(dir_wili_2018, 'lid.176.bin')
#  
#  dtbl_res_in = fastText::language_identification(input_obj = wili_test_x$V1,
#                                                  pre_trained_language_model_path = file_bin,
#                                                  k = 1,
#                                                  th = 0.0,
#                                                  threads = 1,
#                                                  verbose = TRUE)

## ----echo = T-----------------------------------------------------------------
#  
#  dtbl_res_in$true_label = wili_test_y$V1
#  # dtbl_res_in
#  
#  isocodes_fasttext_subs = isocodes_fasttext[, c(1,3)]   # merge the predicted labels with the 3-letter isocodes
#  
#  merg_labels = merge(dtbl_res_in, isocodes_fasttext_subs, by.x = 'iso_lang_1', by.y = 'Alpha_2')
#  # as.vector(colSums(is.na(merg_labels)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = merg_labels$true_label,
#                 preds_data = merg_labels$Alpha_3_B,
#                 method = 'fastText (.ftz pre-trained model)')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  tbl = table(merg_labels$true_label, merg_labels$Alpha_3_B)
#  
#  df = as.data.frame.table(tbl)
#  colnames(df) = c('country_vert', 'country_horiz', 'Freq')
#  # head(df)
#  
#  require(magrittr)
#  require(dplyr)
#  require(ggplot2)
#  
#  df <- df %>%
#    mutate(country_vert = factor(country_vert),                 # alphabetical order by default
#           country_horiz = factor(country_horiz, levels = rev(unique(country_horiz))))
#  
#  plt_tbl = ggplot(df, aes(x=country_vert, y=country_horiz, fill=Freq)) +
#    geom_tile() + theme_bw() + coord_equal() +
#    scale_fill_distiller(palette="Greens", direction=1) +
#    ggplot2::theme(axis.text.x = element_text(angle = 45, vjust = 1.0, hjust = 1.0))
#  
#  plt_tbl
#  

## ----echo = T-----------------------------------------------------------------
#  
#  require(cld2)
#  
#  t_start = proc.time()
#  cld2_vec = cld2::detect_language(text = wili_test_x$V1, plain_text = TRUE, lang_code = TRUE)
#  
#  cld2_dtbl = data.table::setDT(list(Alpha_2 = cld2_vec))
#  cld2_dtbl$true_label = wili_test_y$V1
#  
#  merg_labels_cld2 = merge(cld2_dtbl, isocodes_fasttext_subs, by = 'Alpha_2')
#  # as.vector(colSums(is.na(merg_labels_cld2)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = merg_labels_cld2$true_label,
#                 preds_data = merg_labels_cld2$Alpha_3_B,
#                 method = 'cld2')
#  

## ----echo = F-----------------------------------------------------------------
#  
#  fastText:::compute_elapsed_time(t_start)
#  

## ----echo = T-----------------------------------------------------------------
#  
#  require(cld3)
#  
#  t_start = proc.time()
#  cld3_vec = cld3::detect_language(text = wili_test_x$V1)
#  
#  cld3_dtbl = data.table::setDT(list(Alpha_2 = cld3_vec))
#  cld3_dtbl$true_label = wili_test_y$V1
#  
#  merg_labels_cld3 = merge(cld3_dtbl, isocodes_fasttext_subs, by = 'Alpha_2')
#  # as.vector(colSums(is.na(merg_labels_cld3)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = merg_labels_cld3$true_label,
#                 preds_data = merg_labels_cld3$Alpha_3_B,
#                 method = 'cld3')
#  

## ----echo = F-----------------------------------------------------------------
#  
#  fastText:::compute_elapsed_time(t_start)
#  

## ----echo = T-----------------------------------------------------------------
#  
#  threads = parallel::detectCores()
#  require(textcat)
#  
#  names(textcat::TC_byte_profiles)
#  

## -----------------------------------------------------------------------------
#  
#  nams_profiles = as.vector(unlist(lapply(strsplit(names(textcat::TC_byte_profiles), '-'), function(x) x[1])))
#  nams_profiles = unique(nams_profiles)
#  cat(glue::glue("Isocode-Names:  {length(unique(isocodes_fasttext$Name_tolower))}  TC_byte_profiles:  {length(names(textcat::TC_byte_profiles))}  Intersected Names: {length(intersect(nams_profiles, unique(isocodes_fasttext$Name_tolower)))}"), '\n')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  t_start = proc.time()
#  textc = as.vector(unlist(parallel::mclapply(1:length(wili_test_x$V1), function(x) {
#    textcat(x = wili_test_x$V1[x], p = textcat::TC_byte_profiles, method = "CT")
#  }, mc.cores = threads)))
#  
#  textc = as.vector(unlist(lapply(strsplit(textc, '-'), function(x) x[1])))
#  textc = trimws(textc, which = 'both')
#  
#  unique(textc)

## ----echo = T-----------------------------------------------------------------
#  
#  textc_dtbl = data.table::setDT(list(Name_tolower = textc))
#  textc_dtbl$true_label = wili_test_y$V1
#  
#  fasttext_isoc_name = isocodes_fasttext[, c(1,5)]
#  merg_labels_textc = merge(textc_dtbl, fasttext_isoc_name, by = 'Name_tolower')
#  # as.vector(colSums(is.na(merg_labels_cld2)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = merg_labels_textc$true_label,
#                 preds_data = merg_labels_textc$Alpha_3_B,
#                 method = 'textcat ( TC_byte_profiles )')

## ----echo = F-----------------------------------------------------------------
#  
#  fastText:::compute_elapsed_time(t_start)
#  

## ----echo = T-----------------------------------------------------------------
#  
#  names(textcat::TC_char_profiles)
#  

## -----------------------------------------------------------------------------
#  
#  nams_profiles = as.vector(unlist(lapply(strsplit(names(textcat::TC_char_profiles), '-'), function(x) x[1])))
#  nams_profiles = unique(nams_profiles)
#  cat(glue::glue("Isocode-Names:  {length(unique(isocodes_fasttext$Name_tolower))}  TC_char_profiles:  {length(names(textcat::TC_char_profiles))}  Intersected Names: {length(intersect(nams_profiles, unique(isocodes_fasttext$Name_tolower)))}"), '\n')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  t_start = proc.time()
#  textc = as.vector(unlist(parallel::mclapply(1:length(wili_test_x$V1), function(x) {
#    textcat(x = wili_test_x$V1[x], p = textcat::TC_char_profiles, method = "CT")
#  }, mc.cores = threads)))
#  
#  textc = as.vector(unlist(lapply(strsplit(textc, '-'), function(x) x[1])))
#  textc = trimws(textc, which = 'both')
#  
#  unique(textc)

## ----echo = T-----------------------------------------------------------------
#  
#  textc_dtbl = data.table::setDT(list(Name_tolower = textc))
#  textc_dtbl$true_label = wili_test_y$V1
#  
#  fasttext_isoc_name = isocodes_fasttext[, c(1,5)]
#  merg_labels_textc = merge(textc_dtbl, fasttext_isoc_name, by = 'Name_tolower')
#  # as.vector(colSums(is.na(merg_labels_cld2)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = merg_labels_textc$true_label,
#                 preds_data = merg_labels_textc$Alpha_3_B,
#                 method = 'textcat ( TC_char_profiles )')

## ----echo = F-----------------------------------------------------------------
#  
#  fastText:::compute_elapsed_time(t_start)
#  

## ----echo = T-----------------------------------------------------------------
#  
#  require(franc)
#  
#  t_start = proc.time()
#  franc_res = as.vector(unlist(parallel::mclapply(1:length(wili_test_x$V1), function(x) {
#    franc(text = wili_test_x$V1[x], min_speakers = 0, min_length = 10, max_length = 2048)
#  }, mc.cores = threads)))
#  
#  franc_dtbl = data.table::setDT(list(franc = franc_res, true_label = wili_test_y$V1))
#  # as.vector(colSums(is.na(franc_dtbl)))
#  
#  print_accuracy(size_input_data = nrow(wili_test_y),
#                 true_data = franc_dtbl$true_label,
#                 preds_data = franc_dtbl$franc,
#                 method = 'franc')
#  

## ----echo = F-----------------------------------------------------------------
#  
#  fastText:::compute_elapsed_time(t_start)
#  

## ----echo = F-----------------------------------------------------------------
#  
#  dtbl_bench = data.table::setDT(list(method = c('fastText (ftz)', 'fastText (bin)', 'cld2', 'cld3', 'textcat (byte)', 'textcat (char)', 'franc'),
#                                      rows = c(50500, 50500, 50500, 50500, 50500, 50500, 50500),
#                                      pred_rows = c(50211, 50168, 34254, 43560, 47324, 43265, 50500),
#                                      pred_perc = c(99.43, 99.34, 67.83, 86.26, 93.71, 85.67, 100.0),
#                                      NAs = c(289, 332, 16246, 6940, 3176, 7235, 0),
#                                      accuracy = c(83.05, 86.55, 83.13, 74.74, 29.91, 31.1, 62.04),
#                                      seconds = c(5, 5, 2, 18, 83, 100, 179),
#                                      threads = c(1, 1, 1, 1, 8, 8, 8)))

## ----echo = F-----------------------------------------------------------------
#  dtbl_bench = dtbl_bench[order(dtbl_bench$accuracy, decreasing = T), ]
#  dtbl_bench
#  

## ----echo = F-----------------------------------------------------------------
#  dtbl_bench = dtbl_bench[order(dtbl_bench$pred_perc, decreasing = T), ]
#  dtbl_bench
#  

## ----echo = F-----------------------------------------------------------------
#  dtbl_bench = dtbl_bench[order(dtbl_bench$NAs, decreasing = F), ]
#  dtbl_bench
#  

## ----echo = F-----------------------------------------------------------------
#  dtbl_bench = dtbl_bench[order(dtbl_bench$seconds, decreasing = F), ]
#  dtbl_bench
#  

## ----eval = F, echo = T-------------------------------------------------------
#  
#  dir_files = file.path(dir_wili_2018, 'declaration_human_rights')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  lst_files = list.files(dir_files, full.names = T, pattern = '.pdf')
#  
#  decl_dat = lapply(1:length(lst_files), function(x) {
#  
#    iter_dat = pdftools::pdf_text(pdf = lst_files[x])
#    lang = trimws(unlist(strsplit(gsub('.pdf', '', basename(lst_files[x])), '_')), which = 'both')
#    lang = lang[length(lang)]
#    vec_txt = as.vector(unlist(trimws(iter_dat, which = 'both')))
#    vec_txt = as.vector(sapply(vec_txt, function(x) gsub('\n', '', x)))
#  
#    idx_lang = which(isocodes_fasttext$Name_tolower == lang)
#    isocode_3_language = rep(isocodes_fasttext$Alpha_3_B[idx_lang], length(vec_txt))
#    isocode_2_language = rep(isocodes_fasttext$Alpha_2[idx_lang], length(vec_txt))
#    language = rep(lang, length(vec_txt))
#  
#    dtbl = data.table::setDT(list(isocode_3_language = isocode_3_language,
#                                  isocode_2_language = isocode_2_language,
#                                  language = language,
#                                  text = vec_txt))
#    dtbl
#  })
#  
#  decl_dat = data.table::rbindlist(decl_dat)
#  

## ----echo = F-----------------------------------------------------------------
#  
#  decl_dat$language
#  decl_dat$isocode_3_language
#  decl_dat$isocode_2_language
#  

## ----echo = T-----------------------------------------------------------------
#  
#  dtbl_res_in = fastText::language_identification(input_obj = decl_dat$text,
#                                                  pre_trained_language_model_path = file_ftz,
#                                                  k = 1,
#                                                  th = 0.0,
#                                                  threads = 1,
#                                                  verbose = TRUE)

## ----echo = F-----------------------------------------------------------------
#  
#  dtbl_res_in
#  

## ----echo = T-----------------------------------------------------------------
#  
#  print_accuracy(size_input_data = length(dtbl_res_in$iso_lang_1),
#                 true_data = decl_dat$isocode_2_language,
#                 preds_data = dtbl_res_in$iso_lang_1,
#                 method = 'fastText (.ftz pre-trained model)')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  cld2_vec = cld2::detect_language(text = decl_dat$text,
#                                   plain_text = TRUE,
#                                   lang_code = TRUE)
#  cld2_vec
#  

## ----echo = T-----------------------------------------------------------------
#  
#  print_accuracy(size_input_data = nrow(decl_dat),
#                 true_data = decl_dat$isocode_2_language,
#                 preds_data = cld2_vec,
#                 method = 'cld2')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  cld3_vec = cld3::detect_language(text = decl_dat$text)
#  cld3_vec
#  

## ----echo = T-----------------------------------------------------------------
#  
#  print_accuracy(size_input_data = nrow(decl_dat),
#                 true_data = decl_dat$isocode_2_language,
#                 preds_data = cld3_vec,
#                 method = 'cld3')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  textc = textcat(x = decl_dat$text, p = textcat::TC_byte_profiles, method = "CT")
#  textc
#  

## ----echo = T-----------------------------------------------------------------
#  
#  textc = as.vector(unlist(lapply(strsplit(textc, '-'), function(x) x[1])))
#  textc = trimws(textc, which = 'both')
#  textc
#  

## ----echo = T-----------------------------------------------------------------
#  
#  print_accuracy(size_input_data = nrow(decl_dat),
#                 true_data = decl_dat$language,
#                 preds_data = textc,
#                 method = 'textcat')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  franc_vec = as.vector(sapply(decl_dat$text, function(x) {
#    franc(text = x, min_length = 10, max_length = 2048)
#  }))
#  
#  franc_vec
#  

## ----echo = T-----------------------------------------------------------------
#  
#  print_accuracy(size_input_data = nrow(decl_dat),
#                 true_data = decl_dat$isocode_3_language,
#                 preds_data = franc_vec,
#                 method = 'franc')
#  

## ----echo = T-----------------------------------------------------------------
#  
#  dtbl_out = decl_dat[, 1:3]
#  colnames(dtbl_out) = c('true_y_iso_3', 'true_y_iso_2', 'true_y_language')
#  # dtbl_out
#  
#  dtbl_preds = data.table::setDT(list(fastText = dtbl_res_in$iso_lang_1,
#                                      cld2 = cld2_vec,
#                                      cld3 = cld3_vec,
#                                      textcat = textc,
#                                      franc = franc_vec))
#  # dtbl_preds
#  
#  dtbl_out = cbind(dtbl_out, dtbl_preds)
#  dtbl_out
#  

## ----echo = T-----------------------------------------------------------------
#  
#  lst_files = list.files(dir_files, full.names = F, pattern = '.pdf')
#  
#  min_letters_en_es = 3         # min. number of characters for the 'en' and 'es' languages
#  sample_words = 100            # sample that many words from each tokenized file
#  
#  decl_dat = lapply(1:length(lst_files), function(x) {
#  
#    iter_dat = pdftools::pdf_text(pdf = file.path(dir_files, lst_files[x]))
#  
#    dat_txt = sapply(iter_dat, function(y) {
#  
#      if (lst_files[x] == 'declaration_human_rights_chinese.pdf') {
#        res_spl_lang = stringi::stri_split_boundaries(str = y,
#                                                      type = 'word',
#                                                      skip_word_none = TRUE,
#                                                      skip_word_letter = TRUE,
#                                                      skip_word_number = TRUE)
#      }
#      else {
#        res_spl_lang = stringi::stri_split(str = y,
#                                           regex = '[ \n,]',
#                                           omit_empty = TRUE,
#                                           tokens_only = TRUE)
#      }
#  
#      res_spl_lang = trimws(res_spl_lang[[1]], which = 'both')
#      idx_empty = which(res_spl_lang == "")
#      if (length(idx_empty) > 0) {
#        res_spl_lang = res_spl_lang[-idx_empty]
#      }
#      if (!is.null(min_letters_en_es) & lst_files[x] != 'declaration_human_rights_chinese.pdf') {
#        nchars = nchar(res_spl_lang)
#        idx_chars = which(nchars >= min_letters_en_es)
#        if (length(idx_chars) > 0) {
#          res_spl_lang = res_spl_lang[idx_chars]
#        }
#      }
#      res_spl_lang
#    })
#  
#    dat_txt = as.vector(unlist(dat_txt))
#    set.seed(1)
#    sample_words = sample(dat_txt, sample_words)
#    sample_words
#  })
#  
#  
#  decl_dat = as.vector(unlist(decl_dat))
#  decl_dat = decl_dat[sample(1:length(decl_dat), length(decl_dat))]
#  multilingual_sentence = paste(decl_dat, collapse = ' ')
#  multilingual_sentence
#  

## ----echo = T-----------------------------------------------------------------
#  
#  num_languages = 3
#  

## ----echo = T-----------------------------------------------------------------
#  
#  dtbl_multiling = fastText::language_identification(input_obj = multilingual_sentence,
#                                                     pre_trained_language_model_path = file_ftz,
#                                                     k = num_languages,
#                                                     th = 0.0,
#                                                     threads = 1,
#                                                     verbose = FALSE)
#  dtbl_multiling
#  

## ----echo = T-----------------------------------------------------------------
#  
#  cld2::detect_language_mixed(text = multilingual_sentence, plain_text = TRUE)$classification
#  

## ----echo = T-----------------------------------------------------------------
#  
#  cld3::detect_language_mixed(text = multilingual_sentence, size = num_languages)
#  

## ----echo = T-----------------------------------------------------------------
#  
#  # we could use the 'whitelist' parameter but the purpose is to identify languages from unknown text
#  
#  franc::franc_all(text = multilingual_sentence, max_length = nchar(multilingual_sentence) + 1)[1:num_languages, ]
#  

