

# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


_SCREAMING_SNAKE_CASE					:  str             =      {
    '''Acehnese Arabic''': '''ace_Arab''',
    '''Acehnese Latin''': '''ace_Latn''',
    '''Mesopotamian Arabic''': '''acm_Arab''',
    '''Ta\'izzi-Adeni Arabic''': '''acq_Arab''',
    '''Tunisian Arabic''': '''aeb_Arab''',
    '''Afrikaans''': '''afr_Latn''',
    '''South Levantine Arabic''': '''ajp_Arab''',
    '''Akan''': '''aka_Latn''',
    '''Amharic''': '''amh_Ethi''',
    '''North Levantine Arabic''': '''apc_Arab''',
    '''Modern Standard Arabic''': '''arb_Arab''',
    '''Modern Standard Arabic Romanized''': '''arb_Latn''',
    '''Najdi Arabic''': '''ars_Arab''',
    '''Moroccan Arabic''': '''ary_Arab''',
    '''Egyptian Arabic''': '''arz_Arab''',
    '''Assamese''': '''asm_Beng''',
    '''Asturian''': '''ast_Latn''',
    '''Awadhi''': '''awa_Deva''',
    '''Central Aymara''': '''ayr_Latn''',
    '''South Azerbaijani''': '''azb_Arab''',
    '''North Azerbaijani''': '''azj_Latn''',
    '''Bashkir''': '''bak_Cyrl''',
    '''Bambara''': '''bam_Latn''',
    '''Balinese''': '''ban_Latn''',
    '''Belarusian''': '''bel_Cyrl''',
    '''Bemba''': '''bem_Latn''',
    '''Bengali''': '''ben_Beng''',
    '''Bhojpuri''': '''bho_Deva''',
    '''Banjar Arabic''': '''bjn_Arab''',
    '''Banjar Latin''': '''bjn_Latn''',
    '''Standard Tibetan''': '''bod_Tibt''',
    '''Bosnian''': '''bos_Latn''',
    '''Buginese''': '''bug_Latn''',
    '''Bulgarian''': '''bul_Cyrl''',
    '''Catalan''': '''cat_Latn''',
    '''Cebuano''': '''ceb_Latn''',
    '''Czech''': '''ces_Latn''',
    '''Chokwe''': '''cjk_Latn''',
    '''Central Kurdish''': '''ckb_Arab''',
    '''Crimean Tatar''': '''crh_Latn''',
    '''Welsh''': '''cym_Latn''',
    '''Danish''': '''dan_Latn''',
    '''German''': '''deu_Latn''',
    '''Southwestern Dinka''': '''dik_Latn''',
    '''Dyula''': '''dyu_Latn''',
    '''Dzongkha''': '''dzo_Tibt''',
    '''Greek''': '''ell_Grek''',
    '''English''': '''eng_Latn''',
    '''Esperanto''': '''epo_Latn''',
    '''Estonian''': '''est_Latn''',
    '''Basque''': '''eus_Latn''',
    '''Ewe''': '''ewe_Latn''',
    '''Faroese''': '''fao_Latn''',
    '''Fijian''': '''fij_Latn''',
    '''Finnish''': '''fin_Latn''',
    '''Fon''': '''fon_Latn''',
    '''French''': '''fra_Latn''',
    '''Friulian''': '''fur_Latn''',
    '''Nigerian Fulfulde''': '''fuv_Latn''',
    '''Scottish Gaelic''': '''gla_Latn''',
    '''Irish''': '''gle_Latn''',
    '''Galician''': '''glg_Latn''',
    '''Guarani''': '''grn_Latn''',
    '''Gujarati''': '''guj_Gujr''',
    '''Haitian Creole''': '''hat_Latn''',
    '''Hausa''': '''hau_Latn''',
    '''Hebrew''': '''heb_Hebr''',
    '''Hindi''': '''hin_Deva''',
    '''Chhattisgarhi''': '''hne_Deva''',
    '''Croatian''': '''hrv_Latn''',
    '''Hungarian''': '''hun_Latn''',
    '''Armenian''': '''hye_Armn''',
    '''Igbo''': '''ibo_Latn''',
    '''Ilocano''': '''ilo_Latn''',
    '''Indonesian''': '''ind_Latn''',
    '''Icelandic''': '''isl_Latn''',
    '''Italian''': '''ita_Latn''',
    '''Javanese''': '''jav_Latn''',
    '''Japanese''': '''jpn_Jpan''',
    '''Kabyle''': '''kab_Latn''',
    '''Jingpho''': '''kac_Latn''',
    '''Kamba''': '''kam_Latn''',
    '''Kannada''': '''kan_Knda''',
    '''Kashmiri Arabic''': '''kas_Arab''',
    '''Kashmiri Devanagari''': '''kas_Deva''',
    '''Georgian''': '''kat_Geor''',
    '''Central Kanuri Arabic''': '''knc_Arab''',
    '''Central Kanuri Latin''': '''knc_Latn''',
    '''Kazakh''': '''kaz_Cyrl''',
    '''Kabiyè''': '''kbp_Latn''',
    '''Kabuverdianu''': '''kea_Latn''',
    '''Khmer''': '''khm_Khmr''',
    '''Kikuyu''': '''kik_Latn''',
    '''Kinyarwanda''': '''kin_Latn''',
    '''Kyrgyz''': '''kir_Cyrl''',
    '''Kimbundu''': '''kmb_Latn''',
    '''Northern Kurdish''': '''kmr_Latn''',
    '''Kikongo''': '''kon_Latn''',
    '''Korean''': '''kor_Hang''',
    '''Lao''': '''lao_Laoo''',
    '''Ligurian''': '''lij_Latn''',
    '''Limburgish''': '''lim_Latn''',
    '''Lingala''': '''lin_Latn''',
    '''Lithuanian''': '''lit_Latn''',
    '''Lombard''': '''lmo_Latn''',
    '''Latgalian''': '''ltg_Latn''',
    '''Luxembourgish''': '''ltz_Latn''',
    '''Luba-Kasai''': '''lua_Latn''',
    '''Ganda''': '''lug_Latn''',
    '''Luo''': '''luo_Latn''',
    '''Mizo''': '''lus_Latn''',
    '''Standard Latvian''': '''lvs_Latn''',
    '''Magahi''': '''mag_Deva''',
    '''Maithili''': '''mai_Deva''',
    '''Malayalam''': '''mal_Mlym''',
    '''Marathi''': '''mar_Deva''',
    '''Minangkabau Arabic ''': '''min_Arab''',
    '''Minangkabau Latin''': '''min_Latn''',
    '''Macedonian''': '''mkd_Cyrl''',
    '''Plateau Malagasy''': '''plt_Latn''',
    '''Maltese''': '''mlt_Latn''',
    '''Meitei Bengali''': '''mni_Beng''',
    '''Halh Mongolian''': '''khk_Cyrl''',
    '''Mossi''': '''mos_Latn''',
    '''Maori''': '''mri_Latn''',
    '''Burmese''': '''mya_Mymr''',
    '''Dutch''': '''nld_Latn''',
    '''Norwegian Nynorsk''': '''nno_Latn''',
    '''Norwegian Bokmål''': '''nob_Latn''',
    '''Nepali''': '''npi_Deva''',
    '''Northern Sotho''': '''nso_Latn''',
    '''Nuer''': '''nus_Latn''',
    '''Nyanja''': '''nya_Latn''',
    '''Occitan''': '''oci_Latn''',
    '''West Central Oromo''': '''gaz_Latn''',
    '''Odia''': '''ory_Orya''',
    '''Pangasinan''': '''pag_Latn''',
    '''Eastern Panjabi''': '''pan_Guru''',
    '''Papiamento''': '''pap_Latn''',
    '''Western Persian''': '''pes_Arab''',
    '''Polish''': '''pol_Latn''',
    '''Portuguese''': '''por_Latn''',
    '''Dari''': '''prs_Arab''',
    '''Southern Pashto''': '''pbt_Arab''',
    '''Ayacucho Quechua''': '''quy_Latn''',
    '''Romanian''': '''ron_Latn''',
    '''Rundi''': '''run_Latn''',
    '''Russian''': '''rus_Cyrl''',
    '''Sango''': '''sag_Latn''',
    '''Sanskrit''': '''san_Deva''',
    '''Santali''': '''sat_Olck''',
    '''Sicilian''': '''scn_Latn''',
    '''Shan''': '''shn_Mymr''',
    '''Sinhala''': '''sin_Sinh''',
    '''Slovak''': '''slk_Latn''',
    '''Slovenian''': '''slv_Latn''',
    '''Samoan''': '''smo_Latn''',
    '''Shona''': '''sna_Latn''',
    '''Sindhi''': '''snd_Arab''',
    '''Somali''': '''som_Latn''',
    '''Southern Sotho''': '''sot_Latn''',
    '''Spanish''': '''spa_Latn''',
    '''Tosk Albanian''': '''als_Latn''',
    '''Sardinian''': '''srd_Latn''',
    '''Serbian''': '''srp_Cyrl''',
    '''Swati''': '''ssw_Latn''',
    '''Sundanese''': '''sun_Latn''',
    '''Swedish''': '''swe_Latn''',
    '''Swahili''': '''swh_Latn''',
    '''Silesian''': '''szl_Latn''',
    '''Tamil''': '''tam_Taml''',
    '''Tatar''': '''tat_Cyrl''',
    '''Telugu''': '''tel_Telu''',
    '''Tajik''': '''tgk_Cyrl''',
    '''Tagalog''': '''tgl_Latn''',
    '''Thai''': '''tha_Thai''',
    '''Tigrinya''': '''tir_Ethi''',
    '''Tamasheq Latin''': '''taq_Latn''',
    '''Tamasheq Tifinagh''': '''taq_Tfng''',
    '''Tok Pisin''': '''tpi_Latn''',
    '''Tswana''': '''tsn_Latn''',
    '''Tsonga''': '''tso_Latn''',
    '''Turkmen''': '''tuk_Latn''',
    '''Tumbuka''': '''tum_Latn''',
    '''Turkish''': '''tur_Latn''',
    '''Twi''': '''twi_Latn''',
    '''Central Atlas Tamazight''': '''tzm_Tfng''',
    '''Uyghur''': '''uig_Arab''',
    '''Ukrainian''': '''ukr_Cyrl''',
    '''Umbundu''': '''umb_Latn''',
    '''Urdu''': '''urd_Arab''',
    '''Northern Uzbek''': '''uzn_Latn''',
    '''Venetian''': '''vec_Latn''',
    '''Vietnamese''': '''vie_Latn''',
    '''Waray''': '''war_Latn''',
    '''Wolof''': '''wol_Latn''',
    '''Xhosa''': '''xho_Latn''',
    '''Eastern Yiddish''': '''ydd_Hebr''',
    '''Yoruba''': '''yor_Latn''',
    '''Yue Chinese''': '''yue_Hant''',
    '''Chinese Simplified''': '''zho_Hans''',
    '''Chinese Traditional''': '''zho_Hant''',
    '''Standard Malay''': '''zsm_Latn''',
    '''Zulu''': '''zul_Latn''',
}






class 	UpperCAmelCase__		(	lowerCamelCase__	):






  """simple docstring"""


  a									= "facebook/nllb-200-distilled-600M"
  a									= (
      "This is a tool that translates text from a language to another. It takes three inputs: `text`, which should "
      "be the text to translate, `src_lang`, which should be the language of the text to translate and `tgt_lang`, "
      "which should be the language for the desired ouput language. Both `src_lang` and `tgt_lang` are written in "
      "plain English, such as \'Romanian\', or \'Albanian\'. It returns the text translated in `tgt_lang`."
  )
  a									= "translator"
  a									= AutoTokenizer
  a									= AutoModelForSeqaSeqLM
  a									= LANGUAGE_CODES

  a									= ["text", "text", "text"]
  a									= ["text"]






  def 				lowercase_		(							self						:		Optional[Any]     ,       __lowerCamelCase						:		int     ,       __lowerCamelCase						:		Tuple     ,       __lowerCamelCase						:		Optional[int] )		->							Optional[Any]:




       if src_lang not in self.lang_to_code:
            raise ValueError(f'''{src_lang} is not a supported language.''' )
       if tgt_lang not in self.lang_to_code:
            raise ValueError(f'''{tgt_lang} is not a supported language.''' )
       SCREAMING_SNAKE_CASE__   =   self.lang_to_code[src_lang]
       SCREAMING_SNAKE_CASE__   =   self.lang_to_code[tgt_lang]
       return self.pre_processor._build_translation_inputs(
           __lowerCamelCase     ,       return_tensors='''pt'''     ,       src_lang=__lowerCamelCase     ,       tgt_lang=__lowerCamelCase )






  def 				lowercase_		(							self						:		Optional[int]     ,       __lowerCamelCase						:		Optional[int] )		->							Dict:
       return self.model.generate(**__lowerCamelCase )







  def 				lowercase_		(							self						:		Tuple     ,       __lowerCamelCase						:		str )		->							Dict:
       return self.post_processor.decode(outputs[0].tolist()     ,       skip_special_tokens=__lowerCamelCase )





import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
_SCREAMING_SNAKE_CASE					:  Tuple             =      data_utils.TransfoXLTokenizer
_SCREAMING_SNAKE_CASE					:  Dict             =      data_utils.TransfoXLCorpus
_SCREAMING_SNAKE_CASE					:  Union[str, Any]             =      data_utils
_SCREAMING_SNAKE_CASE					:  Any             =      data_utils
def     UpperCAmelCase_       (						_A				,		_A				,		_A				,		_A    ):





     '''simple docstring'''




     if transfo_xl_dataset_file:
          # Convert a pre-processed corpus (see original TensorFlow repo)
          with open(_A				,		'''rb'''    ) as fp:
               SCREAMING_SNAKE_CASE__   =   pickle.load(_A				,		encoding='''latin1'''    )
          # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
          SCREAMING_SNAKE_CASE__   =   pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
          print(F'''Save vocabulary to {pytorch_vocab_dump_path}'''    )
          SCREAMING_SNAKE_CASE__   =   corpus.vocab.__dict__
          torch.save(_A				,		_A    )

          SCREAMING_SNAKE_CASE__   =   corpus.__dict__
          corpus_dict_no_vocab.pop('''vocab'''				,		_A    )
          SCREAMING_SNAKE_CASE__   =   pytorch_dump_folder_path + '''/''' + CORPUS_NAME
          print(F'''Save dataset to {pytorch_dataset_dump_path}'''    )
          torch.save(_A				,		_A    )

     if tf_checkpoint_path:
          # Convert a pre-trained TensorFlow model
          SCREAMING_SNAKE_CASE__   =   os.path.abspath(_A    )
          SCREAMING_SNAKE_CASE__   =   os.path.abspath(_A    )

          print(F'''Converting Transformer XL checkpoint from {tf_path} with config at {config_path}.'''    )
          # Initialise PyTorch model
          if transfo_xl_config_file == "":
               SCREAMING_SNAKE_CASE__   =   TransfoXLConfig()
          else:
               SCREAMING_SNAKE_CASE__   =   TransfoXLConfig.from_json_file(_A    )
          print(F'''Building PyTorch model from configuration: {config}'''    )
          SCREAMING_SNAKE_CASE__   =   TransfoXLLMHeadModel(_A    )

          SCREAMING_SNAKE_CASE__   =   load_tf_weights_in_transfo_xl(_A				,		_A				,		_A    )
          # Save pytorch-model
          SCREAMING_SNAKE_CASE__   =   os.path.join(_A				,		_A    )
          SCREAMING_SNAKE_CASE__   =   os.path.join(_A				,		_A    )
          print(F'''Save PyTorch model to {os.path.abspath(_A    )}'''    )
          torch.save(model.state_dict()				,		_A    )
          print(F'''Save configuration file to {os.path.abspath(_A    )}'''    )
          with open(_A				,		'''w'''				,		encoding='''utf-8'''    ) as f:
               f.write(config.to_json_string()    )


if __name__ == "__main__":
       _SCREAMING_SNAKE_CASE					:  Any             =      argparse.ArgumentParser()
       parser.add_argument(
           '''--pytorch_dump_folder_path''',
           default=None,
           type=str,
           required=True,
           help='''Path to the folder to store the PyTorch model or dataset/vocab.''',
       )
       parser.add_argument(
           '''--tf_checkpoint_path''',
           default='''''',
           type=str,
           help='''An optional path to a TensorFlow checkpoint path to be converted.''',
       )
       parser.add_argument(
           '''--transfo_xl_config_file''',
           default='''''',
           type=str,
           help=(
               '''An optional config json file corresponding to the pre-trained BERT model. \n'''
               '''This specifies the model architecture.'''
           ),
       )
       parser.add_argument(
           '''--transfo_xl_dataset_file''',
           default='''''',
           type=str,
           help='''An optional dataset file to be converted in a vocabulary.''',
       )
       _SCREAMING_SNAKE_CASE					:  Union[str, Any]             =      parser.parse_args()
       convert_transfo_xl_checkpoint_to_pytorch(
           args.tf_checkpoint_path,
           args.transfo_xl_config_file,
           args.pytorch_dump_folder_path,
           args.transfo_xl_dataset_file,
       )


