
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_  :	Optional[int] 				=					logging.get_logger(__name__)

lowerCamelCase_  :	int 				=					"""▁"""

lowerCamelCase_  :	str 				=					{
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

lowerCamelCase_  :	Optional[int] 				=					{
    """vocab_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json""",
    },
    """spm_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model""",
    },
    """tokenizer_config_file""": {
        """facebook/m2m100_418M""": """https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json""",
        """facebook/m2m100_1.2B""": """https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json""",
    },
}

lowerCamelCase_  :	int 				=					{
    """facebook/m2m100_418M""": 10_24,
}

# fmt: off
lowerCamelCase_  :	List[str] 				=					{
    """m2m100""": ["""af""", """am""", """ar""", """ast""", """az""", """ba""", """be""", """bg""", """bn""", """br""", """bs""", """ca""", """ceb""", """cs""", """cy""", """da""", """de""", """el""", """en""", """es""", """et""", """fa""", """ff""", """fi""", """fr""", """fy""", """ga""", """gd""", """gl""", """gu""", """ha""", """he""", """hi""", """hr""", """ht""", """hu""", """hy""", """id""", """ig""", """ilo""", """is""", """it""", """ja""", """jv""", """ka""", """kk""", """km""", """kn""", """ko""", """lb""", """lg""", """ln""", """lo""", """lt""", """lv""", """mg""", """mk""", """ml""", """mn""", """mr""", """ms""", """my""", """ne""", """nl""", """no""", """ns""", """oc""", """or""", """pa""", """pl""", """ps""", """pt""", """ro""", """ru""", """sd""", """si""", """sk""", """sl""", """so""", """sq""", """sr""", """ss""", """su""", """sv""", """sw""", """ta""", """th""", """tl""", """tn""", """tr""", """uk""", """ur""", """uz""", """vi""", """wo""", """xh""", """yi""", """yo""", """zh""", """zu"""],
    """wmt21""": ["""en""", """ha""", """is""", """ja""", """cs""", """ru""", """zh""", """de"""]
}

class     _UpperCamelCase			(   _A					):






			'''simple docstring'''

			__UpperCamelCase :  str    =	VOCAB_FILES_NAMES
			__UpperCamelCase :  int    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase :  Optional[int]    =	PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase :  List[str]    =	["""input_ids""", """attention_mask"""]

			__UpperCamelCase :  List[int]    =	[]
			__UpperCamelCase :  List[int]    =	[]

			def __init__(    self	:    Optional[int]  ,		snake_case_	:    Any  ,		snake_case_	:    Optional[Any]  ,		snake_case_	:    Any=None  ,		snake_case_	:    List[str]=None  ,		snake_case_	:    Tuple="<s>"  ,		snake_case_	:    List[str]="</s>"  ,		snake_case_	:    List[str]="</s>"  ,		snake_case_	:    List[Any]="<pad>"  ,		snake_case_	:    int="<unk>"  ,		snake_case_	:    Union[str, Any]="m2m100"  ,		snake_case_	:    Optional[Dict[str, Any]] = None  ,		snake_case_	:    Optional[Any]=8  ,		**snake_case_	:    List[Any]  ,		):
							UpperCamelCase_:							Dict		=  {} if sp_model_kwargs is None else sp_model_kwargs

							UpperCamelCase_:							Tuple		=  language_codes
							UpperCamelCase_:							List[str]		=  FAIRSEQ_LANGUAGE_CODES[language_codes]
							UpperCamelCase_:							List[str]		=  {lang_code: f'''__{lang_code}__''' for lang_code in fairseq_language_code}

							UpperCamelCase_:							str		=  kwargs.get("""additional_special_tokens"""  ,		[] )
							kwargs["additional_special_tokens"] += [
							    self.get_lang_token(snake_case_ )
							    for lang_code in fairseq_language_code
							    if self.get_lang_token(snake_case_ ) not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    src_lang=snake_case_  ,		tgt_lang=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		sep_token=snake_case_  ,		unk_token=snake_case_  ,		pad_token=snake_case_  ,		language_codes=snake_case_  ,		sp_model_kwargs=self.sp_model_kwargs  ,		num_madeup_words=snake_case_  ,		**snake_case_  ,		)

							UpperCamelCase_:							Optional[Any]		=  vocab_file
							UpperCamelCase_:							Optional[int]		=  load_json(snake_case_ )
							UpperCamelCase_:							str		=  {v: k for k, v in self.encoder.items()}
							UpperCamelCase_:							str		=  spm_file
							UpperCamelCase_:							Optional[int]		=  load_spm(snake_case_  ,		self.sp_model_kwargs )

							UpperCamelCase_:							List[Any]		=  len(self.encoder )

							UpperCamelCase_:							Dict		=  {
							    self.get_lang_token(snake_case_ ): self.encoder_size + i for i, lang_code in enumerate(snake_case_ )
							}
							UpperCamelCase_:							int		=  {lang_code: self.encoder_size + i for i, lang_code in enumerate(snake_case_ )}
							UpperCamelCase_:							Optional[int]		=  {v: k for k, v in self.lang_token_to_id.items()}

							UpperCamelCase_:							Tuple		=  src_lang if src_lang is not None else """en"""
							UpperCamelCase_:							int		=  tgt_lang
							UpperCamelCase_:							str		=  self.get_lang_id(self._src_lang )
							self.set_src_lang_special_tokens(self._src_lang )

							UpperCamelCase_:							Any		=  num_madeup_words

			@property
			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							return len(self.encoder ) + len(self.lang_token_to_id )

			@property
			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							return self._src_lang

			@src_lang.setter
			def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    str ):
							UpperCamelCase_:							Any		=  new_src_lang
							self.set_src_lang_special_tokens(self._src_lang )

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		snake_case_	:    str ):
							return self.sp_model.encode(snake_case_  ,		out_type=snake_case_ )

			def  lowerCAmelCase__		(    self	:    int  ,		snake_case_	:    str ):
							if token in self.lang_token_to_id:
											return self.lang_token_to_id[token]
							return self.encoder.get(snake_case_  ,		self.encoder[self.unk_token] )

			def  lowerCAmelCase__		(    self	:    List[str]  ,		snake_case_	:    int ):
							if index in self.id_to_lang_token:
											return self.id_to_lang_token[index]
							return self.decoder.get(snake_case_  ,		self.unk_token )

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    Optional[int] ):
							UpperCamelCase_:							Optional[int]		=  []
							UpperCamelCase_:							Optional[int]		=  """"""
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															out_string += self.sp_model.decode(snake_case_ ) + token
															UpperCamelCase_:							Any		=  []
											else:
															current_sub_tokens.append(snake_case_ )
							out_string += self.sp_model.decode(snake_case_ )
							return out_string.strip()

			def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None  ,		snake_case_	:    bool = False ):

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=snake_case_  ,		token_ids_a=snake_case_  ,		already_has_special_tokens=snake_case_ )

							UpperCamelCase_:							List[Any]		=  [1] * len(self.prefix_tokens )
							UpperCamelCase_:							Union[str, Any]		=  [1] * len(self.suffix_tokens )
							if token_ids_a is None:
											return prefix_ones + ([0] * len(snake_case_ )) + suffix_ones
							return prefix_ones + ([0] * len(snake_case_ )) + ([0] * len(snake_case_ )) + suffix_ones

			def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None ):
							if token_ids_a is None:
											return self.prefix_tokens + token_ids_a + self.suffix_tokens
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens

			def  lowerCAmelCase__		(    self	:    Optional[int] ):
							UpperCamelCase_:							int		=  {self.convert_ids_to_tokens(snake_case_ ): i for i in range(self.vocab_size )}
							vocab.update(self.added_tokens_encoder )
							return vocab

			def __getstate__(    self	:    Dict ):
							UpperCamelCase_:							Tuple		=  self.__dict__.copy()
							UpperCamelCase_:							int		=  None
							return state

			def __setstate__(    self	:    str  ,		snake_case_	:    Dict ):
							UpperCamelCase_:							Optional[int]		=  d

							# for backward compatibility
							if not hasattr(self  ,		"""sp_model_kwargs""" ):
											UpperCamelCase_:							Union[str, Any]		=  {}

							UpperCamelCase_:							Any		=  load_spm(self.spm_file  ,		self.sp_model_kwargs )

			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    str  ,		snake_case_	:    Optional[str] = None ):
							UpperCamelCase_:							List[Any]		=  Path(snake_case_ )
							if not save_dir.is_dir():
											raise OSError(f'''{save_directory} should be a directory''' )
							UpperCamelCase_:							Optional[int]		=  save_dir / (
							    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""vocab_file"""]
							)
							UpperCamelCase_:							List[str]		=  save_dir / (
							    (filename_prefix + """-""" if filename_prefix else """""") + self.vocab_files_names["""spm_file"""]
							)

							save_json(self.encoder  ,		snake_case_ )

							if os.path.abspath(self.spm_file ) != os.path.abspath(snake_case_ ) and os.path.isfile(self.spm_file ):
											copyfile(self.spm_file  ,		snake_case_ )
							elif not os.path.isfile(self.spm_file ):
											with open(snake_case_  ,		"""wb""" ) as fi:
															UpperCamelCase_:							Tuple		=  self.sp_model.serialized_model_proto()
															fi.write(snake_case_ )

							return (str(snake_case_ ), str(snake_case_ ))

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    List[str]  ,		snake_case_	:    str = "en"  ,		snake_case_	:    Optional[List[str]] = None  ,		snake_case_	:    str = "ro"  ,		**snake_case_	:    List[str]  ,		):
							UpperCamelCase_:							Optional[Any]		=  src_lang
							UpperCamelCase_:							str		=  tgt_lang
							self.set_src_lang_special_tokens(self.src_lang )
							return super().prepare_seqaseq_batch(snake_case_  ,		snake_case_  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    List[str]  ,		snake_case_	:    Any  ,		snake_case_	:    Optional[str]  ,		snake_case_	:    Optional[str]  ,		**snake_case_	:    List[str] ):
							if src_lang is None or tgt_lang is None:
											raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""" )
							UpperCamelCase_:							List[Any]		=  src_lang
							UpperCamelCase_:							Union[str, Any]		=  self(snake_case_  ,		add_special_tokens=snake_case_  ,		**snake_case_ )
							UpperCamelCase_:							List[Any]		=  self.get_lang_id(snake_case_ )
							UpperCamelCase_:							Tuple		=  tgt_lang_id
							return inputs

			def  lowerCAmelCase__		(    self	:    Dict ):
							self.set_src_lang_special_tokens(self.src_lang )

			def  lowerCAmelCase__		(    self	:    List[Any] ):
							self.set_tgt_lang_special_tokens(self.tgt_lang )

			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    str ):
							UpperCamelCase_:							str		=  self.get_lang_token(snake_case_ )
							UpperCamelCase_:							Dict		=  self.lang_token_to_id[lang_token]
							UpperCamelCase_:							Optional[int]		=  [self.cur_lang_id]
							UpperCamelCase_:							Union[str, Any]		=  [self.eos_token_id]

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		snake_case_	:    str ):
							UpperCamelCase_:							Optional[int]		=  self.get_lang_token(snake_case_ )
							UpperCamelCase_:							Optional[int]		=  self.lang_token_to_id[lang_token]
							UpperCamelCase_:							List[str]		=  [self.cur_lang_id]
							UpperCamelCase_:							Dict		=  [self.eos_token_id]

			def  lowerCAmelCase__		(    self	:    str  ,		snake_case_	:    str ):
							return self.lang_code_to_token[lang]




			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    str ):
							UpperCamelCase_:							Tuple		=  self.get_lang_token(snake_case_ )
							return self.lang_token_to_id[lang_token]







def     A__   (  lowerCamelCase       ,      lowerCamelCase  )					->		sentencepiece.SentencePieceProcessor:
				UpperCamelCase_:							Union[str, Any]		=  sentencepiece.SentencePieceProcessor(**lowerCamelCase  )
				spm.Load(str(lowerCamelCase  )  )
				return spm







def     A__   (  lowerCamelCase  )					->		Union[Dict, List]:
				with open(lowerCamelCase       ,      """r"""  ) as f:
								return json.load(lowerCamelCase  )







def     A__   (  lowerCamelCase       ,      lowerCamelCase  )					->		None:
				with open(lowerCamelCase       ,      """w"""  ) as f:
								json.dump(lowerCamelCase       ,      lowerCamelCase       ,      indent=2  )








from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching







def     A__   (  )					->		Union[str, Any]:
				import os as original_os
				from os import path as original_path
				from os import rename as original_rename
				from os.path import dirname as original_dirname
				from os.path import join as original_join

				assert _test_patching.os is original_os
				assert _test_patching.path is original_path
				assert _test_patching.join is original_join

				assert _test_patching.renamed_os is original_os
				assert _test_patching.renamed_path is original_path
				assert _test_patching.renamed_join is original_join

				UpperCamelCase_:							Optional[int]		=  """__test_patch_submodule_mock__"""
				with patch_submodule(_test_patching       ,      """os.path.join"""       ,      lowerCamelCase  ):
								# Every way to access os.path.join must be patched, and the rest must stay untouched

								# check os.path.join
								assert isinstance(_test_patching.os       ,      _PatchedModuleObj  )
								assert isinstance(_test_patching.os.path       ,      _PatchedModuleObj  )
								assert _test_patching.os.path.join is mock

								# check path.join
								assert isinstance(_test_patching.path       ,      _PatchedModuleObj  )
								assert _test_patching.path.join is mock

								# check join
								assert _test_patching.join is mock

								# check that the other attributes are untouched
								assert _test_patching.os.rename is original_rename
								assert _test_patching.path.dirname is original_dirname
								assert _test_patching.os.path.dirname is original_dirname

								# Even renamed modules or objects must be patched

								# check renamed_os.path.join
								assert isinstance(_test_patching.renamed_os       ,      _PatchedModuleObj  )
								assert isinstance(_test_patching.renamed_os.path       ,      _PatchedModuleObj  )
								assert _test_patching.renamed_os.path.join is mock

								# check renamed_path.join
								assert isinstance(_test_patching.renamed_path       ,      _PatchedModuleObj  )
								assert _test_patching.renamed_path.join is mock

								# check renamed_join
								assert _test_patching.renamed_join is mock

								# check that the other attributes are untouched
								assert _test_patching.renamed_os.rename is original_rename
								assert _test_patching.renamed_path.dirname is original_dirname
								assert _test_patching.renamed_os.path.dirname is original_dirname

				# check that everthing is back to normal when the patch is over

				assert _test_patching.os is original_os
				assert _test_patching.path is original_path
				assert _test_patching.join is original_join

				assert _test_patching.renamed_os is original_os
				assert _test_patching.renamed_path is original_path
				assert _test_patching.renamed_join is original_join







def     A__   (  )					->		Union[str, Any]:
				assert _test_patching.open is open

				UpperCamelCase_:							List[Any]		=  """__test_patch_submodule_builtin_mock__"""
				# _test_patching has "open" in its globals
				assert _test_patching.open is open
				with patch_submodule(_test_patching       ,      """open"""       ,      lowerCamelCase  ):
								assert _test_patching.open is mock

				# check that everthing is back to normal when the patch is over

				assert _test_patching.open is open







def     A__   (  )					->		Optional[Any]:
				# pandas.read_csv is not present in _test_patching
				UpperCamelCase_:							Optional[Any]		=  """__test_patch_submodule_missing_mock__"""
				with patch_submodule(_test_patching       ,      """pandas.read_csv"""       ,      lowerCamelCase  ):
								pass







def     A__   (  )					->		Any:
				# builtin should always be mocked even if they're not in the globals
				# in case they're loaded at one point
				UpperCamelCase_:							List[Any]		=  """__test_patch_submodule_missing_builtin_mock__"""
				# _test_patching doesn't have "len" in its globals
				assert getattr(_test_patching       ,      """len"""       ,      lowerCamelCase  ) is None
				with patch_submodule(_test_patching       ,      """len"""       ,      lowerCamelCase  ):
								assert _test_patching.len is mock
				assert _test_patching.len is len







def     A__   (  )					->		Any:
				UpperCamelCase_:							Dict		=  """__test_patch_submodule_start_and_stop_mock__"""
				UpperCamelCase_:							List[str]		=  patch_submodule(_test_patching       ,      """open"""       ,      lowerCamelCase  )
				assert _test_patching.open is open
				patch.start()
				assert _test_patching.open is mock
				patch.stop()
				assert _test_patching.open is open







def     A__   (  )					->		List[str]:
				from os import rename as original_rename
				from os.path import dirname as original_dirname
				from os.path import join as original_join

				UpperCamelCase_:							Optional[Any]		=  """__test_patch_submodule_successive_join__"""
				UpperCamelCase_:							Any		=  """__test_patch_submodule_successive_dirname__"""
				UpperCamelCase_:							Dict		=  """__test_patch_submodule_successive_rename__"""
				assert _test_patching.os.path.join is original_join
				assert _test_patching.os.path.dirname is original_dirname
				assert _test_patching.os.rename is original_rename

				with patch_submodule(_test_patching       ,      """os.path.join"""       ,      lowerCamelCase  ):
								with patch_submodule(_test_patching       ,      """os.rename"""       ,      lowerCamelCase  ):
												with patch_submodule(_test_patching       ,      """os.path.dirname"""       ,      lowerCamelCase  ):
																assert _test_patching.os.path.join is mock_join
																assert _test_patching.os.path.dirname is mock_dirname
																assert _test_patching.os.rename is mock_rename

    # try another order
				with patch_submodule(_test_patching       ,      """os.rename"""       ,      lowerCamelCase  ):
								with patch_submodule(_test_patching       ,      """os.path.join"""       ,      lowerCamelCase  ):
												with patch_submodule(_test_patching       ,      """os.path.dirname"""       ,      lowerCamelCase  ):
																assert _test_patching.os.path.join is mock_join
																assert _test_patching.os.path.dirname is mock_dirname
																assert _test_patching.os.rename is mock_rename

				assert _test_patching.os.path.join is original_join
				assert _test_patching.os.path.dirname is original_dirname
				assert _test_patching.os.rename is original_rename







def     A__   (  )					->		Union[str, Any]:
				UpperCamelCase_:							Dict		=  """__test_patch_submodule_doesnt_exist_mock__"""
				with patch_submodule(_test_patching       ,      """__module_that_doesn_exist__.__attribute_that_doesn_exist__"""       ,      lowerCamelCase  ):
								pass
				with patch_submodule(_test_patching       ,      """os.__attribute_that_doesn_exist__"""       ,      lowerCamelCase  ):
								pass






