






"""simple docstring"""



import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import BatchEncoding, PreTrainedTokenizer
from ...utils import logging


A__	:    Optional[int]			 = logging.get_logger(__name__)

A__	:    List[Any]			 = '▁'

A__	:    str			 = {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
    'tokenizer_config_file': 'tokenizer_config.json',
}

A__	:    Dict			 = {
    'vocab_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/vocab.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/vocab.json',
    },
    'spm_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/sentencepiece.bpe.model',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/sentencepiece.bpe.model',
    },
    'tokenizer_config_file': {
        'facebook/m2m100_418M': 'https://huggingface.co/facebook/m2m100_418M/resolve/main/tokenizer_config.json',
        'facebook/m2m100_1.2B': 'https://huggingface.co/facebook/m2m100_1.2B/resolve/main/tokenizer_config.json',
    },
}

A__	:    Dict			 = {
    'facebook/m2m100_418M': 1_024,
}

# fmt: off
A__	:    List[Any]			 = {
    'm2m100': ['af', 'am', 'ar', 'ast', 'az', 'ba', 'be', 'bg', 'bn', 'br', 'bs', 'ca', 'ceb', 'cs', 'cy', 'da', 'de', 'el', 'en', 'es', 'et', 'fa', 'ff', 'fi', 'fr', 'fy', 'ga', 'gd', 'gl', 'gu', 'ha', 'he', 'hi', 'hr', 'ht', 'hu', 'hy', 'id', 'ig', 'ilo', 'is', 'it', 'ja', 'jv', 'ka', 'kk', 'km', 'kn', 'ko', 'lb', 'lg', 'ln', 'lo', 'lt', 'lv', 'mg', 'mk', 'ml', 'mn', 'mr', 'ms', 'my', 'ne', 'nl', 'no', 'ns', 'oc', 'or', 'pa', 'pl', 'ps', 'pt', 'ro', 'ru', 'sd', 'si', 'sk', 'sl', 'so', 'sq', 'sr', 'ss', 'su', 'sv', 'sw', 'ta', 'th', 'tl', 'tn', 'tr', 'uk', 'ur', 'uz', 'vi', 'wo', 'xh', 'yi', 'yo', 'zh', 'zu'],
    'wmt21': ['en', 'ha', 'is', 'ja', 'cs', 'ru', 'zh', 'de']
}







class        lowercase__      (							snake_case__       ):
			_UpperCAmelCase		:Optional[Any]             =			VOCAB_FILES_NAMES
			_UpperCAmelCase		:List[Any]             =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCAmelCase		:Dict             =			PRETRAINED_VOCAB_FILES_MAP
			_UpperCAmelCase		:str             =			["input_ids", "attention_mask"]

			_UpperCAmelCase		:List[int]             =			[]
			_UpperCAmelCase		:List[int]             =			[]


			def __init__(				self					:  List[Any]       ,	snake_case__					:  Dict       ,	snake_case__					:  Union[str, Any]       ,	snake_case__					:  str=None       ,	snake_case__					:  List[str]=None       ,	snake_case__					:  Union[str, Any]="<s>"       ,	snake_case__					:  Dict="</s>"       ,	snake_case__					:  Dict="</s>"       ,	snake_case__					:  List[str]="<pad>"       ,	snake_case__					:  Union[str, Any]="<unk>"       ,	snake_case__					:  Optional[Any]="m2m100"       ,	snake_case__					:  Optional[Dict[str, Any]] = None       ,	snake_case__					:  Dict=8       ,	**snake_case__					:  Any       ,	):
										lowerCamelCase_	:							Union[str, Any]									={} if sp_model_kwargs is None else sp_model_kwargs

										lowerCamelCase_	:							Optional[Any]									=language_codes
										lowerCamelCase_	:							List[str]									=FAIRSEQ_LANGUAGE_CODES[language_codes]
										lowerCamelCase_	:							Any									={lang_code: F"""__{lang_code}__""" for lang_code in fairseq_language_code}

										lowerCamelCase_	:							Tuple									=kwargs.get("additional_special_tokens"       ,	[] )
										kwargs["additional_special_tokens"] += [
										    self.get_lang_token(snake_case__ )
										    for lang_code in fairseq_language_code
										    if self.get_lang_token(snake_case__ ) not in kwargs["additional_special_tokens"]
										]

										super().__init__(
										    src_lang=snake_case__       ,	tgt_lang=snake_case__       ,	bos_token=snake_case__       ,	eos_token=snake_case__       ,	sep_token=snake_case__       ,	unk_token=snake_case__       ,	pad_token=snake_case__       ,	language_codes=snake_case__       ,	sp_model_kwargs=self.sp_model_kwargs       ,	num_madeup_words=snake_case__       ,	**snake_case__       ,	)

										lowerCamelCase_	:							List[Any]									=vocab_file
										lowerCamelCase_	:							Union[str, Any]									=load_json(snake_case__ )
										lowerCamelCase_	:							Optional[int]									={v: k for k, v in self.encoder.items()}
										lowerCamelCase_	:							List[str]									=spm_file
										lowerCamelCase_	:							Optional[Any]									=load_spm(snake_case__       ,	self.sp_model_kwargs )

										lowerCamelCase_	:							Optional[Any]									=len(self.encoder )

										lowerCamelCase_	:							Dict									={
										    self.get_lang_token(snake_case__ ): self.encoder_size + i for i, lang_code in enumerate(snake_case__ )
										}
										lowerCamelCase_	:							Union[str, Any]									={lang_code: self.encoder_size + i for i, lang_code in enumerate(snake_case__ )}
										lowerCamelCase_	:							List[str]									={v: k for k, v in self.lang_token_to_id.items()}

										lowerCamelCase_	:							Optional[Any]									=src_lang if src_lang is not None else "en"
										lowerCamelCase_	:							Optional[int]									=tgt_lang
										lowerCamelCase_	:							int									=self.get_lang_id(self._src_lang )
										self.set_src_lang_special_tokens(self._src_lang )

										lowerCamelCase_	:							int									=num_madeup_words


			@property
			def      UpperCAmelCase__						(				self					:  int ):
										return len(self.encoder ) + len(self.lang_token_to_id )


			@property
			def      UpperCAmelCase__						(				self					:  str ):
										return self._src_lang


			@src_lang.setter
			def      UpperCAmelCase__						(				self					:  Dict       ,	snake_case__					:  str ):
										lowerCamelCase_	:							str									=new_src_lang
										self.set_src_lang_special_tokens(self._src_lang )


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  str ):
										return self.sp_model.encode(snake_case__       ,	out_type=snake_case__ )


			def      UpperCAmelCase__						(				self					:  Optional[int]       ,	snake_case__					:  Optional[Any] ):
										if token in self.lang_token_to_id:
																	return self.lang_token_to_id[token]
										return self.encoder.get(snake_case__       ,	self.encoder[self.unk_token] )


			def      UpperCAmelCase__						(				self					:  Any       ,	snake_case__					:  int ):
										if index in self.id_to_lang_token:
																	return self.id_to_lang_token[index]
										return self.decoder.get(snake_case__       ,	self.unk_token )


			def      UpperCAmelCase__						(				self					:  str       ,	snake_case__					:  Optional[Any] ):
										lowerCamelCase_	:							int									=[]
										lowerCamelCase_	:							Any									=""
										for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																								out_string += self.sp_model.decode(snake_case__ ) + token
																								lowerCamelCase_	:							str									=[]
																	else:
																								current_sub_tokens.append(snake_case__ )
										out_string += self.sp_model.decode(snake_case__ )
										return out_string.strip()


			def      UpperCAmelCase__						(				self					:  Optional[Any]       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None       ,	snake_case__					:  bool = False ):

										if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=snake_case__       ,	token_ids_a=snake_case__       ,	already_has_special_tokens=snake_case__ )

										lowerCamelCase_	:							Union[str, Any]									=[1] * len(self.prefix_tokens )
										lowerCamelCase_	:							Tuple									=[1] * len(self.suffix_tokens )
										if token_ids_a is None:
																	return prefix_ones + ([0] * len(snake_case__ )) + suffix_ones
										return prefix_ones + ([0] * len(snake_case__ )) + ([0] * len(snake_case__ )) + suffix_ones


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  List[int]       ,	snake_case__					:  Optional[List[int]] = None ):
										if token_ids_a is None:
																	return self.prefix_tokens + token_ids_a + self.suffix_tokens
										# We don't expect to process pairs, but leave the pair logic for API consistency
										return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens


			def      UpperCAmelCase__						(				self					:  Union[str, Any] ):
										lowerCamelCase_	:							Any									={self.convert_ids_to_tokens(snake_case__ ): i for i in range(self.vocab_size )}
										vocab.update(self.added_tokens_encoder )
										return vocab


			def __getstate__(				self					:  List[Any] ):
										lowerCamelCase_	:							Dict									=self.__dict__.copy()
										lowerCamelCase_	:							Optional[int]									=None
										return state


			def __setstate__(				self					:  Dict       ,	snake_case__					:  Dict ):
										lowerCamelCase_	:							Optional[Any]									=d

										# for backward compatibility
										if not hasattr(self       ,	"sp_model_kwargs" ):
																	lowerCamelCase_	:							int									={}

										lowerCamelCase_	:							int									=load_spm(self.spm_file       ,	self.sp_model_kwargs )


			def      UpperCAmelCase__						(				self					:  str       ,	snake_case__					:  str       ,	snake_case__					:  Optional[str] = None ):
										lowerCamelCase_	:							int									=Path(snake_case__ )
										if not save_dir.is_dir():
																	raise OSError(F"""{save_directory} should be a directory""" )
										lowerCamelCase_	:							int									=save_dir / (
										    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
										)
										lowerCamelCase_	:							Optional[int]									=save_dir / (
										    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
										)

										save_json(self.encoder       ,	snake_case__ )

										if os.path.abspath(self.spm_file ) != os.path.abspath(snake_case__ ) and os.path.isfile(self.spm_file ):
																	copyfile(self.spm_file       ,	snake_case__ )
										elif not os.path.isfile(self.spm_file ):
																	with open(snake_case__       ,	"wb" ) as fi:
																								lowerCamelCase_	:							Dict									=self.sp_model.serialized_model_proto()
																								fi.write(snake_case__ )

										return (str(snake_case__ ), str(snake_case__ ))


			def      UpperCAmelCase__						(				self					:  List[Any]       ,	snake_case__					:  List[str]       ,	snake_case__					:  str = "en"       ,	snake_case__					:  Optional[List[str]] = None       ,	snake_case__					:  str = "ro"       ,	**snake_case__					:  Optional[Any]       ,	):
										lowerCamelCase_	:							List[str]									=src_lang
										lowerCamelCase_	:							Optional[int]									=tgt_lang
										self.set_src_lang_special_tokens(self.src_lang )
										return super().prepare_seqaseq_batch(snake_case__       ,	snake_case__       ,	**snake_case__ )


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  Any       ,	snake_case__					:  Optional[str]       ,	snake_case__					:  Optional[str]       ,	**snake_case__					:  str ):
										if src_lang is None or tgt_lang is None:
																	raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model" )
										lowerCamelCase_	:							Optional[int]									=src_lang
										lowerCamelCase_	:							Optional[int]									=self(snake_case__       ,	add_special_tokens=snake_case__       ,	**snake_case__ )
										lowerCamelCase_	:							str									=self.get_lang_id(snake_case__ )
										lowerCamelCase_	:							int									=tgt_lang_id
										return inputs


			def      UpperCAmelCase__						(				self					:  Tuple ):
										self.set_src_lang_special_tokens(self.src_lang )


			def      UpperCAmelCase__						(				self					:  int ):
										self.set_tgt_lang_special_tokens(self.tgt_lang )


			def      UpperCAmelCase__						(				self					:  int       ,	snake_case__					:  str ):
										lowerCamelCase_	:							Union[str, Any]									=self.get_lang_token(snake_case__ )
										lowerCamelCase_	:							str									=self.lang_token_to_id[lang_token]
										lowerCamelCase_	:							int									=[self.cur_lang_id]
										lowerCamelCase_	:							List[str]									=[self.eos_token_id]


			def      UpperCAmelCase__						(				self					:  Tuple       ,	snake_case__					:  str ):
										lowerCamelCase_	:							Optional[Any]									=self.get_lang_token(snake_case__ )
										lowerCamelCase_	:							Optional[int]									=self.lang_token_to_id[lang_token]
										lowerCamelCase_	:							int									=[self.cur_lang_id]
										lowerCamelCase_	:							Union[str, Any]									=[self.eos_token_id]


			def      UpperCAmelCase__						(				self					:  str       ,	snake_case__					:  str ):
										return self.lang_code_to_token[lang]


			def      UpperCAmelCase__						(				self					:  Union[str, Any]       ,	snake_case__					:  str ):
										lowerCamelCase_	:							Dict									=self.get_lang_token(snake_case__ )
										return self.lang_token_to_id[lang_token]


def 			_snake_case    (					lowerCamelCase__     :   str    ,						lowerCamelCase__     :   Dict[str, Any]  )    ->     sentencepiece.SentencePieceProcessor:
							lowerCamelCase_	:							Union[str, Any]									=sentencepiece.SentencePieceProcessor(**lowerCamelCase__  )
							spm.Load(str(lowerCamelCase__  )  )
							return spm

def 			_snake_case    (					lowerCamelCase__     :   str  )    ->     Union[Dict, List]:
							with open(lowerCamelCase__    ,						"r"  ) as f:
														return json.load(lowerCamelCase__  )

def 			_snake_case    (					lowerCamelCase__     :   Optional[int]    ,						lowerCamelCase__     :   str  )    ->     None:
							with open(lowerCamelCase__    ,						"w"  ) as f:
														json.dump(lowerCamelCase__    ,						lowerCamelCase__    ,						indent=2  )











"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


A__	:    Optional[int]			 = {
    'configuration_speech_to_text': ['SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Speech2TextConfig'],
    'processing_speech_to_text': ['Speech2TextProcessor'],
}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Dict			 = ['Speech2TextTokenizer']

try:
						if not is_speech_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Optional[Any]			 = ['Speech2TextFeatureExtractor']

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Dict			 = [
						    'TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'TFSpeech2TextForConditionalGeneration',
						    'TFSpeech2TextModel',
						    'TFSpeech2TextPreTrainedModel',
						]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__	:    Optional[int]			 = [
						    'SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
						    'Speech2TextForConditionalGeneration',
						    'Speech2TextModel',
						    'Speech2TextPreTrainedModel',
						]


if TYPE_CHECKING:
						from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
						from .processing_speech_to_text import SpeechaTextProcessor

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_speech_to_text import SpeechaTextTokenizer

						try:
												if not is_speech_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_speech_to_text import (
												    TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFSpeechaTextForConditionalGeneration,
												    TFSpeechaTextModel,
												    TFSpeechaTextPreTrainedModel,
												)

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_speech_to_text import (
												    SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
												    SpeechaTextForConditionalGeneration,
												    SpeechaTextModel,
												    SpeechaTextPreTrainedModel,
												)

else:
						import sys

						A__	:    List[Any]			 = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



