


'''simple docstring'''
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case											=					logging.get_logger(__name__)

__snake_case											=					'''▁'''

__snake_case											=					{
    '''vocab_file''': '''vocab.json''',
    '''spm_file''': '''sentencepiece.bpe.model''',
}

__snake_case											=					{
    '''vocab_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'''
        ),
    },
    '''spm_file''': {
        '''facebook/s2t-small-librispeech-asr''': (
            '''https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'''
        )
    },
}

__snake_case											=					{
    '''facebook/s2t-small-librispeech-asr''': 1024,
}

__snake_case											=					['''pt''', '''fr''', '''ru''', '''nl''', '''ro''', '''it''', '''es''', '''de''']

__snake_case											=					{'''mustc''': MUSTC_LANGS}
class        lowercase     (							A__ ):

		"""simple docstring"""

		_a   			=    VOCAB_FILES_NAMES
		_a   			=    PRETRAINED_VOCAB_FILES_MAP
		_a   			=    MAX_MODEL_INPUT_SIZES
		_a   			=    ['input_ids', 'attention_mask']

		_a   			=    []




		def __init__(		self   ,       UpperCamelCase_   ,       UpperCamelCase_   ,       UpperCamelCase_="<s>"   ,       UpperCamelCase_="</s>"   ,       UpperCamelCase_="<pad>"   ,       UpperCamelCase_="<unk>"   ,       UpperCamelCase_=False   ,       UpperCamelCase_=False   ,       UpperCamelCase_=None   ,       UpperCamelCase_=None   ,       UpperCamelCase_ = None   ,       **UpperCamelCase_   ,       ):

							'''simple docstring'''


							UpperCamelCase__       :str							 =			{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=UpperCamelCase_   ,       eos_token=UpperCamelCase_   ,       unk_token=UpperCamelCase_   ,       pad_token=UpperCamelCase_   ,       do_upper_case=UpperCamelCase_   ,       do_lower_case=UpperCamelCase_   ,       tgt_lang=UpperCamelCase_   ,       lang_codes=UpperCamelCase_   ,       sp_model_kwargs=self.sp_model_kwargs   ,       **UpperCamelCase_   ,       )
							UpperCamelCase__       :Dict							 =			do_upper_case
							UpperCamelCase__       :Tuple							 =			do_lower_case

							UpperCamelCase__       :Union[str, Any]							 =			load_json(UpperCamelCase_							)
							UpperCamelCase__       :Dict							 =			{v: k for k, v in self.encoder.items()}
							UpperCamelCase__       :Union[str, Any]							 =			spm_file
							UpperCamelCase__       :Optional[Any]							 =			load_spm(UpperCamelCase_   ,       self.sp_model_kwargs							)

							if lang_codes is not None:
												UpperCamelCase__       :str							 =			lang_codes
												UpperCamelCase__       :Optional[int]							 =			LANGUAGES[lang_codes]
												UpperCamelCase__       :Optional[Any]							 =			[F'''<lang:{lang}>''' for lang in self.langs]
												UpperCamelCase__       :Tuple							 =			{lang: self.sp_model.PieceToId(F'''<lang:{lang}>'''							) for lang in self.langs}

												UpperCamelCase__       :Optional[int]							 =			self.lang_tokens
												UpperCamelCase__       :str							 =			tgt_lang if tgt_lang is not None else self.langs[0]

												self.set_tgt_lang_special_tokens(self._tgt_lang							)
							else:
												UpperCamelCase__       :List[Any]							 =			{}




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return len(self.encoder							)




		@property
		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							return self._tgt_lang




		@tgt_lang.setter
		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			new_tgt_lang
							self.set_tgt_lang_special_tokens(UpperCamelCase_							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			self.lang_code_to_id[tgt_lang]
							UpperCamelCase__       :List[Any]							 =			[lang_code_id]




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.sp_model.encode(UpperCamelCase_   ,       out_type=UpperCamelCase_							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.encoder.get(UpperCamelCase_   ,       self.encoder[self.unk_token]							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							return self.decoder.get(UpperCamelCase_   ,       self.unk_token							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :List[Any]							 =			[]
							UpperCamelCase__       :List[Any]							 =			''''''
							for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																	UpperCamelCase__       :Dict							 =			self.sp_model.decode(UpperCamelCase_							)
																	out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
																	UpperCamelCase__       :Dict							 =			[]
												else:
																	current_sub_tokens.append(UpperCamelCase_							)
							UpperCamelCase__       :Any							 =			self.sp_model.decode(UpperCamelCase_							)
							out_string += decoded.upper() if self.do_upper_case else decoded
							return out_string.strip()




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_=None							):

							'''simple docstring'''


							if token_ids_a is None:
												return self.prefix_tokens + token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None   ,       UpperCamelCase_ = False							):

							'''simple docstring'''


							if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCamelCase_   ,       token_ids_a=UpperCamelCase_   ,       already_has_special_tokens=UpperCamelCase_							)

							UpperCamelCase__       :Optional[Any]							 =			[1] * len(self.prefix_tokens							)
							UpperCamelCase__       :List[str]							 =			[1]
							if token_ids_a is None:
												return prefix_ones + ([0] * len(UpperCamelCase_							)) + suffix_ones
							return prefix_ones + ([0] * len(UpperCamelCase_							)) + ([0] * len(UpperCamelCase_							)) + suffix_ones




		def    lowerCAmelCase__    (		self							):

							'''simple docstring'''


							UpperCamelCase__       :Tuple							 =			self.encoder.copy()
							vocab.update(self.added_tokens_encoder							)
							return vocab




		def __getstate__(		self							):

							'''simple docstring'''


							UpperCamelCase__       :Union[str, Any]							 =			self.__dict__.copy()
							UpperCamelCase__       :int							 =			None
							return state




		def __setstate__(		self   ,       UpperCamelCase_							):

							'''simple docstring'''


							UpperCamelCase__       :Optional[Any]							 =			d

							# for backward compatibility
							if not hasattr(self   ,       '''sp_model_kwargs'''							):
												UpperCamelCase__       :Any							 =			{}

							UpperCamelCase__       :str							 =			load_spm(self.spm_file   ,       self.sp_model_kwargs							)




		def    lowerCAmelCase__    (		self   ,       UpperCamelCase_   ,       UpperCamelCase_ = None							):

							'''simple docstring'''


							UpperCamelCase__       :Any							 =			Path(UpperCamelCase_							)
							assert save_dir.is_dir(), F'''{save_directory} should be a directory'''
							UpperCamelCase__       :List[str]							 =			save_dir / (
							    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''vocab_file''']
							)
							UpperCamelCase__       :Tuple							 =			save_dir / (
							    (filename_prefix + '''-''' if filename_prefix else '''''') + self.vocab_files_names['''spm_file''']
							)

							save_json(self.encoder   ,       UpperCamelCase_							)

							if os.path.abspath(self.spm_file							) != os.path.abspath(UpperCamelCase_							) and os.path.isfile(self.spm_file							):
												copyfile(self.spm_file   ,       UpperCamelCase_							)
							elif not os.path.isfile(self.spm_file							):
												with open(UpperCamelCase_   ,       '''wb'''							) as fi:
																	UpperCamelCase__       :Optional[Any]							 =			self.sp_model.serialized_model_proto()
																	fi.write(UpperCamelCase_							)

							return (str(UpperCamelCase_							), str(UpperCamelCase_							))



def       a				(      __a			, __a       )  ->     sentencepiece.SentencePieceProcessor:



					'''simple docstring'''






					UpperCamelCase__       :Any							 =			sentencepiece.SentencePieceProcessor(**__a       )
					spm.Load(str(__a       )       )
					return spm


def       a				(      __a       )  ->     Union[Dict, List]:



					'''simple docstring'''






					with open(__a			, '''r'''       ) as f:
										return json.load(__a       )







def       a				(      __a			, __a       )  ->     None:



					'''simple docstring'''






					with open(__a			, '''w'''       ) as f:
										json.dump(__a			, __a			, indent=2       )



'''simple docstring'''
from math import ceil


def       a				(      __a			, __a       )  ->     Any:



					'''simple docstring'''






					UpperCamelCase__       :str							 =			list(range(0			, __a       )       )

					UpperCamelCase__       :Optional[int]							 =			[item for sublist in list(device_map.values()       ) for item in sublist]

					# Duplicate check
					UpperCamelCase__       :Optional[int]							 =			[]
					for i in device_map_blocks:
										if device_map_blocks.count(__a       ) > 1 and i not in duplicate_blocks:
															duplicate_blocks.append(__a       )
    # Missing blocks
					UpperCamelCase__       :List[str]							 =			[i for i in blocks if i not in device_map_blocks]
					UpperCamelCase__       :Optional[Any]							 =			[i for i in device_map_blocks if i not in blocks]

					if len(__a       ) != 0:
										raise ValueError(
										    '''Duplicate attention blocks specified in device_map. Attention blocks must be specified to one device.'''
										    ''' These attention blocks were specified more than once: ''' + str(__a       )       )
					if len(__a       ) != 0:
										raise ValueError(
										    '''There are attention blocks for this model that are not specified in the device_map. Add these attention '''
										    '''blocks to a device on the device_map: ''' + str(__a       )       )
					if len(__a       ) != 0:
										raise ValueError(
										    '''The device_map contains more attention blocks than this model has. Remove these from the device_map:'''
										    + str(__a       )       )







def       a				(      __a			, __a       )  ->     Tuple:



					'''simple docstring'''






					UpperCamelCase__       :Optional[Any]							 =			list(range(__a       )       )
					UpperCamelCase__       :Any							 =			int(ceil(n_layers / len(__a       )       )       )
					UpperCamelCase__       :List[Any]							 =			[layers[i : i + n_blocks] for i in range(0			, __a			, __a       )]

					return dict(zip(__a			, __a       )       )