



'''simple docstring'''
import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_			:     List[str]  	=  logging.get_logger(__name__)

snake_case_			:     int  	=  '▁'

snake_case_			:     Any  	=  {
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
}

snake_case_			:     Any  	=  {
    'vocab_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'
        ),
    },
    'spm_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'
        )
    },
}

snake_case_			:     int  	=  {
    'facebook/s2t-small-librispeech-asr': 1024,
}

snake_case_			:     Optional[int]  	=  ['pt', 'fr', 'ru', 'nl', 'ro', 'it', 'es', 'de']

snake_case_			:     Optional[int]  	=  {'mustc': MUSTC_LANGS}






class 							lowercase__     (							lowercase       ):
	lowercase__      =      VOCAB_FILES_NAMES
	lowercase__      =      PRETRAINED_VOCAB_FILES_MAP
	lowercase__      =      MAX_MODEL_INPUT_SIZES
	lowercase__      =      ["""input_ids""", """attention_mask"""]

	lowercase__      =      []





	def __init__(			self :    Union[str, Any]			,lowerCamelCase__ :    int			,lowerCamelCase__ :    str			,lowerCamelCase__ :    Optional[Any]="<s>"			,lowerCamelCase__ :    List[str]="</s>"			,lowerCamelCase__ :    Optional[Any]="<pad>"			,lowerCamelCase__ :    Optional[int]="<unk>"			,lowerCamelCase__ :    Tuple=False			,lowerCamelCase__ :    Dict=False			,lowerCamelCase__ :    Tuple=None			,lowerCamelCase__ :    List[str]=None			,lowerCamelCase__ :    Optional[Dict[str, Any]] = None			,**lowerCamelCase__ :    Any			,):







							'''simple docstring'''
							_UpperCamelCase :		int			    =       {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    bos_token=lowerCamelCase__			,eos_token=lowerCamelCase__			,unk_token=lowerCamelCase__			,pad_token=lowerCamelCase__			,do_upper_case=lowerCamelCase__			,do_lower_case=lowerCamelCase__			,tgt_lang=lowerCamelCase__			,lang_codes=lowerCamelCase__			,sp_model_kwargs=self.sp_model_kwargs			,**lowerCamelCase__			,)
							_UpperCamelCase :		Any			    =       do_upper_case
							_UpperCamelCase :		List[str]			    =       do_lower_case

							_UpperCamelCase :		Union[str, Any]			    =       load_json(lowerCamelCase__			)
							_UpperCamelCase :		Optional[Any]			    =       {v: k for k, v in self.encoder.items()}
							_UpperCamelCase :		Tuple			    =       spm_file
							_UpperCamelCase :		Optional[Any]			    =       load_spm(lowerCamelCase__			,self.sp_model_kwargs			)

							if lang_codes is not None:
													_UpperCamelCase :		Tuple			    =       lang_codes
													_UpperCamelCase :		str			    =       LANGUAGES[lang_codes]
													_UpperCamelCase :		List[str]			    =       [F'<lang:{lang}>' for lang in self.langs]
													_UpperCamelCase :		Union[str, Any]			    =       {lang: self.sp_model.PieceToId(F'<lang:{lang}>'			) for lang in self.langs}

													_UpperCamelCase :		Dict			    =       self.lang_tokens
													_UpperCamelCase :		str			    =       tgt_lang if tgt_lang is not None else self.langs[0]

													self.set_tgt_lang_special_tokens(self._tgt_lang			)
							else:
													_UpperCamelCase :		Any			    =       {}





	@property
	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							return len(self.encoder			)





	@property
	def      UpperCamelCase_  (			self :    Optional[int]			):







							'''simple docstring'''
							return self._tgt_lang





	@tgt_lang.setter
	def      UpperCamelCase_  (			self :    str			,lowerCamelCase__ :    List[str]			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       new_tgt_lang
							self.set_tgt_lang_special_tokens(lowerCamelCase__			)





	def      UpperCamelCase_  (			self :    Any			,lowerCamelCase__ :    str			):







							'''simple docstring'''
							_UpperCamelCase :		Union[str, Any]			    =       self.lang_code_to_id[tgt_lang]
							_UpperCamelCase :		List[str]			    =       [lang_code_id]





	def      UpperCamelCase_  (			self :    int			,lowerCamelCase__ :    str			):







							'''simple docstring'''
							return self.sp_model.encode(lowerCamelCase__			,out_type=lowerCamelCase__			)





	def      UpperCamelCase_  (			self :    List[str]			,lowerCamelCase__ :    Tuple			):







							'''simple docstring'''
							return self.encoder.get(lowerCamelCase__			,self.encoder[self.unk_token]			)





	def      UpperCamelCase_  (			self :    Optional[Any]			,lowerCamelCase__ :    int			):







							'''simple docstring'''
							return self.decoder.get(lowerCamelCase__			,self.unk_token			)





	def      UpperCamelCase_  (			self :    Tuple			,lowerCamelCase__ :    List[str]			):







							'''simple docstring'''
							_UpperCamelCase :		int			    =       []
							_UpperCamelCase :		Optional[int]			    =       ''
							for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																			_UpperCamelCase :		List[str]			    =       self.sp_model.decode(lowerCamelCase__			)
																			out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
																			_UpperCamelCase :		List[str]			    =       []
													else:
																			current_sub_tokens.append(lowerCamelCase__			)
							_UpperCamelCase :		Optional[int]			    =       self.sp_model.decode(lowerCamelCase__			)
							out_string += decoded.upper() if self.do_upper_case else decoded
							return out_string.strip()





	def      UpperCamelCase_  (			self :    Optional[Any]			,lowerCamelCase__ :    Tuple			,lowerCamelCase__ :    List[str]=None			):







							'''simple docstring'''
							if token_ids_a is None:
													return self.prefix_tokens + token_ids_a + [self.eos_token_id]
							# We don't expect to process pairs, but leave the pair logic for API consistency
							return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]





	def      UpperCamelCase_  (			self :    int			,lowerCamelCase__ :    List[int]			,lowerCamelCase__ :    Optional[List[int]] = None			,lowerCamelCase__ :    bool = False			):







							'''simple docstring'''

							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowerCamelCase__			,token_ids_a=lowerCamelCase__			,already_has_special_tokens=lowerCamelCase__			)

							_UpperCamelCase :		int			    =       [1] * len(self.prefix_tokens			)
							_UpperCamelCase :		List[str]			    =       [1]
							if token_ids_a is None:
													return prefix_ones + ([0] * len(lowerCamelCase__			)) + suffix_ones
							return prefix_ones + ([0] * len(lowerCamelCase__			)) + ([0] * len(lowerCamelCase__			)) + suffix_ones





	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[int]			    =       self.encoder.copy()
							vocab.update(self.added_tokens_encoder			)
							return vocab





	def __getstate__(			self :    Union[str, Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[Any]			    =       self.__dict__.copy()
							_UpperCamelCase :		Optional[int]			    =       None
							return state





	def __setstate__(			self :    Optional[int]			,lowerCamelCase__ :    Dict			):







							'''simple docstring'''
							_UpperCamelCase :		List[Any]			    =       d

							# for backward compatibility
							if not hasattr(self			,'sp_model_kwargs'			):
													_UpperCamelCase :		str			    =       {}

							_UpperCamelCase :		str			    =       load_spm(self.spm_file			,self.sp_model_kwargs			)





	def      UpperCamelCase_  (			self :    Dict			,lowerCamelCase__ :    str			,lowerCamelCase__ :    Optional[str] = None			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[int]			    =       Path(lowerCamelCase__			)
							assert save_dir.is_dir(), F'{save_directory} should be a directory'
							_UpperCamelCase :		List[str]			    =       save_dir / (
							    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
							)
							_UpperCamelCase :		Tuple			    =       save_dir / (
							    (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
							)

							save_json(self.encoder			,lowerCamelCase__			)

							if os.path.abspath(self.spm_file			) != os.path.abspath(lowerCamelCase__			) and os.path.isfile(self.spm_file			):
													copyfile(self.spm_file			,lowerCamelCase__			)
							elif not os.path.isfile(self.spm_file			):
													with open(lowerCamelCase__			,'wb'			) as fi:
																			_UpperCamelCase :		Dict			    =       self.sp_model.serialized_model_proto()
																			fi.write(lowerCamelCase__			)

							return (str(lowerCamelCase__			), str(lowerCamelCase__			))






def A__   (					UpperCAmelCase_  ,  UpperCAmelCase_							):
						_UpperCamelCase :		str			    =       sentencepiece.SentencePieceProcessor(**UpperCAmelCase_							)
						spm.Load(str(UpperCAmelCase_							)							)
						return spm






def A__   (					UpperCAmelCase_							):
						with open(UpperCAmelCase_  ,  'r'							) as f:
												return json.load(UpperCAmelCase_							)






def A__   (					UpperCAmelCase_  ,  UpperCAmelCase_							):
						with open(UpperCAmelCase_  ,  'w'							) as f:
												json.dump(UpperCAmelCase_  ,  UpperCAmelCase_  ,  indent=2							)







'''simple docstring'''
from torch import nn






def A__   (					UpperCAmelCase_							):
						if act_fn in ["swish", "silu"]:
												return nn.SiLU()
						elif act_fn == "mish":
												return nn.Mish()
						elif act_fn == "gelu":
												return nn.GELU()
						else:
												raise ValueError(f'Unsupported activation function: {act_fn}'							)


