


"""simple docstring"""







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a__    :							Optional[int]            =					logging.get_logger(__name__)

a__    :							List[Any]            =					'▁'

a__    :							Union[str, Any]            =					{'vocab_file': 'sentencepiece.bpe.model'}

a__    :							Optional[int]            =					{
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

a__    :							Any            =					{
    'facebook/xglm-564M': 2_0_4_8,
}




class 			UpperCamelCase_     (			lowercase__):

	"""simple docstring"""





	snake_case__    :  Optional[Any]    =      VOCAB_FILES_NAMES
	snake_case__    :  str    =      PRETRAINED_VOCAB_FILES_MAP
	snake_case__    :  Optional[Any]    =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__    :  Optional[int]    =      ['input_ids', 'attention_mask']




	def __init__(			self      :    Dict      ,      UpperCAmelCase__      :    str      ,      UpperCAmelCase__      :    Optional[Any]="<s>"      ,      UpperCAmelCase__      :    List[Any]="</s>"      ,      UpperCAmelCase__      :    List[Any]="</s>"      ,      UpperCAmelCase__      :    Optional[Any]="<s>"      ,      UpperCAmelCase__      :    List[Any]="<unk>"      ,      UpperCAmelCase__      :    Optional[int]="<pad>"      ,      UpperCAmelCase__      :    Optional[Dict[str, Any]] = None      ,      **UpperCAmelCase__      :    List[Any]      ,      )	->     Optional[Any]:
						__SCREAMING_SNAKE_CASE						       =    {} if sp_model_kwargs is None else sp_model_kwargs

						# Compatibility with the original tokenizer
						__SCREAMING_SNAKE_CASE						       =    7
						__SCREAMING_SNAKE_CASE						       =    [F"""<madeupword{i}>""" for i in range(self.num_madeup_words       )]

						__SCREAMING_SNAKE_CASE						       =    kwargs.get("additional_special_tokens"      ,      []       )
						kwargs["additional_special_tokens"] += [
						    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
						]

						super().__init__(
						    bos_token=_UpperCamelCase      ,      eos_token=_UpperCamelCase      ,      unk_token=_UpperCamelCase      ,      sep_token=_UpperCamelCase      ,      cls_token=_UpperCamelCase      ,      pad_token=_UpperCamelCase      ,      sp_model_kwargs=self.sp_model_kwargs      ,      **_UpperCamelCase      ,      )

						__SCREAMING_SNAKE_CASE						       =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.Load(str(_UpperCamelCase       )       )
						__SCREAMING_SNAKE_CASE						       =    vocab_file

						# Original fairseq vocab and spm vocab must be "aligned":
						# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
						# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
						# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
						# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

						# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
						__SCREAMING_SNAKE_CASE						       =    1

						# Mimic fairseq token-to-id alignment for the first 4 token
						__SCREAMING_SNAKE_CASE						       =    {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

						__SCREAMING_SNAKE_CASE						       =    len(self.sp_model       )
						__SCREAMING_SNAKE_CASE						       =    {F"""<madeupword{i}>""": sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words       )}
						self.fairseq_tokens_to_ids.update(_UpperCamelCase       )

						__SCREAMING_SNAKE_CASE						       =    {v: k for k, v in self.fairseq_tokens_to_ids.items()}




	def __getstate__(			self      :    str       )	->     Any:
						__SCREAMING_SNAKE_CASE						       =    self.__dict__.copy()
						__SCREAMING_SNAKE_CASE						       =    None
						__SCREAMING_SNAKE_CASE						       =    self.sp_model.serialized_model_proto()
						return state




	def __setstate__(			self      :    List[str]      ,      UpperCAmelCase__      :    int       )	->     str:
						__SCREAMING_SNAKE_CASE						       =    d

						# for backward compatibility
						if not hasattr(self      ,      "sp_model_kwargs"       ):
											__SCREAMING_SNAKE_CASE						       =    {}

						__SCREAMING_SNAKE_CASE						       =    spm.SentencePieceProcessor(**self.sp_model_kwargs       )
						self.sp_model.LoadFromSerializedProto(self.sp_model_proto       )




	def   UpperCAmelCase_				(			self      :    Optional[int]      ,      UpperCAmelCase__      :    List[int]      ,      UpperCAmelCase__      :    Optional[List[int]] = None       )	->     str:

						if token_ids_a is None:
											return [self.sep_token_id] + token_ids_a
						__SCREAMING_SNAKE_CASE						       =    [self.sep_token_id]
						return sep + token_ids_a + sep + sep + token_ids_a




	def   UpperCAmelCase_				(			self      :    Dict      ,      UpperCAmelCase__      :    List[int]      ,      UpperCAmelCase__      :    Optional[List[int]] = None      ,      UpperCAmelCase__      :    bool = False       )	->     Optional[int]:

						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_UpperCamelCase      ,      token_ids_a=_UpperCamelCase      ,      already_has_special_tokens=_UpperCamelCase       )

						if token_ids_a is None:
											return [1] + ([0] * len(_UpperCamelCase       ))
						return [1] + ([0] * len(_UpperCamelCase       )) + [1, 1] + ([0] * len(_UpperCamelCase       ))




	def   UpperCAmelCase_				(			self      :    List[str]      ,      UpperCAmelCase__      :    List[int]      ,      UpperCAmelCase__      :    Optional[List[int]] = None       )	->     Optional[int]:
						__SCREAMING_SNAKE_CASE						       =    [self.sep_token_id]

						if token_ids_a is None:
											return len(sep + token_ids_a       ) * [0]
						return len(sep + token_ids_a + sep + sep + token_ids_a       ) * [0]




	@property
	def   UpperCAmelCase_				(			self      :    Any       )	->     List[str]:
						return len(self.sp_model       ) + self.fairseq_offset + self.num_madeup_words




	def   UpperCAmelCase_				(			self      :    Dict       )	->     Any:
						__SCREAMING_SNAKE_CASE						       =    {self.convert_ids_to_tokens(_UpperCamelCase       ): i for i in range(self.vocab_size       )}
						vocab.update(self.added_tokens_encoder       )
						return vocab




	def   UpperCAmelCase_				(			self      :    int      ,      UpperCAmelCase__      :    str       )	->     Optional[Any]:
						return self.sp_model.encode(_UpperCamelCase      ,      out_type=_UpperCamelCase       )




	def   UpperCAmelCase_				(			self      :    Dict      ,      UpperCAmelCase__      :    Any       )	->     Optional[int]:
						if token in self.fairseq_tokens_to_ids:
											return self.fairseq_tokens_to_ids[token]
						__SCREAMING_SNAKE_CASE						       =    self.sp_model.PieceToId(_UpperCamelCase       )

						# Need to return unknown token if the SP model returned 0
						return spm_id + self.fairseq_offset if spm_id else self.unk_token_id




	def   UpperCAmelCase_				(			self      :    Tuple      ,      UpperCAmelCase__      :    Union[str, Any]       )	->     Tuple:
						if index in self.fairseq_ids_to_tokens:
											return self.fairseq_ids_to_tokens[index]
						return self.sp_model.IdToPiece(index - self.fairseq_offset       )




	def   UpperCAmelCase_				(			self      :    Optional[int]      ,      UpperCAmelCase__      :    int       )	->     Union[str, Any]:
						__SCREAMING_SNAKE_CASE						       =    """""".join(_UpperCamelCase       ).replace(_UpperCamelCase      ,      " "       ).strip()
						return out_string




	def   UpperCAmelCase_				(			self      :    Optional[int]      ,      UpperCAmelCase__      :    str      ,      UpperCAmelCase__      :    Optional[str] = None       )	->     str:
						if not os.path.isdir(_UpperCamelCase       ):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
											return
						__SCREAMING_SNAKE_CASE						       =    os.path.join(
						    _UpperCamelCase      ,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

						if os.path.abspath(self.vocab_file       ) != os.path.abspath(_UpperCamelCase       ) and os.path.isfile(self.vocab_file       ):
											copyfile(self.vocab_file      ,      _UpperCamelCase       )
						elif not os.path.isfile(self.vocab_file       ):
											with open(_UpperCamelCase      ,      "wb"       ) as fi:
																__SCREAMING_SNAKE_CASE						       =    self.sp_model.serialized_model_proto()
																fi.write(_UpperCamelCase       )

						return (out_vocab_file,)




"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import _LazyModule


a__    :							Optional[int]            =					{'''tokenization_byt5''': ['''ByT5Tokenizer''']}


if TYPE_CHECKING:
		from .tokenization_byta import ByTaTokenizer
else:
		import sys

		a__    :							Any            =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
