



'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case_			:     int  	=  logging.get_logger(__name__)

snake_case_			:     Union[str, Any]  	=  '▁'

snake_case_			:     Dict  	=  {'vocab_file': 'sentencepiece.bpe.model'}

snake_case_			:     Union[str, Any]  	=  {
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

snake_case_			:     str  	=  {
    'facebook/xglm-564M': 2048,
}






class 							lowercase__     (							SCREAMING_SNAKE_CASE__       ):
	lowercase__      =      VOCAB_FILES_NAMES
	lowercase__      =      PRETRAINED_VOCAB_FILES_MAP
	lowercase__      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase__      =      ["""input_ids""", """attention_mask"""]





	def __init__(			self :    str			,lowerCamelCase__ :    Tuple			,lowerCamelCase__ :    Dict="<s>"			,lowerCamelCase__ :    Union[str, Any]="</s>"			,lowerCamelCase__ :    Optional[Any]="</s>"			,lowerCamelCase__ :    str="<s>"			,lowerCamelCase__ :    Tuple="<unk>"			,lowerCamelCase__ :    Optional[int]="<pad>"			,lowerCamelCase__ :    Optional[Dict[str, Any]] = None			,**lowerCamelCase__ :    List[Any]			,):







							'''simple docstring'''
							_UpperCamelCase :		str			    =       {} if sp_model_kwargs is None else sp_model_kwargs

							# Compatibility with the original tokenizer
							_UpperCamelCase :		int			    =       7
							_UpperCamelCase :		List[str]			    =       [F'<madeupword{i}>' for i in range(self.num_madeup_words			)]

							_UpperCamelCase :		Any			    =       kwargs.get('additional_special_tokens'			,[]			)
							kwargs["additional_special_tokens"] += [
							    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
							]

							super().__init__(
							    bos_token=lowerCamelCase__			,eos_token=lowerCamelCase__			,unk_token=lowerCamelCase__			,sep_token=lowerCamelCase__			,cls_token=lowerCamelCase__			,pad_token=lowerCamelCase__			,sp_model_kwargs=self.sp_model_kwargs			,**lowerCamelCase__			,)

							_UpperCamelCase :		List[Any]			    =       spm.SentencePieceProcessor(**self.sp_model_kwargs			)
							self.sp_model.Load(str(lowerCamelCase__			)			)
							_UpperCamelCase :		str			    =       vocab_file

							# Original fairseq vocab and spm vocab must be "aligned":
							# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
							# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
							# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
							# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

							# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
							_UpperCamelCase :		Tuple			    =       1

							# Mimic fairseq token-to-id alignment for the first 4 token
							_UpperCamelCase :		Union[str, Any]			    =       {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

							_UpperCamelCase :		List[Any]			    =       len(self.sp_model			)
							_UpperCamelCase :		List[str]			    =       {F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words			)}
							self.fairseq_tokens_to_ids.update(lowerCamelCase__			)

							_UpperCamelCase :		Union[str, Any]			    =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}





	def __getstate__(			self :    Optional[int]			):







							'''simple docstring'''
							_UpperCamelCase :		List[Any]			    =       self.__dict__.copy()
							_UpperCamelCase :		List[str]			    =       None
							_UpperCamelCase :		str			    =       self.sp_model.serialized_model_proto()
							return state





	def __setstate__(			self :    List[str]			,lowerCamelCase__ :    str			):







							'''simple docstring'''
							_UpperCamelCase :		Union[str, Any]			    =       d

							# for backward compatibility
							if not hasattr(self			,'sp_model_kwargs'			):
													_UpperCamelCase :		Any			    =       {}

							_UpperCamelCase :		str			    =       spm.SentencePieceProcessor(**self.sp_model_kwargs			)
							self.sp_model.LoadFromSerializedProto(self.sp_model_proto			)





	def      UpperCamelCase_  (			self :    Tuple			,lowerCamelCase__ :    List[int]			,lowerCamelCase__ :    Optional[List[int]] = None			):







							'''simple docstring'''

							if token_ids_a is None:
													return [self.sep_token_id] + token_ids_a
							_UpperCamelCase :		Union[str, Any]			    =       [self.sep_token_id]
							return sep + token_ids_a + sep + sep + token_ids_a





	def      UpperCamelCase_  (			self :    List[str]			,lowerCamelCase__ :    List[int]			,lowerCamelCase__ :    Optional[List[int]] = None			,lowerCamelCase__ :    bool = False			):







							'''simple docstring'''

							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowerCamelCase__			,token_ids_a=lowerCamelCase__			,already_has_special_tokens=lowerCamelCase__			)

							if token_ids_a is None:
													return [1] + ([0] * len(lowerCamelCase__			))
							return [1] + ([0] * len(lowerCamelCase__			)) + [1, 1] + ([0] * len(lowerCamelCase__			))





	def      UpperCamelCase_  (			self :    List[Any]			,lowerCamelCase__ :    List[int]			,lowerCamelCase__ :    Optional[List[int]] = None			):







							'''simple docstring'''
							_UpperCamelCase :		List[str]			    =       [self.sep_token_id]

							if token_ids_a is None:
													return len(sep + token_ids_a			) * [0]
							return len(sep + token_ids_a + sep + sep + token_ids_a			) * [0]





	@property
	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							return len(self.sp_model			) + self.fairseq_offset + self.num_madeup_words





	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							_UpperCamelCase :		Union[str, Any]			    =       {self.convert_ids_to_tokens(lowerCamelCase__			): i for i in range(self.vocab_size			)}
							vocab.update(self.added_tokens_encoder			)
							return vocab





	def      UpperCamelCase_  (			self :    Optional[Any]			,lowerCamelCase__ :    str			):







							'''simple docstring'''
							return self.sp_model.encode(lowerCamelCase__			,out_type=lowerCamelCase__			)





	def      UpperCamelCase_  (			self :    str			,lowerCamelCase__ :    List[Any]			):







							'''simple docstring'''
							if token in self.fairseq_tokens_to_ids:
													return self.fairseq_tokens_to_ids[token]
							_UpperCamelCase :		List[str]			    =       self.sp_model.PieceToId(lowerCamelCase__			)

							# Need to return unknown token if the SP model returned 0
							return spm_id + self.fairseq_offset if spm_id else self.unk_token_id





	def      UpperCamelCase_  (			self :    List[Any]			,lowerCamelCase__ :    List[Any]			):







							'''simple docstring'''
							if index in self.fairseq_ids_to_tokens:
													return self.fairseq_ids_to_tokens[index]
							return self.sp_model.IdToPiece(index - self.fairseq_offset			)





	def      UpperCamelCase_  (			self :    Any			,lowerCamelCase__ :    List[Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Optional[int]			    =       ''.join(lowerCamelCase__			).replace(lowerCamelCase__			,' '			).strip()
							return out_string





	def      UpperCamelCase_  (			self :    List[str]			,lowerCamelCase__ :    str			,lowerCamelCase__ :    Optional[str] = None			):







							'''simple docstring'''
							if not os.path.isdir(lowerCamelCase__			):
													logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
													return
							_UpperCamelCase :		Union[str, Any]			    =       os.path.join(
							    lowerCamelCase__			,(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

							if os.path.abspath(self.vocab_file			) != os.path.abspath(lowerCamelCase__			) and os.path.isfile(self.vocab_file			):
													copyfile(self.vocab_file			,lowerCamelCase__			)
							elif not os.path.isfile(self.vocab_file			):
													with open(lowerCamelCase__			,'wb'			) as fi:
																			_UpperCamelCase :		str			    =       self.sp_model.serialized_model_proto()
																			fi.write(lowerCamelCase__			)

							return (out_vocab_file,)







'''simple docstring'''
from abc import ABC, abstractmethod
from argparse import ArgumentParser






class 							lowercase__     (							lowercase       ):





	@staticmethod
	@abstractmethod
	def      UpperCamelCase_  (			lowerCamelCase__ :    ArgumentParser			):







							'''simple docstring'''
							raise NotImplementedError()





	@abstractmethod
	def      UpperCamelCase_  (			self :    Any			):







							'''simple docstring'''
							raise NotImplementedError()


