





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast


class 						lowerCamelCase       (unittest.TestCase      ):







       """simple docstring"""




       def  A_     ( self     :					str						)				->			Dict:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						mock.Mock()
              SCREAMING_SNAKE_CASE__       :      int     =						5_0_0
              SCREAMING_SNAKE_CASE__       :      List[str]     =						{}
              SCREAMING_SNAKE_CASE__       :      Tuple     =						HTTPError
              SCREAMING_SNAKE_CASE__       :      Any     =						{}

              # Download this model to make sure it's in the cache.
              SCREAMING_SNAKE_CASE__       :      Optional[int]     =						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)

              # Under the mock environment we get a 500 error when trying to reach the tokenizer.
              with mock.patch("requests.Session.request",			return_value=_UpperCAmelCase						) as mock_head:
                     SCREAMING_SNAKE_CASE__       :      Optional[int]     =						BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"						)
                     # This check we did call the fake head request
                     mock_head.assert_called()






       @require_tokenizers
       def  A_     ( self     :					Union[str, Any]						)				->			str:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      List[str]     =						mock.Mock()
              SCREAMING_SNAKE_CASE__       :      Any     =						5_0_0
              SCREAMING_SNAKE_CASE__       :      int     =						{}
              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						HTTPError
              SCREAMING_SNAKE_CASE__       :      Dict     =						{}

              # Download this model to make sure it's in the cache.
              SCREAMING_SNAKE_CASE__       :      List[str]     =						GPTaTokenizerFast.from_pretrained("gpt2"						)

              # Under the mock environment we get a 500 error when trying to reach the tokenizer.
              with mock.patch("requests.Session.request",			return_value=_UpperCAmelCase						) as mock_head:
                     SCREAMING_SNAKE_CASE__       :      Tuple     =						GPTaTokenizerFast.from_pretrained("gpt2"						)
                     # This check we did call the fake head request
                     mock_head.assert_called()






       def  A_     ( self     :					Tuple						)				->			List[str]:





              """simple docstring"""
              # This test is for deprecated behavior and can be removed in v5
              try:
                     SCREAMING_SNAKE_CASE__       :      Dict     =						tempfile.mktemp()
                     with open(_UpperCAmelCase,			"wb"						) as f:
                            http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model",			_UpperCAmelCase						)

                     SCREAMING_SNAKE_CASE__       :      List[Any]     =						AlbertTokenizer.from_pretrained(_UpperCAmelCase						)
              finally:
                     os.remove(_UpperCAmelCase						)

              # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
              # the current folder and have the right name.
              if os.path.isfile("tokenizer.json"						):
                     # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                     return
              try:
                     with open("tokenizer.json",			"wb"						) as f:
                            http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json",			_UpperCAmelCase						)
                     SCREAMING_SNAKE_CASE__       :      Tuple     =						AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"						)
                     # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                     self.assertEqual(tokenizer.vocab_size,			1_0_0_0						)
                     # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

              finally:
                     os.remove("tokenizer.json"						)






       def  A_     ( self     :					str						)				->			Optional[Any]:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      List[str]     =						AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"						)


@is_staging_test
class 						lowerCamelCase       (unittest.TestCase      ):







       """simple docstring"""




       UpperCAmelCase_      						=   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "bla", "blou"]






       @classmethod
       def  A_     ( cls     :					str						)				->			List[str]:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						TOKEN
              HfFolder.save_token(_UpperCAmelCase						)






       @classmethod
       def  A_     ( cls     :					List[str]						)				->			int:





              """simple docstring"""
              try:
                     delete_repo(token=cls._token,			repo_id="test-tokenizer"						)
              except HTTPError:
                     pass

              try:
                     delete_repo(token=cls._token,			repo_id="valid_org/test-tokenizer-org"						)
              except HTTPError:
                     pass

              try:
                     delete_repo(token=cls._token,			repo_id="test-dynamic-tokenizer"						)
              except HTTPError:
                     pass






       def  A_     ( self     :					Any						)				->			str:





              """simple docstring"""
              with tempfile.TemporaryDirectory() as tmp_dir:
                     SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						os.path.join(_UpperCAmelCase,			"vocab.txt"						)
                     with open(_UpperCAmelCase,			"w",			encoding="utf-8"						) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
                     SCREAMING_SNAKE_CASE__       :      Tuple     =						BertTokenizer(_UpperCAmelCase						)

              tokenizer.push_to_hub("test-tokenizer",			use_auth_token=self._token						)
              SCREAMING_SNAKE_CASE__       :      Optional[int]     =						BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''						)
              self.assertDictEqual(new_tokenizer.vocab,			tokenizer.vocab						)

              # Reset repo
              delete_repo(token=self._token,			repo_id="test-tokenizer"						)

              # Push to hub via save_pretrained
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(_UpperCAmelCase,			repo_id="test-tokenizer",			push_to_hub=_UpperCAmelCase,			use_auth_token=self._token						)

              SCREAMING_SNAKE_CASE__       :      str     =						BertTokenizer.from_pretrained(F'''{USER}/test-tokenizer'''						)
              self.assertDictEqual(new_tokenizer.vocab,			tokenizer.vocab						)






       def  A_     ( self     :					List[Any]						)				->			str:





              """simple docstring"""
              with tempfile.TemporaryDirectory() as tmp_dir:
                     SCREAMING_SNAKE_CASE__       :      List[str]     =						os.path.join(_UpperCAmelCase,			"vocab.txt"						)
                     with open(_UpperCAmelCase,			"w",			encoding="utf-8"						) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
                     SCREAMING_SNAKE_CASE__       :      str     =						BertTokenizer(_UpperCAmelCase						)

              tokenizer.push_to_hub("valid_org/test-tokenizer-org",			use_auth_token=self._token						)
              SCREAMING_SNAKE_CASE__       :      int     =						BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"						)
              self.assertDictEqual(new_tokenizer.vocab,			tokenizer.vocab						)

              # Reset repo
              delete_repo(token=self._token,			repo_id="valid_org/test-tokenizer-org"						)

              # Push to hub via save_pretrained
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(
                         _UpperCAmelCase,			repo_id="valid_org/test-tokenizer-org",			push_to_hub=_UpperCAmelCase,			use_auth_token=self._token						)

              SCREAMING_SNAKE_CASE__       :      Dict     =						BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"						)
              self.assertDictEqual(new_tokenizer.vocab,			tokenizer.vocab						)






       @require_tokenizers
       def  A_     ( self     :					int						)				->			Tuple:





              """simple docstring"""
              CustomTokenizer.register_for_auto_class()
              with tempfile.TemporaryDirectory() as tmp_dir:
                     SCREAMING_SNAKE_CASE__       :      int     =						os.path.join(_UpperCAmelCase,			"vocab.txt"						)
                     with open(_UpperCAmelCase,			"w",			encoding="utf-8"						) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)
                     SCREAMING_SNAKE_CASE__       :      Dict     =						CustomTokenizer(_UpperCAmelCase						)

              # No fast custom tokenizer
              tokenizer.push_to_hub("test-dynamic-tokenizer",			use_auth_token=self._token						)

              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''',			trust_remote_code=_UpperCAmelCase						)
              # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__,			"CustomTokenizer"						)

              # Fast and slow custom tokenizer
              CustomTokenizerFast.register_for_auto_class()
              with tempfile.TemporaryDirectory() as tmp_dir:
                     SCREAMING_SNAKE_CASE__       :      int     =						os.path.join(_UpperCAmelCase,			"vocab.txt"						)
                     with open(_UpperCAmelCase,			"w",			encoding="utf-8"						) as vocab_writer:
                            vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]						)						)

                     SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						BertTokenizerFast.from_pretrained(_UpperCAmelCase						)
                     bert_tokenizer.save_pretrained(_UpperCAmelCase						)
                     SCREAMING_SNAKE_CASE__       :      List[Any]     =						CustomTokenizerFast.from_pretrained(_UpperCAmelCase						)

              tokenizer.push_to_hub("test-dynamic-tokenizer",			use_auth_token=self._token						)

              SCREAMING_SNAKE_CASE__       :      Optional[int]     =						AutoTokenizer.from_pretrained(F'''{USER}/test-dynamic-tokenizer''',			trust_remote_code=_UpperCAmelCase						)
              # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__,			"CustomTokenizerFast"						)
              SCREAMING_SNAKE_CASE__       :      List[str]     =						AutoTokenizer.from_pretrained(
                  F'''{USER}/test-dynamic-tokenizer''',			use_fast=_UpperCAmelCase,			trust_remote_code=_UpperCAmelCase						)
              # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__,			"CustomTokenizer"						)


class 						lowerCamelCase       (unittest.TestCase      ):







       """simple docstring"""




       def  A_     ( self     :					List[str]						)				->			Optional[int]:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      int     =						Trie()
              trie.add("Hello 友達"						)
              self.assertEqual(trie.data,			{"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}						)
              trie.add("Hello"						)
              trie.data
              self.assertEqual(trie.data,			{"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}						)






       def  A_     ( self     :					Optional[int]						)				->			Optional[Any]:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      List[str]     =						Trie()
              self.assertEqual(trie.split("[CLS] This is a extra_id_100"						),			["[CLS] This is a extra_id_100"]						)
              trie.add("[CLS]"						)
              trie.add("extra_id_1"						)
              trie.add("extra_id_100"						)
              self.assertEqual(trie.split("[CLS] This is a extra_id_100"						),			["[CLS]", " This is a ", "extra_id_100"]						)






       def  A_     ( self     :					int						)				->			Tuple:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      int     =						Trie()
              trie.add("A"						)
              self.assertEqual(trie.split("ABC"						),			["A", "BC"]						)
              self.assertEqual(trie.split("BCA"						),			["BC", "A"]						)






       def  A_     ( self     :					Any						)				->			Any:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						Trie()
              trie.add("TOKEN]"						)
              trie.add("[SPECIAL_TOKEN]"						)
              self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"						),			["This is something ", "[SPECIAL_TOKEN]"]						)






       def  A_     ( self     :					List[str]						)				->			int:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      List[Any]     =						Trie()
              trie.add("A"						)
              trie.add("P"						)
              trie.add("[SPECIAL_TOKEN]"						)
              self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"						),			["This is something ", "[SPECIAL_TOKEN]"]						)






       def  A_     ( self     :					Tuple						)				->			List[Any]:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      str     =						Trie()
              trie.add("AB"						)
              trie.add("B"						)
              trie.add("C"						)
              self.assertEqual(trie.split("ABC"						),			["AB", "C"]						)






       def  A_     ( self     :					Union[str, Any]						)				->			Dict:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      Any     =						Trie()
              trie.add("ABC"						)
              trie.add("B"						)
              trie.add("CD"						)
              self.assertEqual(trie.split("ABCD"						),			["ABC", "D"]						)






       def  A_     ( self     :					Optional[int]						)				->			Dict:





              """simple docstring"""
              SCREAMING_SNAKE_CASE__       :      List[Any]     =						Trie()
              SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						trie.cut_text("ABC",			[0, 0, 2, 1, 2, 3]						)
              self.assertEqual(_UpperCAmelCase,			["AB", "C"]						)













import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase :    int  							= logging.get_logger(__name__)

_lowerCamelCase :    Optional[Any]  							= '''▁'''

_lowerCamelCase :    Dict  							= {'''vocab_file''': '''sentencepiece.bpe.model'''}

_lowerCamelCase :    int  							= {
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_lowerCamelCase :    Optional[Any]  							= {
    '''xlm-roberta-base''': 5_1_2,
    '''xlm-roberta-large''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_1_2,
}


class 						lowerCamelCase       (__lowerCamelCase      ):







							"""simple docstring"""




							UpperCAmelCase_      						=   VOCAB_FILES_NAMES
							UpperCAmelCase_      						=   PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase_      						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase_      						=   ["input_ids", "attention_mask"]






							def __init__( self     :					Dict,			_UpperCAmelCase     :					str,			_UpperCAmelCase     :					Optional[int]="<s>",			_UpperCAmelCase     :					Optional[int]="</s>",			_UpperCAmelCase     :					Dict="</s>",			_UpperCAmelCase     :					List[Any]="<s>",			_UpperCAmelCase     :					Union[str, Any]="<unk>",			_UpperCAmelCase     :					List[Any]="<pad>",			_UpperCAmelCase     :					str="<mask>",			_UpperCAmelCase     :					Optional[Dict[str, Any]] = None,			**_UpperCAmelCase     :					List[Any],			)				->			None:





														"""simple docstring"""
														# Mask token behave like a normal word, i.e. include the space before it
														SCREAMING_SNAKE_CASE__       :      int     =						AddedToken(_UpperCAmelCase,			lstrip=_UpperCAmelCase,			rstrip=_UpperCAmelCase						) if isinstance(_UpperCAmelCase,			_UpperCAmelCase						) else mask_token

														SCREAMING_SNAKE_CASE__       :      List[str]     =						{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=_UpperCAmelCase,			eos_token=_UpperCAmelCase,			unk_token=_UpperCAmelCase,			sep_token=_UpperCAmelCase,			cls_token=_UpperCAmelCase,			pad_token=_UpperCAmelCase,			mask_token=_UpperCAmelCase,			sp_model_kwargs=self.sp_model_kwargs,			**_UpperCAmelCase,			)

														SCREAMING_SNAKE_CASE__       :      str     =						spm.SentencePieceProcessor(**self.sp_model_kwargs						)
														self.sp_model.Load(str(_UpperCAmelCase						)						)
														SCREAMING_SNAKE_CASE__       :      Tuple     =						vocab_file

														# Original fairseq vocab and spm vocab must be "aligned":
														# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
														# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
														# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
														# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

														# Mimic fairseq token-to-id alignment for the first 4 token
														SCREAMING_SNAKE_CASE__       :      List[str]     =						{"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

														# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
														SCREAMING_SNAKE_CASE__       :      Dict     =						1

														SCREAMING_SNAKE_CASE__       :      int     =						len(self.sp_model						) + self.fairseq_offset
														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						{v: k for k, v in self.fairseq_tokens_to_ids.items()}






							def __getstate__( self     :					str						)				->			Optional[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      List[Any]     =						self.__dict__.copy()
														SCREAMING_SNAKE_CASE__       :      List[Any]     =						None
														SCREAMING_SNAKE_CASE__       :      Dict     =						self.sp_model.serialized_model_proto()
														return state






							def __setstate__( self     :					int,			_UpperCAmelCase     :					List[Any]						)				->			Any:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      Tuple     =						d

														# for backward compatibility
														if not hasattr(self,			"sp_model_kwargs"						):
																					SCREAMING_SNAKE_CASE__       :      Dict     =						{}

														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						spm.SentencePieceProcessor(**self.sp_model_kwargs						)
														self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)






							def  A_     ( self     :					Any,			_UpperCAmelCase     :					List[int],			_UpperCAmelCase     :					Optional[List[int]] = None						)				->			List[int]:





														"""simple docstring"""

														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														SCREAMING_SNAKE_CASE__       :      List[str]     =						[self.cls_token_id]
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						[self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep






							def  A_     ( self     :					List[Any],			_UpperCAmelCase     :					List[int],			_UpperCAmelCase     :					Optional[List[int]] = None,			_UpperCAmelCase     :					bool = False						)				->			List[int]:





														"""simple docstring"""

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=_UpperCAmelCase,			token_ids_a=_UpperCAmelCase,			already_has_special_tokens=_UpperCAmelCase						)

														if token_ids_a is None:
																					return [1] + ([0] * len(_UpperCAmelCase						)) + [1]
														return [1] + ([0] * len(_UpperCAmelCase						)) + [1, 1] + ([0] * len(_UpperCAmelCase						)) + [1]






							def  A_     ( self     :					Union[str, Any],			_UpperCAmelCase     :					List[int],			_UpperCAmelCase     :					Optional[List[int]] = None						)				->			List[int]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						[self.sep_token_id]
														SCREAMING_SNAKE_CASE__       :      Optional[int]     =						[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep						) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]






							@property
							def  A_     ( self     :					List[str]						)				->			List[str]:





														"""simple docstring"""
														return len(self.sp_model						) + self.fairseq_offset + 1  # Add the <mask> token






							def  A_     ( self     :					List[Any]						)				->			Optional[int]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      str     =						{self.convert_ids_to_tokens(_UpperCAmelCase						): i for i in range(self.vocab_size						)}
														vocab.update(self.added_tokens_encoder						)
														return vocab






							def  A_     ( self     :					List[str],			_UpperCAmelCase     :					str						)				->			List[str]:





														"""simple docstring"""
														return self.sp_model.encode(_UpperCAmelCase,			out_type=_UpperCAmelCase						)






							def  A_     ( self     :					Optional[Any],			_UpperCAmelCase     :					Union[str, Any]						)				->			Union[str, Any]:





														"""simple docstring"""
														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														SCREAMING_SNAKE_CASE__       :      Union[str, Any]     =						self.sp_model.PieceToId(_UpperCAmelCase						)

														# Need to return unknown token if the SP model returned 0
														return spm_id + self.fairseq_offset if spm_id else self.unk_token_id






							def  A_     ( self     :					Tuple,			_UpperCAmelCase     :					List[str]						)				->			List[str]:





														"""simple docstring"""
														if index in self.fairseq_ids_to_tokens:
																					return self.fairseq_ids_to_tokens[index]
														return self.sp_model.IdToPiece(index - self.fairseq_offset						)






							def  A_     ( self     :					Any,			_UpperCAmelCase     :					int						)				->			List[Any]:





														"""simple docstring"""
														SCREAMING_SNAKE_CASE__       :      int     =						"".join(_UpperCAmelCase						).replace(_UpperCAmelCase,			" "						).strip()
														return out_string






							def  A_     ( self     :					Union[str, Any],			_UpperCAmelCase     :					str,			_UpperCAmelCase     :					Optional[str] = None						)				->			Tuple[str]:





														"""simple docstring"""
														if not os.path.isdir(_UpperCAmelCase						):
																					logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
																					return
														SCREAMING_SNAKE_CASE__       :      Optional[Any]     =						os.path.join(
														    _UpperCAmelCase,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)

														if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCAmelCase						) and os.path.isfile(self.vocab_file						):
																					copyfile(self.vocab_file,			_UpperCAmelCase						)
														elif not os.path.isfile(self.vocab_file						):
																					with open(_UpperCAmelCase,			"wb"						) as fi:
																												SCREAMING_SNAKE_CASE__       :      Any     =						self.sp_model.serialized_model_proto()
																												fi.write(_UpperCAmelCase						)

														return (out_vocab_file,)






