


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__  :str             =							logging.get_logger(__name__)

lowerCAmelCase__  :Any             =							"▁"

lowerCAmelCase__  :Any             =							{"vocab_file": "sentencepiece.bpe.model"}

lowerCAmelCase__  :Optional[Any]             =							{
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

lowerCAmelCase__  :Any             =							{
    "xlm-roberta-base": 5_1_2,
    "xlm-roberta-large": 5_1_2,
    "xlm-roberta-large-finetuned-conll02-dutch": 5_1_2,
    "xlm-roberta-large-finetuned-conll02-spanish": 5_1_2,
    "xlm-roberta-large-finetuned-conll03-english": 5_1_2,
    "xlm-roberta-large-finetuned-conll03-german": 5_1_2,
}

class 		__a				(    snake_case__      ):
		_a     :  Optional[int]           =     VOCAB_FILES_NAMES
		_a     :  List[Any]           =     PRETRAINED_VOCAB_FILES_MAP
		_a     :  Any           =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		_a     :  List[Any]           =     ['input_ids', 'attention_mask']
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE="<s>"      ,   _SCREAMING_SNAKE_CASE="</s>"      ,   _SCREAMING_SNAKE_CASE="</s>"      ,   _SCREAMING_SNAKE_CASE="<s>"      ,   _SCREAMING_SNAKE_CASE="<unk>"      ,   _SCREAMING_SNAKE_CASE="<pad>"      ,   _SCREAMING_SNAKE_CASE="<mask>"      ,   _SCREAMING_SNAKE_CASE = None      ,   **_SCREAMING_SNAKE_CASE      ,   )			->	Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=      AddedToken(UpperCAmelCase_      ,   lstrip=UpperCAmelCase_      ,   rstrip=UpperCAmelCase_						) if isinstance(UpperCAmelCase_      ,   UpperCAmelCase_						) else mask_token

									_UpperCAmelCase					=      {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    bos_token=UpperCAmelCase_      ,   eos_token=UpperCAmelCase_      ,   unk_token=UpperCAmelCase_      ,   sep_token=UpperCAmelCase_      ,   cls_token=UpperCAmelCase_      ,   pad_token=UpperCAmelCase_      ,   mask_token=UpperCAmelCase_      ,   sp_model_kwargs=self.sp_model_kwargs      ,   **UpperCAmelCase_      ,   )

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.Load(str(UpperCAmelCase_						)						)
									_UpperCAmelCase					=      vocab_file

									# Original fairseq vocab and spm vocab must be "aligned":
									# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
									# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
									# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
									# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

									# Mimic fairseq token-to-id alignment for the first 4 token
									_UpperCAmelCase					=      {"<s>": 0, "<pad>": 1, "</s>": 2, "<unk>": 3}

									# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
									_UpperCAmelCase					=      1

									_UpperCAmelCase					=      len(self.sp_model						) + self.fairseq_offset
									_UpperCAmelCase					=      {v: k for k, v in self.fairseq_tokens_to_ids.items()}
		def __getstate__(      self						)			->	Optional[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      self.__dict__.copy()
									_UpperCAmelCase					=      None
									_UpperCAmelCase					=      self.sp_model.serialized_model_proto()
									return state
		def __setstate__(      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      d

									# for backward compatibility
									if not hasattr(self      ,   'sp_model_kwargs'						):
																_UpperCAmelCase					=      {}

									_UpperCAmelCase					=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
									self.sp_model.LoadFromSerializedProto(self.sp_model_proto						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	Tuple:




									"""simple docstring"""





									if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]
									_UpperCAmelCase					=      [self.sep_token_id]
									return cls + token_ids_a + sep + sep + token_ids_a + sep
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None      ,   _SCREAMING_SNAKE_CASE = False						)			->	Union[str, Any]:




									"""simple docstring"""





									if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=UpperCAmelCase_      ,   token_ids_a=UpperCAmelCase_      ,   already_has_special_tokens=UpperCAmelCase_						)

									if token_ids_a is None:
																return [1] + ([0] * len(UpperCAmelCase_						)) + [1]
									return [1] + ([0] * len(UpperCAmelCase_						)) + [1, 1] + ([0] * len(UpperCAmelCase_						)) + [1]
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.sep_token_id]
									_UpperCAmelCase					=      [self.cls_token_id]

									if token_ids_a is None:
																return len(cls + token_ids_a + sep						) * [0]
									return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]
		@property
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									return len(self.sp_model						) + self.fairseq_offset + 1  # Add the <mask> token
		def      UpperCAmelCase__    (      self						)			->	List[str]:




									"""simple docstring"""





									_UpperCAmelCase					=      {self.convert_ids_to_tokens(UpperCAmelCase_						): i for i in range(self.vocab_size						)}
									vocab.update(self.added_tokens_encoder						)
									return vocab
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	List[str]:




									"""simple docstring"""





									return self.sp_model.encode(UpperCAmelCase_      ,   out_type=UpperCAmelCase_						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	Dict:




									"""simple docstring"""





									if token in self.fairseq_tokens_to_ids:
																return self.fairseq_tokens_to_ids[token]
									_UpperCAmelCase					=      self.sp_model.PieceToId(UpperCAmelCase_						)

									# Need to return unknown token if the SP model returned 0
									return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	str:




									"""simple docstring"""





									if index in self.fairseq_ids_to_tokens:
																return self.fairseq_ids_to_tokens[index]
									return self.sp_model.IdToPiece(index - self.fairseq_offset						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      "".join(UpperCAmelCase_						).replace(UpperCAmelCase_      ,   ' '						).strip()
									return out_string






		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE = None						)			->	str:




									"""simple docstring"""





									if not os.path.isdir(UpperCAmelCase_						):
																logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
																return
									_UpperCAmelCase					=      os.path.join(
									    UpperCAmelCase_      ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

									if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase_						) and os.path.isfile(self.vocab_file						):
																copyfile(self.vocab_file      ,   UpperCAmelCase_						)
									elif not os.path.isfile(self.vocab_file						):
																with open(UpperCAmelCase_      ,   'wb'						) as fi:
																							_UpperCAmelCase					=      self.sp_model.serialized_model_proto()
																							fi.write(UpperCAmelCase_						)

									return (out_vocab_file,)




from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCAmelCase__  :List[str]             =							{
    '''configuration_x_clip''': [
        '''XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''XCLIPConfig''',
        '''XCLIPTextConfig''',
        '''XCLIPVisionConfig''',
    ],
    '''processing_x_clip''': ['''XCLIPProcessor'''],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCAmelCase__  :Any             =							[
		    '''XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
		    '''XCLIPModel''',
		    '''XCLIPPreTrainedModel''',
		    '''XCLIPTextModel''',
		    '''XCLIPVisionModel''',
		]

if TYPE_CHECKING:
		from .configuration_x_clip import (
		    XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    XCLIPConfig,
		    XCLIPTextConfig,
		    XCLIPVisionConfig,
		)
		from .processing_x_clip import XCLIPProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_x_clip import (
				    XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
				    XCLIPModel,
				    XCLIPPreTrainedModel,
				    XCLIPTextModel,
				    XCLIPVisionModel,
				)

else:
		import sys

		lowerCAmelCase__  :str             =							_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
