'''simple docstring'''
import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        XLMForMultipleChoice,
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMWithLMHeadModel,
    )
    from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST


class        a__  :


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase=13      ,    _UpperCamelCase=7      ,    _UpperCamelCase=True      ,    _UpperCamelCase=True      ,    _UpperCamelCase=True      ,    _UpperCamelCase=True      ,    _UpperCamelCase=True      ,    _UpperCamelCase=False      ,    _UpperCamelCase=False      ,    _UpperCamelCase=False      ,    _UpperCamelCase=2      ,    _UpperCamelCase=99      ,    _UpperCamelCase=0      ,    _UpperCamelCase=32      ,    _UpperCamelCase=5      ,    _UpperCamelCase=4      ,    _UpperCamelCase=0.1      ,    _UpperCamelCase=0.1      ,    _UpperCamelCase=512      ,    _UpperCamelCase=2      ,    _UpperCamelCase=0.0_2      ,    _UpperCamelCase=2      ,    _UpperCamelCase=4      ,    _UpperCamelCase="last"      ,    _UpperCamelCase=True      ,    _UpperCamelCase=None      ,    _UpperCamelCase=0      ,    ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       parent
      _lowercase      :				int      					=       batch_size
      _lowercase      :				Any      					=       seq_length
      _lowercase      :				Union[str, Any]      					=       is_training
      _lowercase      :				Tuple      					=       use_input_lengths
      _lowercase      :				List[str]      					=       use_token_type_ids
      _lowercase      :				Dict      					=       use_labels
      _lowercase      :				Any      					=       gelu_activation
      _lowercase      :				Union[str, Any]      					=       sinusoidal_embeddings
      _lowercase      :				Union[str, Any]      					=       causal
      _lowercase      :				Optional[Any]      					=       asm
      _lowercase      :				str      					=       n_langs
      _lowercase      :				List[str]      					=       vocab_size
      _lowercase      :				Optional[Any]      					=       n_special
      _lowercase      :				List[str]      					=       hidden_size
      _lowercase      :				int      					=       num_hidden_layers
      _lowercase      :				Dict      					=       num_attention_heads
      _lowercase      :				Tuple      					=       hidden_dropout_prob
      _lowercase      :				int      					=       attention_probs_dropout_prob
      _lowercase      :				List[Any]      					=       max_position_embeddings
      _lowercase      :				Optional[Any]      					=       type_sequence_label_size
      _lowercase      :				Optional[Any]      					=       initializer_range
      _lowercase      :				Dict      					=       num_labels
      _lowercase      :				int      					=       num_choices
      _lowercase      :				str      					=       summary_type
      _lowercase      :				Tuple      					=       use_proj
      _lowercase      :				str      					=       scope
      _lowercase      :				List[Any]      					=       bos_token_id


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       ids_tensor([self.batch_size, self.seq_length]      ,    self.vocab_size  )
      _lowercase      :				str      					=       random_attention_mask([self.batch_size, self.seq_length]  )

      _lowercase      :				int      					=       None
      if self.use_input_lengths:
         _lowercase      :				List[str]      					=       (
             ids_tensor([self.batch_size]      ,    vocab_size=2  ) + self.seq_length - 2
         )  # small variation of seq_length

      _lowercase      :				Tuple      					=       None
      if self.use_token_type_ids:
         _lowercase      :				Any      					=       ids_tensor([self.batch_size, self.seq_length]      ,    self.n_langs  )

      _lowercase      :				Any      					=       None
      _lowercase      :				List[str]      					=       None
      _lowercase      :				int      					=       None
      if self.use_labels:
         _lowercase      :				Tuple      					=       ids_tensor([self.batch_size]      ,    self.type_sequence_label_size  )
         _lowercase      :				str      					=       ids_tensor([self.batch_size, self.seq_length]      ,    self.num_labels  )
         _lowercase      :				Union[str, Any]      					=       ids_tensor([self.batch_size]      ,    2  ).float()
         _lowercase      :				Optional[int]      					=       ids_tensor([self.batch_size]      ,    self.num_choices  )

      _lowercase      :				str      					=       self.get_config()

      return (
          config,
          input_ids,
          token_type_ids,
          input_lengths,
          sequence_labels,
          token_labels,
          is_impossible_labels,
          choice_labels,
          input_mask,
      )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return XLMConfig(
          vocab_size=self.vocab_size      ,    n_special=self.n_special      ,    emb_dim=self.hidden_size      ,    n_layers=self.num_hidden_layers      ,    n_heads=self.num_attention_heads      ,    dropout=self.hidden_dropout_prob      ,    attention_dropout=self.attention_probs_dropout_prob      ,    gelu_activation=self.gelu_activation      ,    sinusoidal_embeddings=self.sinusoidal_embeddings      ,    asm=self.asm      ,    causal=self.causal      ,    n_langs=self.n_langs      ,    max_position_embeddings=self.max_position_embeddings      ,    initializer_range=self.initializer_range      ,    summary_type=self.summary_type      ,    use_proj=self.use_proj      ,    num_labels=self.num_labels      ,    bos_token_id=self.bos_token_id      ,    )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Dict      					=       XLMModel(config=_a  )
      model.to(_a  )
      model.eval()
      _lowercase      :				Optional[int]      					=       model(_a      ,    lengths=_a      ,    langs=_a  )
      _lowercase      :				int      					=       model(_a      ,    langs=_a  )
      _lowercase      :				Dict      					=       model(_a  )
      self.parent.assertEqual(result.last_hidden_state.shape      ,    (self.batch_size, self.seq_length, self.hidden_size)  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       XLMWithLMHeadModel(_a  )
      model.to(_a  )
      model.eval()

      _lowercase      :				Any      					=       model(_a      ,    token_type_ids=_a      ,    labels=_a  )
      self.parent.assertEqual(result.loss.shape      ,    ()  )
      self.parent.assertEqual(result.logits.shape      ,    (self.batch_size, self.seq_length, self.vocab_size)  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Any      					=       XLMForQuestionAnsweringSimple(_a  )
      model.to(_a  )
      model.eval()

      _lowercase      :				Any      					=       model(_a  )

      _lowercase      :				Optional[Any]      					=       model(_a      ,    start_positions=_a      ,    end_positions=_a  )
      _lowercase      :				Optional[Any]      					=       outputs
      self.parent.assertEqual(result.start_logits.shape      ,    (self.batch_size, self.seq_length)  )
      self.parent.assertEqual(result.end_logits.shape      ,    (self.batch_size, self.seq_length)  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       XLMForQuestionAnswering(_a  )
      model.to(_a  )
      model.eval()

      _lowercase      :				int      					=       model(_a  )

      _lowercase      :				Any      					=       model(
          _a      ,    start_positions=_a      ,    end_positions=_a      ,    cls_index=_a      ,    is_impossible=_a      ,    p_mask=_a      ,    )

      _lowercase      :				Tuple      					=       model(
          _a      ,    start_positions=_a      ,    end_positions=_a      ,    cls_index=_a      ,    is_impossible=_a      ,    )

      ((_lowercase       )					,				       )      :				Dict      					=       result_with_labels.to_tuple()

      _lowercase      :				Optional[int]      					=       model(_a      ,    start_positions=_a      ,    end_positions=_a  )

      ((_lowercase       )					,				       )      :				Optional[int]      					=       result_with_labels.to_tuple()

      self.parent.assertEqual(result_with_labels.loss.shape      ,    ()  )
      self.parent.assertEqual(result.start_top_log_probs.shape      ,    (self.batch_size, model.config.start_n_top)  )
      self.parent.assertEqual(result.start_top_index.shape      ,    (self.batch_size, model.config.start_n_top)  )
      self.parent.assertEqual(
          result.end_top_log_probs.shape      ,    (self.batch_size, model.config.start_n_top * model.config.end_n_top)  )
      self.parent.assertEqual(
          result.end_top_index.shape      ,    (self.batch_size, model.config.start_n_top * model.config.end_n_top)  )
      self.parent.assertEqual(result.cls_logits.shape      ,    (self.batch_size,)  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       XLMForSequenceClassification(_a  )
      model.to(_a  )
      model.eval()

      _lowercase      :				List[Any]      					=       model(_a  )
      _lowercase      :				int      					=       model(_a      ,    labels=_a  )
      self.parent.assertEqual(result.loss.shape      ,    ()  )
      self.parent.assertEqual(result.logits.shape      ,    (self.batch_size, self.type_sequence_label_size)  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       self.num_labels
      _lowercase      :				Dict      					=       XLMForTokenClassification(_a  )
      model.to(_a  )
      model.eval()

      _lowercase      :				Optional[Any]      					=       model(_a      ,    attention_mask=_a      ,    labels=_a  )
      self.parent.assertEqual(result.logits.shape      ,    (self.batch_size, self.seq_length, self.num_labels)  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Any      					=       self.num_choices
      _lowercase      :				List[str]      					=       XLMForMultipleChoice(config=_a  )
      model.to(_a  )
      model.eval()
      _lowercase      :				int      					=       input_ids.unsqueeze(1  ).expand(-1      ,    self.num_choices      ,    -1  ).contiguous()
      _lowercase      :				List[Any]      					=       token_type_ids.unsqueeze(1  ).expand(-1      ,    self.num_choices      ,    -1  ).contiguous()
      _lowercase      :				Union[str, Any]      					=       input_mask.unsqueeze(1  ).expand(-1      ,    self.num_choices      ,    -1  ).contiguous()
      _lowercase      :				int      					=       model(
          _a      ,    attention_mask=_a      ,    token_type_ids=_a      ,    labels=_a      ,    )
      self.parent.assertEqual(result.logits.shape      ,    (self.batch_size, self.num_choices)  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       self.prepare_config_and_inputs()
      (
          (
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				(
          _lowercase       
      )					,				       
      )      :				List[Any]      					=       config_and_inputs
      _lowercase      :				List[str]      					=       {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths}
      return config, inputs_dict




@require_torch
class        a__  (							__lowercase  ,				__lowercase  ,				__lowercase  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					Any     	=							(
       (
           XLMModel,
           XLMWithLMHeadModel,
           XLMForQuestionAnswering,
           XLMForSequenceClassification,
           XLMForQuestionAnsweringSimple,
           XLMForTokenClassification,
           XLMForMultipleChoice,
       )
       if is_torch_available()
       else ()
   )
   _SCREAMING_SNAKE_CASE   :					Dict     	=							(
       (XLMWithLMHeadModel,) if is_torch_available() else ()
   )  # TODO (PVP): Check other models whether language generation is also applicable
   _SCREAMING_SNAKE_CASE   :					str     	=							(
       {
           'feature-extraction': XLMModel,
           'fill-mask': XLMWithLMHeadModel,
           'question-answering': XLMForQuestionAnsweringSimple,
           'text-classification': XLMForSequenceClassification,
           'text-generation': XLMWithLMHeadModel,
           'token-classification': XLMForTokenClassification,
           'zero-shot': XLMForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase  ):





      """simple docstring"""





      if (
          pipeline_test_casse_name == "QAPipelineTests"
          and tokenizer_name is not None
          and not tokenizer_name.endswith("Fast"  )
      ):
         # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
         # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
         # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
         return True

      return False


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase=False  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       super()._prepare_for_class(_a      ,    _a      ,    return_labels=_a  )

      if return_labels:
         if model_class.__name__ == "XLMForQuestionAnswering":
            _lowercase      :				int      					=       torch.zeros(
                self.model_tester.batch_size      ,    dtype=torch.long      ,    device=_a  )
            _lowercase      :				Any      					=       torch.zeros(
                self.model_tester.batch_size      ,    dtype=torch.long      ,    device=_a  )

      return inputs_dict


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       XLMModelTester(self  )
      _lowercase      :				Any      					=       ConfigTester(self      ,    config_class=_a      ,    emb_dim=37  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self.config_tester.run_common_tests()


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_model(*_a  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_lm_head(*_a  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_simple_qa(*_a  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_qa(*_a  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_sequence_classif(*_a  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_token_classif(*_a  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_xlm_for_multiple_choice(*_a  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase=False      ,    _UpperCamelCase=1  ):





      """simple docstring"""





      self.assertIsInstance(_a      ,    _a  )
      self.assertListEqual(
          [isinstance(_a      ,    _a  ) for iter_attentions in attentions]      ,    [True] * len(_a  )  )
      self.assertEqual(len(_a  )      ,    (max_length - min_length) * num_beam_groups  )

      for idx, iter_attentions in enumerate(_a  ):
         # adds PAD dummy token
         _lowercase      :				List[str]      					=       min_length + idx + 1
         _lowercase      :				Tuple      					=       min_length + idx + 1

         _lowercase      :				List[Any]      					=       (
             batch_size * num_beam_groups,
             config.num_attention_heads,
             tgt_len,
             src_len,
         )
         # check attn size
         self.assertListEqual(
             [layer_attention.shape for layer_attention in iter_attentions]      ,    [expected_shape] * len(_a  )  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase=False      ,    _UpperCamelCase=1  ):





      """simple docstring"""





      self.assertIsInstance(_a      ,    _a  )
      self.assertListEqual(
          [isinstance(_a      ,    _a  ) for iter_hidden_states in hidden_states]      ,    [True] * len(_a  )      ,    )
      self.assertEqual(len(_a  )      ,    (max_length - min_length) * num_beam_groups  )

      for idx, iter_hidden_states in enumerate(_a  ):
         # adds PAD dummy token
         _lowercase      :				str      					=       min_length + idx + 1
         _lowercase      :				Optional[Any]      					=       (batch_size * num_beam_groups, seq_len, config.hidden_size)
         # check hidden size
         self.assertListEqual(
             [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]      ,    [expected_shape] * len(_a  )      ,    )
      pass






   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowercase      :				Optional[int]      					=       XLMModel.from_pretrained(_a  )
         self.assertIsNotNone(_a  )




@require_torch
class        a__  (							unittest.TestCase  ):


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048"  )
      model.to(_a  )
      _lowercase      :				Tuple      					=       torch.tensor([[14, 447]]      ,    dtype=torch.long      ,    device=_a  )  # the president
      _lowercase      :				Dict      					=       [
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
          14,
          447,
      ]  # the president the president the president the president the president the president the president the president the president the president
      # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
      _lowercase      :				Any      					=       model.generate(_a      ,    do_sample=_a  )
      self.assertListEqual(output_ids[0].cpu().numpy().tolist()      ,    _a  )









'''simple docstring'''
import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case         =  logging.get_logger(__name__)

_snake_case         =  {'vocab_file': 'spiece.model'}

_snake_case         =  {
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

_snake_case         =  {
    'google/bigbird-roberta-base': 4_096,
    'google/bigbird-roberta-large': 4_096,
    'google/bigbird-base-trivia-itc': 4_096,
}


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Optional[Any]     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							['input_ids', 'attention_mask']
   _SCREAMING_SNAKE_CASE   :					List[int]     	=							[]


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<s>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="[SEP]"      ,    _UpperCamelCase="[MASK]"      ,    _UpperCamelCase="[CLS]"      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else bos_token
      _lowercase      :				Optional[int]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else eos_token
      _lowercase      :				Optional[Any]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else unk_token
      _lowercase      :				Union[str, Any]      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else pad_token
      _lowercase      :				Any      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else cls_token
      _lowercase      :				str      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else sep_token

      # Mask token behave like a normal word, i.e. include the space before it
      _lowercase      :				Any      					=       AddedToken(_UpperCamelCase      ,    lstrip=_UpperCamelCase      ,    rstrip=_UpperCamelCase  ) if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ) else mask_token

      _lowercase      :				Dict      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=_UpperCamelCase      ,    eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    sep_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    cls_token=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )

      _lowercase      :				str      					=       vocab_file

      _lowercase      :				List[Any]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(_UpperCamelCase  )


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return self.sp_model.get_piece_size()


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				str      					=       self.__dict__.copy()
      _lowercase      :				Union[str, Any]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				Optional[Any]      					=       {}

      _lowercase      :				Optional[Any]      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.piece_to_id(_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				List[Any]      					=       self.sp_model.IdToPiece(_UpperCamelCase  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       []
      _lowercase      :				int      					=       ""
      _lowercase      :				Tuple      					=       False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            if not prev_is_special:
               out_string += " "
            out_string += self.sp_model.decode(_UpperCamelCase  ) + token
            _lowercase      :				Union[str, Any]      					=       True
            _lowercase      :				Optional[int]      					=       []
         else:
            current_sub_tokens.append(_UpperCamelCase  )
            _lowercase      :				str      					=       False
      out_string += self.sp_model.decode(_UpperCamelCase  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = False      ,    _UpperCamelCase = None      ,    _UpperCamelCase = True      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Any      					=       kwargs.pop("use_source_tokenizer"      ,    _UpperCamelCase  )

      _lowercase      :				Dict      					=       self.convert_ids_to_tokens(_UpperCamelCase      ,    skip_special_tokens=_UpperCamelCase  )

      # To avoid mixing byte-level and unicode for byte-level BPT
      # we need to build string separately for added tokens and byte-level tokens
      # cf. https://github.com/huggingface/transformers/issues/1133
      _lowercase      :				Dict      					=       []
      _lowercase      :				Any      					=       []
      for token in filtered_tokens:
         if skip_special_tokens and token in self.all_special_ids:
            continue
         if token in self.added_tokens_encoder:
            if current_sub_text:
               sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase  )  )
               _lowercase      :				Optional[Any]      					=       []
            sub_texts.append(_UpperCamelCase  )
         else:
            current_sub_text.append(_UpperCamelCase  )
      if current_sub_text:
         sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase  )  )

      # Mimic the behavior of the Rust tokenizer:
      # No space before [MASK] and [SEP]
      if spaces_between_special_tokens:
         _lowercase      :				List[str]      					=       re.sub(R" (\[(MASK|SEP)\])"      ,    R"\1"      ,    " ".join(_UpperCamelCase  )  )
      else:
         _lowercase      :				List[Any]      					=       "".join(_UpperCamelCase  )

      _lowercase      :				Optional[int]      					=       (
          clean_up_tokenization_spaces
          if clean_up_tokenization_spaces is not None
          else self.clean_up_tokenization_spaces
      )
      if clean_up_tokenization_spaces:
         _lowercase      :				Tuple      					=       self.clean_up_tokenization(_UpperCamelCase  )
         return clean_text
      else:
         return text


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				Optional[Any]      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				List[str]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      _lowercase      :				str      					=       [self.cls_token_id]
      _lowercase      :				List[str]      					=       [self.sep_token_id]
      return cls + token_ids_a + sep + token_ids_a + sep


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=_UpperCamelCase      ,    token_ids_a=_UpperCamelCase      ,    already_has_special_tokens=_UpperCamelCase  )

      if token_ids_a is None:
         return [1] + ([0] * len(_UpperCamelCase  )) + [1]
      return [1] + ([0] * len(_UpperCamelCase  )) + [1] + ([0] * len(_UpperCamelCase  )) + [1]






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      _lowercase      :				str      					=       [self.sep_token_id]
      _lowercase      :				Union[str, Any]      					=       [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
      return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]






