

'''simple docstring'''
from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers.models.xglm.modeling_tf_xglm import (
        TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXGLMForCausalLM,
        TFXGLMModel,
    )


@require_tf
class        a__  :
   _SCREAMING_SNAKE_CASE   :					List[str]     	=							XGLMConfig
   _SCREAMING_SNAKE_CASE   :					List[Any]     	=							{}
   _SCREAMING_SNAKE_CASE   :					Any     	=							'gelu'


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase=14      ,    _UpperCamelCase=7      ,    _UpperCamelCase=True      ,    _UpperCamelCase=True      ,    _UpperCamelCase=True      ,    _UpperCamelCase=99      ,    _UpperCamelCase=32      ,    _UpperCamelCase=2      ,    _UpperCamelCase=4      ,    _UpperCamelCase=37      ,    _UpperCamelCase="gelu"      ,    _UpperCamelCase=0.1      ,    _UpperCamelCase=0.1      ,    _UpperCamelCase=512      ,    _UpperCamelCase=0.0_2      ,    ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       parent
      _lowercase      :				List[Any]      					=       batch_size
      _lowercase      :				Dict      					=       seq_length
      _lowercase      :				str      					=       is_training
      _lowercase      :				Any      					=       use_input_mask
      _lowercase      :				Tuple      					=       use_labels
      _lowercase      :				Union[str, Any]      					=       vocab_size
      _lowercase      :				List[Any]      					=       d_model
      _lowercase      :				Optional[Any]      					=       num_hidden_layers
      _lowercase      :				List[str]      					=       num_attention_heads
      _lowercase      :				Optional[Any]      					=       ffn_dim
      _lowercase      :				List[Any]      					=       activation_function
      _lowercase      :				Union[str, Any]      					=       activation_dropout
      _lowercase      :				List[Any]      					=       attention_dropout
      _lowercase      :				Tuple      					=       max_position_embeddings
      _lowercase      :				List[Any]      					=       initializer_range
      _lowercase      :				Union[str, Any]      					=       None
      _lowercase      :				List[Any]      					=       0
      _lowercase      :				Dict      					=       2
      _lowercase      :				List[str]      					=       1


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return XGLMConfig.from_pretrained("facebook/xglm-564M"  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       tf.clip_by_value(
          ids_tensor([self.batch_size, self.seq_length]      ,    self.vocab_size  )      ,    clip_value_min=0      ,    clip_value_max=3  )

      _lowercase      :				List[str]      					=       None
      if self.use_input_mask:
         _lowercase      :				Any      					=       random_attention_mask([self.batch_size, self.seq_length]  )

      _lowercase      :				Any      					=       self.get_config()

      _lowercase      :				Any      					=       floats_tensor([self.num_hidden_layers, self.num_attention_heads]      ,    2  )

      return (
          config,
          input_ids,
          input_mask,
          head_mask,
      )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return XGLMConfig(
          vocab_size=self.vocab_size      ,    d_model=self.hidden_size      ,    num_layers=self.num_hidden_layers      ,    attention_heads=self.num_attention_heads      ,    ffn_dim=self.ffn_dim      ,    activation_function=self.activation_function      ,    activation_dropout=self.activation_dropout      ,    attention_dropout=self.attention_dropout      ,    max_position_embeddings=self.max_position_embeddings      ,    initializer_range=self.initializer_range      ,    use_cache=_UpperCamelCase      ,    bos_token_id=self.bos_token_id      ,    eos_token_id=self.eos_token_id      ,    pad_token_id=self.pad_token_id      ,    return_dict=_UpperCamelCase      ,    )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       self.prepare_config_and_inputs()

      (
          (
          _lowercase
      )					,				(
          _lowercase
      )					,				(
          _lowercase
      )					,				(
          _lowercase
      )					,				
      )      :				Tuple      					=       config_and_inputs

      _lowercase      :				Optional[Any]      					=       {
          "input_ids": input_ids,
          "head_mask": head_mask,
      }

      return config, inputs_dict




@require_tf
class        a__  (							lowerCamelCase_  ,				lowerCamelCase_  ,				unittest.TestCase  ):
   _SCREAMING_SNAKE_CASE   :					int     	=							(TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
   _SCREAMING_SNAKE_CASE   :					Dict     	=							(TFXGLMForCausalLM,) if is_tf_available() else ()
   _SCREAMING_SNAKE_CASE   :					str     	=							(
       {'feature-extraction': TFXGLMModel, 'text-generation': TFXGLMForCausalLM} if is_tf_available() else {}
   )
   _SCREAMING_SNAKE_CASE   :					int     	=							False
   _SCREAMING_SNAKE_CASE   :					Dict     	=							False
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							False


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       TFXGLMModelTester(self  )
      _lowercase      :				Dict      					=       ConfigTester(self      ,    config_class=_UpperCamelCase      ,    n_embd=37  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      self.config_tester.run_common_tests()


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         _lowercase      :				Optional[Any]      					=       TFXGLMModel.from_pretrained(_UpperCamelCase  )
         self.assertIsNotNone(_UpperCamelCase  )






   @unittest.skip(reason="Currently, model embeddings are going to undergo a major refactor."  )
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      super().test_resize_token_embeddings()




@require_tf
class        a__  (							unittest.TestCase  ):


   @slow
   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase=True  ):





      """simple docstring"""





      _lowercase      :				Any      					=       TFXGLMForCausalLM.from_pretrained("facebook/xglm-564M"  )
      _lowercase      :				Any      					=       tf.convert_to_tensor([[2, 268, 9865]]      ,    dtype=tf.intaa  )  # The dog
      # </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
      # fmt: off
      _lowercase      :				str      					=       [2, 268, 9865, 67, 11, 1988, 57252, 9865, 5, 984, 67, 1988, 213838, 1658, 53, 70446, 33, 6657, 278, 1581]
      # fmt: on
      _lowercase      :				Dict      					=       model.generate(_UpperCamelCase      ,    do_sample=_UpperCamelCase      ,    num_beams=1  )
      if verify_outputs:
         self.assertListEqual(output_ids[0].numpy().tolist()      ,    _UpperCamelCase  )


   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				int      					=       XGLMTokenizer.from_pretrained("facebook/xglm-564M"  )
      _lowercase      :				List[Any]      					=       TFXGLMForCausalLM.from_pretrained("facebook/xglm-564M"  )

      tf.random.set_seed(0  )
      _lowercase      :				Tuple      					=       tokenizer("Today is a nice day and"      ,    return_tensors="tf"  )
      _lowercase      :				List[Any]      					=       tokenized.input_ids
      # forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
      with tf.device(":/CPU:0"  ):
         _lowercase      :				Optional[Any]      					=       model.generate(_UpperCamelCase      ,    do_sample=_UpperCamelCase      ,    seed=[7, 0]  )
      _lowercase      :				Optional[int]      					=       tokenizer.decode(output_ids[0]      ,    skip_special_tokens=_UpperCamelCase  )

      _lowercase      :				Any      					=       (
          "Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due"
      )
      self.assertEqual(_UpperCamelCase      ,    _UpperCamelCase  )






   @slow
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				List[str]      					=       TFXGLMForCausalLM.from_pretrained("facebook/xglm-564M"  )
      _lowercase      :				Tuple      					=       XGLMTokenizer.from_pretrained("facebook/xglm-564M"  )

      _lowercase      :				Tuple      					=       "left"

      # use different length sentences to test batching
      _lowercase      :				Any      					=       [
          "This is an extremelly long sentence that only exists to test the ability of the model to cope with "
          "left-padding, such as in batched generation. The output for the sequence below should be the same "
          "regardless of whether left padding is applied or not. When",
          "Hello, my dog is a little",
      ]

      _lowercase      :				str      					=       tokenizer(_UpperCamelCase      ,    return_tensors="tf"      ,    padding=_UpperCamelCase  )
      _lowercase      :				int      					=       inputs["input_ids"]

      _lowercase      :				Optional[Any]      					=       model.generate(input_ids=_UpperCamelCase      ,    attention_mask=inputs["attention_mask"]      ,    max_new_tokens=12  )

      _lowercase      :				Dict      					=       tokenizer(sentences[0]      ,    return_tensors="tf"  ).input_ids
      _lowercase      :				List[Any]      					=       model.generate(input_ids=_UpperCamelCase      ,    max_new_tokens=12  )

      _lowercase      :				Union[str, Any]      					=       tokenizer(sentences[1]      ,    return_tensors="tf"  ).input_ids
      _lowercase      :				List[str]      					=       model.generate(input_ids=_UpperCamelCase      ,    max_new_tokens=12  )

      _lowercase      :				Any      					=       tokenizer.batch_decode(_UpperCamelCase      ,    skip_special_tokens=_UpperCamelCase  )
      _lowercase      :				Optional[Any]      					=       tokenizer.decode(output_non_padded[0]      ,    skip_special_tokens=_UpperCamelCase  )
      _lowercase      :				str      					=       tokenizer.decode(output_padded[0]      ,    skip_special_tokens=_UpperCamelCase  )

      _lowercase      :				List[Any]      					=       [
          "This is an extremelly long sentence that only exists to test the ability of the model to cope with "
          "left-padding, such as in batched generation. The output for the sequence below should be the same "
          "regardless of whether left padding is applied or not. When left padding is applied, the sequence will be "
          "a single",
          "Hello, my dog is a little bit of a shy one, but he is very friendly",
      ]
      self.assertListEqual(_UpperCamelCase      ,    _UpperCamelCase  )
      self.assertListEqual(_UpperCamelCase      ,    [non_padded_sentence, padded_sentence]  )









'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case         =  '▁'

_snake_case         =  {'vocab_file': 'spiece.model'}

_snake_case         =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'}
}

_snake_case         =  {
    'google/pegasus-xsum': 512,
}


_snake_case         =  logging.get_logger(__name__)


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					str     	=							VOCAB_FILES_NAMES

   _SCREAMING_SNAKE_CASE   :					Tuple     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							['input_ids', 'attention_mask']


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<mask_2>"      ,    _UpperCamelCase="<mask_1>"      ,    _UpperCamelCase=None      ,    _UpperCamelCase=103      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       offset
      if additional_special_tokens is not None:
         if not isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            raise TypeError(
                f'''additional_special_tokens should be of type {type(_UpperCamelCase  )}, but is'''
                f''' {type(_UpperCamelCase  )}'''  )

         _lowercase      :				Dict      					=       (
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f'''<unk_{i}>''' for i in range(len(_UpperCamelCase  )      ,    self.offset - 1  )
         ]

         if len(set(_UpperCamelCase  )  ) != len(_UpperCamelCase  ):
            raise ValueError(
                "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''  )
         _lowercase      :				List[str]      					=       additional_special_tokens_extended
      else:
         _lowercase      :				Dict      					=       [mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f'''<unk_{i}>''' for i in range(2      ,    self.offset  )]

      _lowercase      :				int      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    mask_token_sent=_UpperCamelCase      ,    offset=_UpperCamelCase      ,    additional_special_tokens=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )
      _lowercase      :				Union[str, Any]      					=       mask_token_sent
      _lowercase      :				str      					=       vocab_file
      _lowercase      :				Dict      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(_UpperCamelCase  )

      # add special tokens to encoder dict
      _lowercase      :				Dict[int, str]      					=       {
          0: self.pad_token,
          1: self.eos_token,
      }

      if self.mask_token_sent is not None:
         self.encoder.update(
             {
                 2: self.mask_token_sent,
                 3: self.mask_token,
             }  )

      if self.offset > 0:
         # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
         # mask_token_sent is already added to list -> so start at 1
         self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,    self.offset - 1  )}  )

      _lowercase      :				Dict[str, int]      					=       {v: k for k, v in self.encoder.items()}


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return len(self.sp_model  ) + self.offset


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       self.__dict__.copy()
      _lowercase      :				Union[str, Any]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				List[Any]      					=       {}

      _lowercase      :				Any      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if token in self.decoder:
         return self.decoder[token]
      elif token in self.added_tokens_decoder:
         return self.added_tokens_decoder[token]
      _lowercase      :				int      					=       self.sp_model.piece_to_id(_UpperCamelCase  )
      return sp_id + self.offset


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if index in self.encoder:
         return self.encoder[index]
      elif index in self.added_tokens_encoder:
         return self.added_tokens_encoder[index]
      else:
         _lowercase      :				Any      					=       self.sp_model.IdToPiece(index - self.offset  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       []
      _lowercase      :				Optional[Any]      					=       ""
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            out_string += self.sp_model.decode(_UpperCamelCase  ) + token
            _lowercase      :				Tuple      					=       []
         else:
            current_sub_tokens.append(_UpperCamelCase  )
      out_string += self.sp_model.decode(_UpperCamelCase  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase=False  ):





      """simple docstring"""





      return 1


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       set(self.all_special_ids  )  # call it once instead of inside list comp
      all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special

      return [1 if x in all_special_ids else 0 for x in seq]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return self._special_token_mask(_UpperCamelCase  )
      elif token_ids_a is None:
         return self._special_token_mask(_UpperCamelCase  ) + [1]
      else:
         return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				List[Any]      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				Optional[int]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)






