"""simple docstring"""
import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
 import torch

 from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel




class        __snake_case	:



      def __init__(						self   ,      lowercase   ,      lowercase=13   ,      lowercase=7   ,      lowercase=True   ,      lowercase=True   ,      lowercase=True   ,      lowercase=True   ,      lowercase=99   ,      lowercase=32   ,      lowercase=5   ,      lowercase=4   ,      lowercase=4   ,      lowercase="gelu"   ,      lowercase=0.0   ,      lowercase=0.1   ,      lowercase=True   ,      lowercase=5_12   ,      lowercase=16   ,      lowercase=2   ,      lowercase=0.02   ,      lowercase=3   ,      lowercase=4   ,      lowercase=None   ,      )     ->      List[str]:





        '''simple docstring'''





        a__:					int      							=     parent
        a__:					List[str]      							=     batch_size
        a__:					Dict      							=     seq_length
        a__:					Any      							=     is_training
        a__:					Any      							=     use_input_mask
        a__:					Any      							=     use_token_type_ids
        a__:					Dict      							=     use_labels
        a__:					Dict      							=     vocab_size
        a__:					Tuple      							=     hidden_size
        a__:					Optional[Any]      							=     num_hidden_layers
        a__:					str      							=     num_attention_heads
        a__:					str      							=     intermediate_multiple_size
        a__:					Dict      							=     hidden_act
        a__:					Optional[Any]      							=     hidden_dropout
        a__:					Tuple      							=     attention_dropout
        a__:					Union[str, Any]      							=     weight_tying
        a__:					List[Any]      							=     max_position_embeddings
        a__:					Tuple      							=     type_vocab_size
        a__:					Any      							=     type_sequence_label_size
        a__:					List[Any]      							=     initializer_range
        a__:					Optional[int]      							=     num_labels
        a__:					Any      							=     num_choices
        a__:					List[str]      							=     scope



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					Tuple      							=     ids_tensor([self.batch_size, self.seq_length]   ,      self.vocab_size)

        a__:					List[Any]      							=     None
        if self.use_input_mask:
          a__:					List[str]      							=     random_attention_mask([self.batch_size, self.seq_length])

        a__:					List[str]      							=     None
        if self.use_labels:
          a__:					Tuple      							=     ids_tensor([self.batch_size, self.seq_length]   ,      self.num_labels)

        a__:					Optional[Any]      							=     self.get_config()

        return config, input_ids, input_mask, token_labels



      def   lowerCamelCase_       (						self)     ->      Tuple:





        '''simple docstring'''





        return GPTNeoXJapaneseConfig(
            vocab_size=self.vocab_size   ,      hidden_size=self.hidden_size   ,      num_hidden_layers=self.num_hidden_layers   ,      num_attention_heads=self.num_attention_heads   ,      intermediate_multiple_size=self.intermediate_multiple_size   ,      hidden_act=self.hidden_act   ,      hidden_dropout=self.hidden_dropout   ,      attention_dropout=self.attention_dropout   ,      weight_tying=self.weight_tying   ,      max_position_embeddings=self.max_position_embeddings   ,      type_vocab_size=self.type_vocab_size   ,      is_decoder=lowercase   ,      initializer_range=self.initializer_range   ,      )



      def   lowerCamelCase_       (						self)     ->      str:





        '''simple docstring'''





        a__							,   a__							,   a__							,   a__:					Dict      							=     self.prepare_config_and_inputs()

        a__:					List[Any]      							=     True

        return config, input_ids, input_mask, token_labels



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      Optional[Any]:





        '''simple docstring'''





        a__:					Optional[Any]      							=     GPTNeoXJapaneseModel(config=lowercase)
        model.to(lowercase)
        model.eval()
        a__:					List[Any]      							=     model(lowercase   ,      attention_mask=lowercase)
        a__:					List[Any]      							=     model(lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape   ,      (self.batch_size, self.seq_length, self.hidden_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      str:





        '''simple docstring'''





        a__:					Any      							=     True
        a__:					Optional[int]      							=     GPTNeoXJapaneseModel(lowercase)
        model.to(lowercase)
        model.eval()
        a__:					str      							=     model(lowercase   ,      attention_mask=lowercase)
        self.parent.assertEqual(result.last_hidden_state.shape   ,      (self.batch_size, self.seq_length, self.hidden_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase   ,      lowercase)     ->      Union[str, Any]:





        '''simple docstring'''





        a__:					Union[str, Any]      							=     GPTNeoXJapaneseForCausalLM(config=lowercase)
        model.to(lowercase)
        model.eval()
        a__:					Union[str, Any]      							=     model(lowercase   ,      attention_mask=lowercase   ,      labels=lowercase)
        self.parent.assertEqual(result.logits.shape   ,      (self.batch_size, self.seq_length, self.vocab_size))



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase   ,      lowercase)     ->      Optional[Any]:





        '''simple docstring'''





        a__:					List[Any]      							=     True
        a__:					Optional[int]      							=     GPTNeoXJapaneseForCausalLM(config=lowercase)
        model.to(lowercase)
        model.eval()

        # first forward pass
        a__:					int      							=     model(lowercase   ,      attention_mask=lowercase   ,      use_cache=lowercase)
        a__:					int      							=     outputs.past_key_values

        # create hypothetical multiple next token and extent to next_input_ids
        a__:					Optional[int]      							=     ids_tensor((self.batch_size, 3)   ,      config.vocab_size)
        a__:					Dict      							=     ids_tensor((self.batch_size, 3)   ,      vocab_size=2)

        # append to next input_ids and
        a__:					Tuple      							=     torch.cat([input_ids, next_tokens]   ,      dim=-1)
        a__:					List[Any]      							=     torch.cat([input_mask, next_mask]   ,      dim=-1)

        a__:					Optional[int]      							=     model(lowercase   ,      attention_mask=lowercase   ,      output_hidden_states=lowercase)
        a__:					Tuple      							=     output_from_no_past['hidden_states'][0]
        a__:					Dict      							=     model(
            lowercase   ,      attention_mask=lowercase   ,      past_key_values=lowercase   ,      output_hidden_states=lowercase   ,      )['hidden_states'][0]

        # select random slice
        a__:					str      							=     ids_tensor((1,)   ,      output_from_past.shape[-1]).item()
        a__:					Tuple      							=     output_from_no_past[:, -3:, random_slice_idx].detach()
        a__:					int      							=     output_from_past[:, :, random_slice_idx].detach()

        self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1])

        # test that outputs are equal for slice
        self.parent.assertTrue(torch.allclose(lowercase   ,      lowercase   ,      atol=1e-3))



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					str      							=     self.prepare_config_and_inputs()
        a__							,   a__							,   a__							,   a__:					Union[str, Any]      							=     config_and_inputs
        a__:					int      							=     {'input_ids': input_ids, 'attention_mask': input_mask}
        return config, inputs_dict







@require_torch
class        __snake_case	(					__lowerCAmelCase	,				__lowerCAmelCase	,				unittest.TestCase    ):
      a__				=		(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
      a__				=		(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
      a__				=		(
          {"""feature-extraction""": GPTNeoXJapaneseModel, """text-generation""": GPTNeoXJapaneseForCausalLM}
          if is_torch_available()
          else {}
      )
      a__				=		False
      a__				=		False
      a__				=		False
      a__				=		False



      def   lowerCamelCase_       (						self)     ->      Dict:





        '''simple docstring'''





        a__:					Optional[int]      							=     GPTNeoXJapaneseModelTester(self)
        a__:					Tuple      							=     ConfigTester(self   ,      config_class=lowercase   ,      hidden_size=37)



      def   lowerCamelCase_       (						self)     ->      List[str]:





        '''simple docstring'''





        self.config_tester.run_common_tests()



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__							,   a__							,   a__							,   a__:					str      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(lowercase   ,      lowercase   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      Union[str, Any]:





        '''simple docstring'''





        a__							,   a__							,   a__							,   a__:					Union[str, Any]      							=     self.model_tester.prepare_config_and_inputs_for_decoder()
        self.model_tester.create_and_check_model_as_decoder(lowercase   ,      lowercase   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__							,   a__							,   a__							,   a__:					Any      							=     self.model_tester.prepare_config_and_inputs_for_decoder()

        a__:					Optional[int]      							=     None

        self.model_tester.create_and_check_model_as_decoder(lowercase   ,      lowercase   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      Any:





        '''simple docstring'''





        a__							,   a__							,   a__							,   a__:					Any      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_decoder_model_past_large_inputs(lowercase   ,      lowercase   ,      lowercase)



      def   lowerCamelCase_       (						self)     ->      Optional[int]:





        '''simple docstring'''





        a__:					Dict      							=     self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_causal_lm(*lowercase)



      @slow
      def   lowerCamelCase_       (						self)     ->      Tuple:





        '''simple docstring'''





        a__:					Dict      							=     'abeja/gpt-neox-japanese-2.7b'

        a__:					Dict      							=     ['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

        a__:					str      							=     [
            'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
            '100年後に必要とされる会社は、「人」が中心の会社です。',
            'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
            '国境の長いトンネルを抜けると、そこは雪国だった。',
            '美味しい日本食といえば、やっぱりお寿司ですよね。',
        ]

        a__:					Union[str, Any]      							=     GPTNeoXJapaneseTokenizer.from_pretrained(lowercase)
        a__:					Dict      							=     GPTNeoXJapaneseForCausalLM.from_pretrained(lowercase)

        a__:					int      							=     []
        for prompt in prompts:
          a__:					List[Any]      							=     tokenizer(lowercase   ,      return_tensors='pt').input_ids
          a__:					Dict      							=     model.generate(lowercase   ,      max_length=50)
          a__:					Dict      							=     tokenizer.batch_decode(lowercase   ,      skip_special_tokens=lowercase)
          predicted_outputs += generated_string
        self.assertListEqual(lowercase   ,      lowercase)



"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase__ 	=     {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase__ 	=     {
    'vocab_file': {
        'unc-nlp/lxmert-base-uncased': 'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'unc-nlp/lxmert-base-uncased': (
            'https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json'
        ),
    },
}

lowercase__ 	=     {
    'unc-nlp/lxmert-base-uncased': 512,
}

lowercase__ 	=     {
    'unc-nlp/lxmert-base-uncased': {'do_lower_case': True},
}




class        __snake_case	(					__lowerCAmelCase    ):
      a__				=		VOCAB_FILES_NAMES
      a__				=		PRETRAINED_VOCAB_FILES_MAP
      a__				=		PRETRAINED_INIT_CONFIGURATION
      a__				=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      a__				=		LxmertTokenizer



      def __init__(						self   ,      lowercase=None   ,      lowercase=None   ,      lowercase=True   ,      lowercase="[UNK]"   ,      lowercase="[SEP]"   ,      lowercase="[PAD]"   ,      lowercase="[CLS]"   ,      lowercase="[MASK]"   ,      lowercase=True   ,      lowercase=None   ,      **lowercase   ,      )     ->      Dict:





        '''simple docstring'''





        super().__init__(
            lowercase   ,      tokenizer_file=lowercase   ,      do_lower_case=lowercase   ,      unk_token=lowercase   ,      sep_token=lowercase   ,      pad_token=lowercase   ,      cls_token=lowercase   ,      mask_token=lowercase   ,      tokenize_chinese_chars=lowercase   ,      strip_accents=lowercase   ,      **lowercase   ,      )

        a__:					Dict      							=     json.loads(self.backend_tokenizer.normalizer.__getstate__())
        if (
            normalizer_state.get('lowercase'   ,      lowercase) != do_lower_case
            or normalizer_state.get('strip_accents'   ,      lowercase) != strip_accents
            or normalizer_state.get('handle_chinese_chars'   ,      lowercase) != tokenize_chinese_chars
        ):
          a__:					int      							=     getattr(lowercase   ,      normalizer_state.pop('type'))
          a__:					Dict      							=     do_lower_case
          a__:					Dict      							=     strip_accents
          a__:					Optional[int]      							=     tokenize_chinese_chars
          a__:					List[Any]      							=     normalizer_class(**lowercase)

        a__:					Optional[int]      							=     do_lower_case



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase=None)     ->      Tuple:





        '''simple docstring'''





        a__:					Optional[Any]      							=     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase = None)     ->      List[int]:





        '''simple docstring'''





        a__:					List[Any]      							=     [self.sep_token_id]
        a__:					List[Any]      							=     [self.cls_token_id]
        if token_ids_a is None:
          return len(cls + token_ids_a + sep) * [0]
        return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



      def   lowerCamelCase_       (						self   ,      lowercase   ,      lowercase = None)     ->      Tuple[str]:





        '''simple docstring'''





        a__:					List[Any]      							=     self._tokenizer.model.save(lowercase   ,      name=lowercase)
        return tuple(lowercase)


