



"""simple docstring"""




import unittest

from transformers import MraConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_torch_available():
 import torch

 from transformers import (
     MraForMaskedLM,
     MraForMultipleChoice,
     MraForQuestionAnswering,
     MraForSequenceClassification,
     MraForTokenClassification,
     MraModel,
 )
 from transformers.models.mra.modeling_mra import MRA_PRETRAINED_MODEL_ARCHIVE_LIST






class      UpperCAmelCase_  :



      def __init__(					self					:     Any					,     A					:     Any					,     A					:     int=2					,     A					:     List[str]=8					,     A					:     Any=True					,     A					:     Tuple=True					,     A					:     Union[str, Any]=True					,     A					:     Any=True					,     A					:     Any=9_9					,     A					:     int=1_6					,     A					:     Union[str, Any]=5					,     A					:     Optional[Any]=2					,     A					:     List[Any]=3_6					,     A					:     Dict="gelu"					,     A					:     List[Any]=0.0					,     A					:     int=0.0					,     A					:     List[str]=5_1_2					,     A					:     int=1_6					,     A					:     Optional[int]=2					,     A					:     Any=0.02					,     A					:     Tuple=3					,     A					:     Union[str, Any]=4					,     A					:     Dict=None					,     ):
           _UpperCAmelCase      :     List[str]        =   parent
           _UpperCAmelCase      :     Tuple        =   batch_size
           _UpperCAmelCase      :     Tuple        =   seq_length
           _UpperCAmelCase      :     str        =   is_training
           _UpperCAmelCase      :     List[Any]        =   use_input_mask
           _UpperCAmelCase      :     Optional[Any]        =   use_token_type_ids
           _UpperCAmelCase      :     Any        =   use_labels
           _UpperCAmelCase      :     Tuple        =   vocab_size
           _UpperCAmelCase      :     Union[str, Any]        =   hidden_size
           _UpperCAmelCase      :     str        =   num_hidden_layers
           _UpperCAmelCase      :     Optional[Any]        =   num_attention_heads
           _UpperCAmelCase      :     str        =   intermediate_size
           _UpperCAmelCase      :     Optional[Any]        =   hidden_act
           _UpperCAmelCase      :     Union[str, Any]        =   hidden_dropout_prob
           _UpperCAmelCase      :     str        =   attention_probs_dropout_prob
           _UpperCAmelCase      :     List[Any]        =   max_position_embeddings
           _UpperCAmelCase      :     Tuple        =   type_vocab_size
           _UpperCAmelCase      :     Any        =   type_sequence_label_size
           _UpperCAmelCase      :     Optional[Any]        =   initializer_range
           _UpperCAmelCase      :     Optional[int]        =   num_labels
           _UpperCAmelCase      :     Optional[Any]        =   num_choices
           _UpperCAmelCase      :     Any        =   scope



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     str        =   ids_tensor([self.batch_size, self.seq_length]					,     self.vocab_size						)

           _UpperCAmelCase      :     Optional[int]        =   None
           if self.use_input_mask:
                _UpperCAmelCase      :     str        =   random_attention_mask([self.batch_size, self.seq_length]						)

           _UpperCAmelCase      :     List[Any]        =   None
           if self.use_token_type_ids:
                _UpperCAmelCase      :     List[str]        =   ids_tensor([self.batch_size, self.seq_length]					,     self.type_vocab_size						)

           _UpperCAmelCase      :     List[Any]        =   None
           _UpperCAmelCase      :     List[str]        =   None
           _UpperCAmelCase      :     Optional[Any]        =   None
           if self.use_labels:
                _UpperCAmelCase      :     Any        =   ids_tensor([self.batch_size]					,     self.type_sequence_label_size						)
                _UpperCAmelCase      :     Tuple        =   ids_tensor([self.batch_size, self.seq_length]					,     self.num_labels						)
                _UpperCAmelCase      :     int        =   ids_tensor([self.batch_size]					,     self.num_choices						)

           _UpperCAmelCase      :     Optional[int]        =   self.get_config()

           return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels



      def 						snake_case_ (					self					:     Optional[Any]						):
           return MraConfig(
               vocab_size=self.vocab_size					,     hidden_size=self.hidden_size					,     num_hidden_layers=self.num_hidden_layers					,     num_attention_heads=self.num_attention_heads					,     intermediate_size=self.intermediate_size					,     hidden_act=self.hidden_act					,     hidden_dropout_prob=self.hidden_dropout_prob					,     attention_probs_dropout_prob=self.attention_probs_dropout_prob					,     max_position_embeddings=self.max_position_embeddings					,     type_vocab_size=self.type_vocab_size					,     is_decoder=SCREAMING_SNAKE_CASE_					,     initializer_range=self.initializer_range					,     )



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     List[str]        =   self.get_config()
           _UpperCAmelCase      :     int        =   3_0_0
           return config



      def 						snake_case_ (					self					:     str						):
           (
               _UpperCAmelCase							
           )      :     Tuple        =   self.prepare_config_and_inputs()

           _UpperCAmelCase      :     str        =   True
           _UpperCAmelCase      :     Any        =   floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
           _UpperCAmelCase      :     int        =   ids_tensor([self.batch_size, self.seq_length]					,     vocab_size=2						)

           return (
               config,
               input_ids,
               token_type_ids,
               input_mask,
               sequence_labels,
               token_labels,
               choice_labels,
               encoder_hidden_states,
               encoder_attention_mask,
           )



      def 						snake_case_ (					self					:     str					,     A					:     str					,     A					:     Any					,     A					:     List[str]					,     A					:     str					,     A					:     Tuple					,     A					:     List[str]					,     A					:     List[Any]						):
           _UpperCAmelCase      :     List[Any]        =   MraModel(config=SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     Dict        =   model(SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_						)
           _UpperCAmelCase      :     List[Any]        =   model(SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_						)
           _UpperCAmelCase      :     Tuple        =   model(SCREAMING_SNAKE_CASE_						)
           self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)						)



      def 						snake_case_ (					self					:     Optional[int]					,     A					:     Dict					,     A					:     List[str]					,     A					:     Tuple					,     A					:     Tuple					,     A					:     Optional[int]					,     A					:     Union[str, Any]					,     A					:     Dict					,     A					:     List[Any]					,     A					:     List[str]					,     ):
           _UpperCAmelCase      :     Optional[Any]        =   True
           _UpperCAmelCase      :     int        =   MraModel(SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     Tuple        =   model(
               SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     encoder_hidden_states=SCREAMING_SNAKE_CASE_					,     encoder_attention_mask=SCREAMING_SNAKE_CASE_					,     )
           _UpperCAmelCase      :     Any        =   model(
               SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     encoder_hidden_states=SCREAMING_SNAKE_CASE_					,     )
           _UpperCAmelCase      :     Any        =   model(SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_						)
           self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)						)



      def 						snake_case_ (					self					:     str					,     A					:     str					,     A					:     str					,     A					:     Any					,     A					:     Tuple					,     A					:     Tuple					,     A					:     str					,     A					:     Optional[int]						):
           _UpperCAmelCase      :     Union[str, Any]        =   MraForMaskedLM(config=SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     int        =   model(SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     labels=SCREAMING_SNAKE_CASE_						)
           self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.seq_length, self.vocab_size)						)



      def 						snake_case_ (					self					:     List[Any]					,     A					:     List[Any]					,     A					:     Any					,     A					:     List[str]					,     A					:     Dict					,     A					:     Tuple					,     A					:     Union[str, Any]					,     A					:     Dict						):
           _UpperCAmelCase      :     str        =   MraForQuestionAnswering(config=SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     Any        =   model(
               SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     start_positions=SCREAMING_SNAKE_CASE_					,     end_positions=SCREAMING_SNAKE_CASE_					,     )
           self.parent.assertEqual(result.start_logits.shape					,     (self.batch_size, self.seq_length)						)
           self.parent.assertEqual(result.end_logits.shape					,     (self.batch_size, self.seq_length)						)



      def 						snake_case_ (					self					:     Dict					,     A					:     Optional[Any]					,     A					:     Tuple					,     A					:     Optional[int]					,     A					:     Optional[Any]					,     A					:     Tuple					,     A					:     Tuple					,     A					:     Any						):
           _UpperCAmelCase      :     Tuple        =   self.num_labels
           _UpperCAmelCase      :     Optional[int]        =   MraForSequenceClassification(SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     Any        =   model(SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     labels=SCREAMING_SNAKE_CASE_						)
           self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.num_labels)						)



      def 						snake_case_ (					self					:     int					,     A					:     List[str]					,     A					:     Tuple					,     A					:     int					,     A					:     Any					,     A					:     Any					,     A					:     Optional[int]					,     A					:     Dict						):
           _UpperCAmelCase      :     str        =   self.num_labels
           _UpperCAmelCase      :     int        =   MraForTokenClassification(config=SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     Optional[Any]        =   model(SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     labels=SCREAMING_SNAKE_CASE_						)
           self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.seq_length, self.num_labels)						)



      def 						snake_case_ (					self					:     Union[str, Any]					,     A					:     str					,     A					:     Any					,     A					:     Tuple					,     A					:     Optional[int]					,     A					:     List[Any]					,     A					:     Optional[int]					,     A					:     Union[str, Any]						):
           _UpperCAmelCase      :     Optional[Any]        =   self.num_choices
           _UpperCAmelCase      :     Union[str, Any]        =   MraForMultipleChoice(config=SCREAMING_SNAKE_CASE_						)
           model.to(SCREAMING_SNAKE_CASE_						)
           model.eval()
           _UpperCAmelCase      :     Optional[int]        =   input_ids.unsqueeze(1						).expand(-1					,     self.num_choices					,     -1						).contiguous()
           _UpperCAmelCase      :     Optional[int]        =   token_type_ids.unsqueeze(1						).expand(-1					,     self.num_choices					,     -1						).contiguous()
           _UpperCAmelCase      :     List[str]        =   input_mask.unsqueeze(1						).expand(-1					,     self.num_choices					,     -1						).contiguous()
           _UpperCAmelCase      :     str        =   model(
               SCREAMING_SNAKE_CASE_					,     attention_mask=SCREAMING_SNAKE_CASE_					,     token_type_ids=SCREAMING_SNAKE_CASE_					,     labels=SCREAMING_SNAKE_CASE_					,     )
           self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.num_choices)						)



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     Any        =   self.prepare_config_and_inputs()
           (
               _UpperCAmelCase							
           )      :     Any        =   config_and_inputs
           _UpperCAmelCase      :     Union[str, Any]        =   {"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
           return config, inputs_dict







@require_torch
class      UpperCAmelCase_  (    _UpperCAmelCase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       Tuple								=			(
          (
              MraModel,
              MraForMaskedLM,
              MraForMultipleChoice,
              MraForQuestionAnswering,
              MraForSequenceClassification,
              MraForTokenClassification,
          )
          if is_torch_available()
          else ()
      )
      __SCREAMING_SNAKE_CASE						:       List[str]								=			False
      __SCREAMING_SNAKE_CASE						:       List[Any]								=			False
      __SCREAMING_SNAKE_CASE						:       List[Any]								=			False
      __SCREAMING_SNAKE_CASE						:       Optional[int]								=			False

      __SCREAMING_SNAKE_CASE						:       List[Any]								=			()



      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     List[str]        =   MraModelTester(self						)
           _UpperCAmelCase      :     str        =   ConfigTester(self					,     config_class=SCREAMING_SNAKE_CASE_					,     hidden_size=3_7						)



      def 						snake_case_ (					self					:     List[str]						):
           self.config_tester.run_common_tests()



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     Tuple        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_						)



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     List[Any]        =   self.model_tester.prepare_config_and_inputs()
           for type in ["absolute", "relative_key", "relative_key_query"]:
                _UpperCAmelCase      :     Optional[int]        =   type
                self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE_						)



      def 						snake_case_ (					self					:     Optional[Any]						):
           _UpperCAmelCase      :     Union[str, Any]        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE_						)



      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     Optional[int]        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE_						)



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     int        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE_						)



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     Optional[Any]        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE_						)



      def 						snake_case_ (					self					:     Tuple						):
           _UpperCAmelCase      :     List[str]        =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE_						)



      @slow
      def 						snake_case_ (					self					:     Dict						):
           for model_name in MRA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                _UpperCAmelCase      :     Dict        =   MraModel.from_pretrained(SCREAMING_SNAKE_CASE_						)
                self.assertIsNotNone(SCREAMING_SNAKE_CASE_						)



      @unittest.skip(reason="MRA does not output attentions"						)
      def 						snake_case_ (					self					:     Union[str, Any]						):
           return







@require_torch
class      UpperCAmelCase_  (    unittest.TestCase						):



      @slow
      def 						snake_case_ (					self					:     str						):
           _UpperCAmelCase      :     List[str]        =   MraModel.from_pretrained("uw-madison/mra-base-512-4"						)
           _UpperCAmelCase      :     int        =   torch.arange(2_5_6						).unsqueeze(0						)

           with torch.no_grad():
                _UpperCAmelCase      :     Union[str, Any]        =   model(SCREAMING_SNAKE_CASE_						)[0]

           _UpperCAmelCase      :     str        =   torch.Size((1, 2_5_6, 7_6_8)						)
           self.assertEqual(output.shape					,     SCREAMING_SNAKE_CASE_						)

           _UpperCAmelCase      :     Optional[int]        =   torch.tensor(
               [[[-0.0_140, 0.0_830, -0.0_381], [0.1_546, 0.1_402, 0.0_220], [0.1_162, 0.0_851, 0.0_165]]]						)

           self.assertTrue(torch.allclose(output[:, :3, :3]					,     SCREAMING_SNAKE_CASE_					,     atol=1e-4						)						)



      @slow
      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     Union[str, Any]        =   MraForMaskedLM.from_pretrained("uw-madison/mra-base-512-4"						)
           _UpperCAmelCase      :     int        =   torch.arange(2_5_6						).unsqueeze(0						)

           with torch.no_grad():
                _UpperCAmelCase      :     Union[str, Any]        =   model(SCREAMING_SNAKE_CASE_						)[0]

           _UpperCAmelCase      :     Optional[Any]        =   5_0_2_6_5

           _UpperCAmelCase      :     Optional[int]        =   torch.Size((1, 2_5_6, vocab_size)						)
           self.assertEqual(output.shape					,     SCREAMING_SNAKE_CASE_						)

           _UpperCAmelCase      :     Any        =   torch.tensor(
               [[[9.2_595, -3.6_038, 11.8_819], [9.3_869, -3.2_693, 11.0_956], [11.8_524, -3.4_938, 13.1_210]]]						)

           self.assertTrue(torch.allclose(output[:, :3, :3]					,     SCREAMING_SNAKE_CASE_					,     atol=1e-4						)						)



      @slow
      def 						snake_case_ (					self					:     Optional[Any]						):
           _UpperCAmelCase      :     Optional[Any]        =   MraForMaskedLM.from_pretrained("uw-madison/mra-base-4096-8-d3"						)
           _UpperCAmelCase      :     List[str]        =   torch.arange(4_0_9_6						).unsqueeze(0						)

           with torch.no_grad():
                _UpperCAmelCase      :     int        =   model(SCREAMING_SNAKE_CASE_						)[0]

           _UpperCAmelCase      :     Optional[Any]        =   5_0_2_6_5

           _UpperCAmelCase      :     Any        =   torch.Size((1, 4_0_9_6, vocab_size)						)
           self.assertEqual(output.shape					,     SCREAMING_SNAKE_CASE_						)

           _UpperCAmelCase      :     Union[str, Any]        =   torch.tensor(
               [[[5.4_789, -2.3_564, 7.5_064], [7.9_067, -1.3_369, 9.9_668], [9.0_712, -1.8_106, 7.0_380]]]						)

           self.assertTrue(torch.allclose(output[:, :3, :3]					,     SCREAMING_SNAKE_CASE_					,     atol=1e-4						)						)





"""simple docstring"""




import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class      UpperCAmelCase_  (    _UpperCamelCase	,      unittest.TestCase						):
      __SCREAMING_SNAKE_CASE						:       Dict								=			GPTSanJapaneseTokenizer
      __SCREAMING_SNAKE_CASE						:       Optional[int]								=			False
      __SCREAMING_SNAKE_CASE						:       List[str]								=			{'do_clean_text': False, 'add_prefix_space': False}



      def 						snake_case_ (					self					:     Any						):
           super().setUp()

           # fmt: off
           _UpperCAmelCase      :     Any        =   ["こん", "こんに", "にちは", "ばんは", "世界,㔺界", "、", "。", "<BR>", "<SP>", "<TAB>", "<URL>", "<EMAIL>", "<TEL>", "<DATE>", "<PRICE>", "<BLOCK>", "<KIGOU>", "<U2000U2BFF>", "<|emoji1|>", "<unk>", "<|bagoftoken|>", "<|endoftext|>"]
           # fmt: on
           _UpperCAmelCase      :     Optional[int]        =   {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
           _UpperCAmelCase      :     List[Any]        =   {"unk_token": "<unk>"}

           _UpperCAmelCase      :     List[Any]        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["vocab_file"]						)
           _UpperCAmelCase      :     Any        =   os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES["emoji_file"]						)
           with open(self.vocab_file					,     "w"					,     encoding="utf-8"						) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]						)						)
           with open(self.emoji_file					,     "w"						) as emoji_writer:
                emoji_writer.write(json.dumps(A						)						)



      def 						snake_case_ (					self					:     int					,     **A					:     List[str]						):
           kwargs.update(self.special_tokens_map						)
           return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname					,     **A						)



      def 						snake_case_ (					self					:     int					,     A					:     Any						):
           _UpperCAmelCase      :     Optional[Any]        =   "こんにちは、世界。 \nこんばんは、㔺界。😀"
           _UpperCAmelCase      :     List[Any]        =   "こんにちは、世界。 \nこんばんは、世界。😀"
           return input_text, output_text



      def 						snake_case_ (					self					:     Optional[Any]					,     A					:     str						):
           _UpperCAmelCase    ,	_UpperCAmelCase      :     str        =   self.get_input_output_texts(A						)
           _UpperCAmelCase      :     List[Any]        =   tokenizer.encode(A					,     add_special_tokens=A						)
           _UpperCAmelCase      :     Union[str, Any]        =   tokenizer.decode(A					,     clean_up_tokenization_spaces=A						)
           return text, ids



      def 						snake_case_ (					self					:     Any						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     Union[str, Any]						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     int						):
           pass  # TODO add if relevant



      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     List[Any]        =   self.get_tokenizer()

           # Testing tokenization
           _UpperCAmelCase      :     Optional[int]        =   "こんにちは、世界。　こんばんは、㔺界。"
           _UpperCAmelCase      :     Dict        =   ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
           _UpperCAmelCase      :     List[Any]        =   tokenizer.tokenize(A						)
           self.assertListEqual(A					,     A						)

           # Testing conversion to ids without special tokens
           _UpperCAmelCase      :     Optional[int]        =   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
           _UpperCAmelCase      :     List[str]        =   tokenizer.convert_tokens_to_ids(A						)
           self.assertListEqual(A					,     A						)

           # Testing conversion to ids with special tokens
           _UpperCAmelCase      :     str        =   tokens + [tokenizer.unk_token]
           _UpperCAmelCase      :     Any        =   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
           _UpperCAmelCase      :     str        =   tokenizer.convert_tokens_to_ids(A						)
           self.assertListEqual(A					,     A						)



      def 						snake_case_ (					self					:     Any						):
           _UpperCAmelCase      :     Union[str, Any]        =   self.get_tokenizer()

           # Testing tokenization
           _UpperCAmelCase      :     Dict        =   "こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"
           _UpperCAmelCase      :     Tuple        =   "こんにちは、、、、世界。こんばんは、、、、世界。"
           _UpperCAmelCase      :     int        =   tokenizer.encode(A						)
           _UpperCAmelCase      :     Optional[Any]        =   tokenizer.decode(A						)
           self.assertEqual(A					,     A						)



      @slow
      def 						snake_case_ (					self					:     Dict						):
           _UpperCAmelCase      :     List[Any]        =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

           # Testing tokenization
           _UpperCAmelCase      :     List[Any]        =   "こんにちは、世界。"
           _UpperCAmelCase      :     List[str]        =   "こんばんは、㔺界。😀"
           _UpperCAmelCase      :     Any        =   "こんにちは、世界。こんばんは、世界。😀"
           _UpperCAmelCase      :     Union[str, Any]        =   tokenizer.encode(prefix_text + input_text						)
           _UpperCAmelCase      :     Tuple        =   tokenizer.encode(""					,     prefix_text=prefix_text + input_text						)
           _UpperCAmelCase      :     Optional[int]        =   tokenizer.encode(A					,     prefix_text=A						)
           _UpperCAmelCase      :     Tuple        =   tokenizer.decode(A						)
           _UpperCAmelCase      :     Optional[Any]        =   tokenizer.decode(A						)
           _UpperCAmelCase      :     Tuple        =   tokenizer.decode(A						)
           self.assertEqual(A					,     A						)
           self.assertEqual(A					,     A						)
           self.assertEqual(A					,     A						)



      @slow
      def 						snake_case_ (					self					:     Optional[Any]						):
           _UpperCAmelCase      :     Dict        =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

           # Testing tokenization
           _UpperCAmelCase      :     Any        =   "こんにちは、世界。"
           _UpperCAmelCase      :     List[Any]        =   "こんばんは、㔺界。😀"

           _UpperCAmelCase      :     Optional[Any]        =   len(tokenizer.encode(A						)						) - 2
           _UpperCAmelCase      :     List[Any]        =   len(tokenizer.encode(A						)						) - 2

           _UpperCAmelCase      :     List[str]        =   [1] + [0] * (len_prefix + len_text + 1)
           _UpperCAmelCase      :     str        =   [1] * (len_prefix + len_text + 1) + [0]
           _UpperCAmelCase      :     int        =   [1] + [1] * (len_prefix) + [0] * (len_text + 1)

           _UpperCAmelCase      :     Union[str, Any]        =   tokenizer(prefix_text + input_text						).token_type_ids
           _UpperCAmelCase      :     Any        =   tokenizer(""					,     prefix_text=prefix_text + input_text						).token_type_ids
           _UpperCAmelCase      :     List[Any]        =   tokenizer(A					,     prefix_text=A						).token_type_ids
           self.assertListEqual(A					,     A						)
           self.assertListEqual(A					,     A						)
           self.assertListEqual(A					,     A						)



      @slow
      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     str        =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

           _UpperCAmelCase      :     Dict        =   tokenizer.encode("あンいワ"						)
           _UpperCAmelCase      :     str        =   tokenizer.encode(""					,     prefix_text="あンいワ"						)
           _UpperCAmelCase      :     Dict        =   tokenizer.encode("いワ"					,     prefix_text="あン"						)

           self.assertEqual(tokenizer.decode(A						)					,     tokenizer.decode(A						)						)
           self.assertEqual(tokenizer.decode(A						)					,     tokenizer.decode(A						)						)
           self.assertNotEqual(A					,     A						)
           self.assertNotEqual(A					,     A						)
           self.assertEqual(x_token_a[1]					,     x_token_a[-1]						)  # SEG token
           self.assertEqual(x_token_a[1]					,     x_token_a[3]						)  # SEG token



      @slow
      def 						snake_case_ (					self					:     List[str]						):
           _UpperCAmelCase      :     Optional[Any]        =   self.tokenizer_class.from_pretrained("Tanrei/GPTSAN-japanese"						)

           _UpperCAmelCase      :     Tuple        =   [["武田信玄", "は、"], ["織田信長", "の配下の、"]]
           _UpperCAmelCase      :     Tuple        =   tokenizer(A					,     padding=A						)
           _UpperCAmelCase      :     str        =   tokenizer.batch_encode_plus(A					,     padding=A						)

           # fmt: off
           _UpperCAmelCase      :     str        =   [[3_5_9_9_3, 8_6_4_0, 2_5_9_4_8, 3_5_9_9_8, 3_0_6_4_7, 3_5_6_7_5, 3_5_9_9_9, 3_5_9_9_9], [3_5_9_9_3, 1_0_3_8_2, 9_8_6_8, 3_5_9_9_8, 3_0_6_4_6, 9_4_5_9, 3_0_6_4_6, 3_5_6_7_5]]
           _UpperCAmelCase      :     str        =   [[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
           _UpperCAmelCase      :     int        =   [[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
           # fmt: on
           self.assertListEqual(x_token.input_ids					,     A						)
           self.assertListEqual(x_token.token_type_ids					,     A						)
           self.assertListEqual(x_token.attention_mask					,     A						)
           self.assertListEqual(x_token_a.input_ids					,     A						)
           self.assertListEqual(x_token_a.token_type_ids					,     A						)
           self.assertListEqual(x_token_a.attention_mask					,     A						)



      def 						snake_case_ (					self					:     List[Any]						):
           # Intentionally convert some words to accommodate character fluctuations unique to Japanese
           pass



      def 						snake_case_ (					self					:     int						):
           # tokenizer has no padding token
           pass
