





"""simple docstring"""
import json
import os
import unittest

from transformers import BatchEncoding, LEDTokenizer, LEDTokenizerFast
from transformers.models.led.tokenization_led import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class  _SCREAMING_SNAKE_CASE(       A   ,     unittest.TestCase  ):
       SCREAMING_SNAKE_CASE_       :				Union[str, Any]     						=  LEDTokenizer
       SCREAMING_SNAKE_CASE_       :				Dict     						=  LEDTokenizerFast
       SCREAMING_SNAKE_CASE_       :				str     						=  True




       def    _UpperCamelCase					(						self							)      ->				int:





             """simple docstring"""






             super().setUp()
             __SCREAMING_SNAKE_CASE       :Tuple		    =  [
                 '''l''',
                 '''o''',
                 '''w''',
                 '''e''',
                 '''r''',
                 '''s''',
                 '''t''',
                 '''i''',
                 '''d''',
                 '''n''',
                 '''\u0120''',
                 '''\u0120l''',
                 '''\u0120n''',
                 '''\u0120lo''',
                 '''\u0120low''',
                 '''er''',
                 '''\u0120lowest''',
                 '''\u0120newer''',
                 '''\u0120wider''',
                 '''<unk>''',
             ]
             __SCREAMING_SNAKE_CASE       :List[str]		    =  dict(zip(SCREAMING_SNAKE_CASE__		,range(len(SCREAMING_SNAKE_CASE__							)							)							)							)
             __SCREAMING_SNAKE_CASE       :Tuple		    =  ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
             __SCREAMING_SNAKE_CASE       :str		    =  {'''unk_token''': '''<unk>'''}

             __SCREAMING_SNAKE_CASE       :Tuple		    =  os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''vocab_file''']							)
             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  os.path.join(self.tmpdirname		,VOCAB_FILES_NAMES['''merges_file''']							)
             with open(self.vocab_file		,'''w'''		,encoding='''utf-8'''							) as fp:
                   fp.write(json.dumps(SCREAMING_SNAKE_CASE__							) + '''\n'''							)
             with open(self.merges_file		,'''w'''		,encoding='''utf-8'''							) as fp:
                   fp.write('''\n'''.join(SCREAMING_SNAKE_CASE__							)							)




       def    _UpperCamelCase					(						self		,**SCREAMING_SNAKE_CASE__							)      ->				List[str]:





             """simple docstring"""






             kwargs.update(self.special_tokens_map							)
             return self.tokenizer_class.from_pretrained(self.tmpdirname		,**SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self		,**SCREAMING_SNAKE_CASE__							)      ->				Optional[Any]:





             """simple docstring"""






             kwargs.update(self.special_tokens_map							)
             return self.rust_tokenizer_class.from_pretrained(self.tmpdirname		,**SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__							)      ->				List[str]:





             """simple docstring"""






             return "lower newer", "lower newer"




       @cached_property
       def    _UpperCamelCase					(						self							)      ->				Union[str, Any]:





             """simple docstring"""






             return LEDTokenizer.from_pretrained('''allenai/led-base-16384'''							)




       @cached_property
       def    _UpperCamelCase					(						self							)      ->				List[Any]:





             """simple docstring"""






             return LEDTokenizerFast.from_pretrained('''allenai/led-base-16384'''							)




       @require_torch
       def    _UpperCamelCase					(						self							)      ->				Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Dict		    =  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
             __SCREAMING_SNAKE_CASE       :str		    =  [0, 2_50, 2_51, 1_78_18, 13, 3_91_86, 19_38, 4, 2]

             for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                   __SCREAMING_SNAKE_CASE       :Any		    =  tokenizer(SCREAMING_SNAKE_CASE__		,max_length=len(SCREAMING_SNAKE_CASE__							)		,padding=SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)
                   self.assertIsInstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)

                   self.assertEqual((2, 9)		,batch.input_ids.shape							)
                   self.assertEqual((2, 9)		,batch.attention_mask.shape							)
                   __SCREAMING_SNAKE_CASE       :Tuple		    =  batch.input_ids.tolist()[0]
                   self.assertListEqual(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)




       @require_torch
       def    _UpperCamelCase					(						self							)      ->				str:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Tuple		    =  ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
             for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                   __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  tokenizer(SCREAMING_SNAKE_CASE__		,padding=SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)
                   self.assertIn('''input_ids'''		,SCREAMING_SNAKE_CASE__							)
                   self.assertIn('''attention_mask'''		,SCREAMING_SNAKE_CASE__							)
                   self.assertNotIn('''labels'''		,SCREAMING_SNAKE_CASE__							)
                   self.assertNotIn('''decoder_attention_mask'''		,SCREAMING_SNAKE_CASE__							)




       @require_torch
       def    _UpperCamelCase					(						self							)      ->				Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[str]		    =  [
                 '''Summary of the text.''',
                 '''Another summary.''',
             ]
             for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                   __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  tokenizer(text_target=SCREAMING_SNAKE_CASE__		,max_length=32		,padding='''max_length'''		,return_tensors='''pt'''							)
                   self.assertEqual(32		,targets['''input_ids'''].shape[1]							)




       @require_torch
       def    _UpperCamelCase					(						self							)      ->				Tuple:





             """simple docstring"""






             for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                   __SCREAMING_SNAKE_CASE       :Optional[int]		    =  tokenizer(
                       ['''I am a small frog''' * 10_24, '''I am a small frog''']		,padding=SCREAMING_SNAKE_CASE__		,truncation=SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)
                   self.assertIsInstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)
                   self.assertEqual(batch.input_ids.shape		,(2, 51_22)							)




       @require_torch
       def    _UpperCamelCase					(						self							)      ->				Optional[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Dict		    =  ['''A long paragraph for summarization.''']
             __SCREAMING_SNAKE_CASE       :List[str]		    =  [
                 '''Summary of the text.''',
             ]
             for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                   __SCREAMING_SNAKE_CASE       :Dict		    =  tokenizer(SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)
                   __SCREAMING_SNAKE_CASE       :Tuple		    =  tokenizer(text_target=SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)
                   __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  inputs['''input_ids''']
                   __SCREAMING_SNAKE_CASE       :Dict		    =  targets['''input_ids''']
                   self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()							)
                   self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()							)
                   self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()							)
                   self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()							)




       @require_torch
       def    _UpperCamelCase					(						self							)      ->				List[str]:





             """simple docstring"""






             for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
                   __SCREAMING_SNAKE_CASE       :Optional[int]		    =  ['''Summary of the text.''', '''Another summary.''']
                   __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  [[0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, -1, -1]]

                   __SCREAMING_SNAKE_CASE       :Tuple		    =  tokenizer(SCREAMING_SNAKE_CASE__		,padding=SCREAMING_SNAKE_CASE__							)
                   __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  [[0] * len(SCREAMING_SNAKE_CASE__							) for x in encoded_output['''input_ids''']]
                   __SCREAMING_SNAKE_CASE       :List[str]		    =  tokenizer.pad(SCREAMING_SNAKE_CASE__							)
                   self.assertSequenceEqual(outputs['''global_attention_mask''']		,SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Dict:





             """simple docstring"""






             pass




       def    _UpperCamelCase					(						self							)      ->				Dict:





             """simple docstring"""






             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
                         __SCREAMING_SNAKE_CASE       :Optional[int]		    =  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :List[str]		    =  self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE__		,**SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :List[Any]		    =  '''A, <mask> AllenNLP sentence.'''
                         __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  tokenizer_r.encode_plus(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__		,return_token_type_ids=SCREAMING_SNAKE_CASE__							)
                         __SCREAMING_SNAKE_CASE       :Dict		    =  tokenizer_p.encode_plus(SCREAMING_SNAKE_CASE__		,add_special_tokens=SCREAMING_SNAKE_CASE__		,return_token_type_ids=SCREAMING_SNAKE_CASE__							)
                         self.assertEqual(sum(tokens_r['''token_type_ids''']							)		,sum(tokens_p['''token_type_ids''']							)							)
                         self.assertEqual(
                             sum(tokens_r['''attention_mask''']							) / len(tokens_r['''attention_mask''']							)		,sum(tokens_p['''attention_mask''']							) / len(tokens_p['''attention_mask''']							)		,)

                         __SCREAMING_SNAKE_CASE       :int		    =  tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']							)
                         __SCREAMING_SNAKE_CASE       :Dict		    =  tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']							)
                         self.assertSequenceEqual(tokens_p['''input_ids''']		,[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]							)
                         self.assertSequenceEqual(tokens_r['''input_ids''']		,[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]							)

                         self.assertSequenceEqual(
                             SCREAMING_SNAKE_CASE__		,['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']							)
                         self.assertSequenceEqual(
                             SCREAMING_SNAKE_CASE__		,['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']							)






"""simple docstring"""
import unittest

from transformers import AutoTokenizer, NystromformerConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      NystromformerForMaskedLM,
      NystromformerForMultipleChoice,
      NystromformerForQuestionAnswering,
      NystromformerForSequenceClassification,
      NystromformerForTokenClassification,
      NystromformerModel,
  )
  from transformers.models.nystromformer.modeling_nystromformer import NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST






class  _SCREAMING_SNAKE_CASE:




       def __init__(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__=13		,SCREAMING_SNAKE_CASE__=7		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=99		,SCREAMING_SNAKE_CASE__=32		,SCREAMING_SNAKE_CASE__=5		,SCREAMING_SNAKE_CASE__=4		,SCREAMING_SNAKE_CASE__=37		,SCREAMING_SNAKE_CASE__="gelu"		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=5_12		,SCREAMING_SNAKE_CASE__=16		,SCREAMING_SNAKE_CASE__=2		,SCREAMING_SNAKE_CASE__=0.0_2		,SCREAMING_SNAKE_CASE__=3		,SCREAMING_SNAKE_CASE__=4		,SCREAMING_SNAKE_CASE__=None		,)      ->				Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Any		    =  parent
             __SCREAMING_SNAKE_CASE       :Tuple		    =  batch_size
             __SCREAMING_SNAKE_CASE       :Tuple		    =  seq_length
             __SCREAMING_SNAKE_CASE       :Any		    =  is_training
             __SCREAMING_SNAKE_CASE       :Tuple		    =  use_input_mask
             __SCREAMING_SNAKE_CASE       :List[Any]		    =  use_token_type_ids
             __SCREAMING_SNAKE_CASE       :int		    =  use_labels
             __SCREAMING_SNAKE_CASE       :Dict		    =  vocab_size
             __SCREAMING_SNAKE_CASE       :int		    =  hidden_size
             __SCREAMING_SNAKE_CASE       :int		    =  num_hidden_layers
             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  num_attention_heads
             __SCREAMING_SNAKE_CASE       :Any		    =  intermediate_size
             __SCREAMING_SNAKE_CASE       :Any		    =  hidden_act
             __SCREAMING_SNAKE_CASE       :str		    =  hidden_dropout_prob
             __SCREAMING_SNAKE_CASE       :List[str]		    =  attention_probs_dropout_prob
             __SCREAMING_SNAKE_CASE       :int		    =  max_position_embeddings
             __SCREAMING_SNAKE_CASE       :Any		    =  type_vocab_size
             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  type_sequence_label_size
             __SCREAMING_SNAKE_CASE       :Optional[int]		    =  initializer_range
             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  num_labels
             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  num_choices
             __SCREAMING_SNAKE_CASE       :str		    =  scope




       def    _UpperCamelCase					(						self							)      ->				Optional[int]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  ids_tensor([self.batch_size, self.seq_length]		,self.vocab_size							)

             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  None
             if self.use_input_mask:
                   __SCREAMING_SNAKE_CASE       :List[str]		    =  random_attention_mask([self.batch_size, self.seq_length]							)

             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  None
             if self.use_token_type_ids:
                   __SCREAMING_SNAKE_CASE       :Tuple		    =  ids_tensor([self.batch_size, self.seq_length]		,self.type_vocab_size							)

             __SCREAMING_SNAKE_CASE       :Dict		    =  None
             __SCREAMING_SNAKE_CASE       :Dict		    =  None
             __SCREAMING_SNAKE_CASE       :Dict		    =  None
             if self.use_labels:
                   __SCREAMING_SNAKE_CASE       :List[Any]		    =  ids_tensor([self.batch_size]		,self.type_sequence_label_size							)
                   __SCREAMING_SNAKE_CASE       :Optional[int]		    =  ids_tensor([self.batch_size, self.seq_length]		,self.num_labels							)
                   __SCREAMING_SNAKE_CASE       :Dict		    =  ids_tensor([self.batch_size]		,self.num_choices							)

             __SCREAMING_SNAKE_CASE       :Tuple		    =  self.get_config()

             return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




       def    _UpperCamelCase					(						self							)      ->				Union[str, Any]:





             """simple docstring"""






             return NystromformerConfig(
                 vocab_size=self.vocab_size		,hidden_size=self.hidden_size		,num_hidden_layers=self.num_hidden_layers		,num_attention_heads=self.num_attention_heads		,intermediate_size=self.intermediate_size		,hidden_act=self.hidden_act		,hidden_dropout_prob=self.hidden_dropout_prob		,attention_probs_dropout_prob=self.attention_probs_dropout_prob		,max_position_embeddings=self.max_position_embeddings		,type_vocab_size=self.type_vocab_size		,is_decoder=SCREAMING_SNAKE_CASE__		,initializer_range=self.initializer_range		,)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)      ->				Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[Any]		    =  NystromformerModel(config=SCREAMING_SNAKE_CASE__							)
             model.to(SCREAMING_SNAKE_CASE__							)
             model.eval()
             __SCREAMING_SNAKE_CASE       :Any		    =  model(SCREAMING_SNAKE_CASE__		,attention_mask=SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE       :List[str]		    =  model(SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__							)
             __SCREAMING_SNAKE_CASE       :int		    =  model(SCREAMING_SNAKE_CASE__							)
             self.parent.assertEqual(result.last_hidden_state.shape		,(self.batch_size, self.seq_length, self.hidden_size)							)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)      ->				int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Dict		    =  NystromformerForMaskedLM(config=SCREAMING_SNAKE_CASE__							)
             model.to(SCREAMING_SNAKE_CASE__							)
             model.eval()
             __SCREAMING_SNAKE_CASE       :Tuple		    =  model(SCREAMING_SNAKE_CASE__		,attention_mask=SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__		,labels=SCREAMING_SNAKE_CASE__							)
             self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.seq_length, self.vocab_size)							)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)      ->				Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[str]		    =  NystromformerForQuestionAnswering(config=SCREAMING_SNAKE_CASE__							)
             model.to(SCREAMING_SNAKE_CASE__							)
             model.eval()
             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  model(
                 SCREAMING_SNAKE_CASE__		,attention_mask=SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__		,start_positions=SCREAMING_SNAKE_CASE__		,end_positions=SCREAMING_SNAKE_CASE__		,)
             self.parent.assertEqual(result.start_logits.shape		,(self.batch_size, self.seq_length)							)
             self.parent.assertEqual(result.end_logits.shape		,(self.batch_size, self.seq_length)							)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)      ->				Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :int		    =  self.num_labels
             __SCREAMING_SNAKE_CASE       :Any		    =  NystromformerForSequenceClassification(SCREAMING_SNAKE_CASE__							)
             model.to(SCREAMING_SNAKE_CASE__							)
             model.eval()
             __SCREAMING_SNAKE_CASE       :Dict		    =  model(SCREAMING_SNAKE_CASE__		,attention_mask=SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__		,labels=SCREAMING_SNAKE_CASE__							)
             self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.num_labels)							)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)      ->				Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  self.num_labels
             __SCREAMING_SNAKE_CASE       :Tuple		    =  NystromformerForTokenClassification(config=SCREAMING_SNAKE_CASE__							)
             model.to(SCREAMING_SNAKE_CASE__							)
             model.eval()
             __SCREAMING_SNAKE_CASE       :Any		    =  model(SCREAMING_SNAKE_CASE__		,attention_mask=SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__		,labels=SCREAMING_SNAKE_CASE__							)
             self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.seq_length, self.num_labels)							)




       def    _UpperCamelCase					(						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)      ->				Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  self.num_choices
             __SCREAMING_SNAKE_CASE       :Dict		    =  NystromformerForMultipleChoice(config=SCREAMING_SNAKE_CASE__							)
             model.to(SCREAMING_SNAKE_CASE__							)
             model.eval()
             __SCREAMING_SNAKE_CASE       :List[str]		    =  input_ids.unsqueeze(1							).expand(-1		,self.num_choices		,-1							).contiguous()
             __SCREAMING_SNAKE_CASE       :Optional[int]		    =  token_type_ids.unsqueeze(1							).expand(-1		,self.num_choices		,-1							).contiguous()
             __SCREAMING_SNAKE_CASE       :List[Any]		    =  input_mask.unsqueeze(1							).expand(-1		,self.num_choices		,-1							).contiguous()
             __SCREAMING_SNAKE_CASE       :Dict		    =  model(
                 SCREAMING_SNAKE_CASE__		,attention_mask=SCREAMING_SNAKE_CASE__		,token_type_ids=SCREAMING_SNAKE_CASE__		,labels=SCREAMING_SNAKE_CASE__		,)
             self.parent.assertEqual(result.logits.shape		,(self.batch_size, self.num_choices)							)




       def    _UpperCamelCase					(						self							)      ->				Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[Any]		    =  self.prepare_config_and_inputs()
             (
                 (
                 __SCREAMING_SNAKE_CASE
             )    ,(
                 __SCREAMING_SNAKE_CASE
             )    ,(
                 __SCREAMING_SNAKE_CASE
             )    ,(
                 __SCREAMING_SNAKE_CASE
             )    ,(
                 __SCREAMING_SNAKE_CASE
             )    ,(
                 __SCREAMING_SNAKE_CASE
             )    ,(
                 __SCREAMING_SNAKE_CASE
             )    ,
             )       :Dict		    =  config_and_inputs
             __SCREAMING_SNAKE_CASE       :str		    =  {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
             return config, inputs_dict






@require_torch
class  _SCREAMING_SNAKE_CASE(       A   ,     A   ,     unittest.TestCase  ):
       SCREAMING_SNAKE_CASE_       :				int     						=  (
           (
               NystromformerModel,
               NystromformerForMaskedLM,
               NystromformerForMultipleChoice,
               NystromformerForQuestionAnswering,
               NystromformerForSequenceClassification,
               NystromformerForTokenClassification,
           )
           if is_torch_available()
           else ()
       )
       SCREAMING_SNAKE_CASE_       :				Union[str, Any]     						=  (
           {
               '''feature-extraction''': NystromformerModel,
               '''fill-mask''': NystromformerForMaskedLM,
               '''question-answering''': NystromformerForQuestionAnswering,
               '''text-classification''': NystromformerForSequenceClassification,
               '''token-classification''': NystromformerForTokenClassification,
               '''zero-shot''': NystromformerForSequenceClassification,
           }
           if is_torch_available()
           else {}
       )
       SCREAMING_SNAKE_CASE_       :				str     						=  False
       SCREAMING_SNAKE_CASE_       :				Tuple     						=  False




       def    _UpperCamelCase					(						self							)      ->				Tuple:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Any		    =  NystromformerModelTester(self							)
             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  ConfigTester(self		,config_class=SCREAMING_SNAKE_CASE__		,hidden_size=37							)




       def    _UpperCamelCase					(						self							)      ->				List[str]:





             """simple docstring"""






             self.config_tester.run_common_tests()




       def    _UpperCamelCase					(						self							)      ->				List[str]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[str]		    =  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Dict:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Any		    =  self.model_tester.prepare_config_and_inputs()
             for type in ["absolute", "relative_key", "relative_key_query"]:
                   __SCREAMING_SNAKE_CASE       :Any		    =  type
                   self.model_tester.create_and_check_model(*SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :List[str]		    =  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_masked_lm(*SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				int:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_multiple_choice(*SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Any		    =  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_question_answering(*SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Union[str, Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Dict		    =  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_sequence_classification(*SCREAMING_SNAKE_CASE__							)




       def    _UpperCamelCase					(						self							)      ->				Optional[int]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  self.model_tester.prepare_config_and_inputs()
             self.model_tester.create_and_check_for_token_classification(*SCREAMING_SNAKE_CASE__							)




       @slow
       def    _UpperCamelCase					(						self							)      ->				Union[str, Any]:





             """simple docstring"""






             for model_name in NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                   __SCREAMING_SNAKE_CASE       :Tuple		    =  NystromformerModel.from_pretrained(SCREAMING_SNAKE_CASE__							)
                   self.assertIsNotNone(SCREAMING_SNAKE_CASE__							)






@require_torch
class  _SCREAMING_SNAKE_CASE(       unittest.TestCase  ):




       @slow
       def    _UpperCamelCase					(						self							)      ->				List[Any]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :int		    =  NystromformerModel.from_pretrained('''uw-madison/nystromformer-512'''							)
             __SCREAMING_SNAKE_CASE       :Dict		    =  torch.tensor([[0, 1, 2, 3, 4, 5]]							)

             with torch.no_grad():
                   __SCREAMING_SNAKE_CASE       :str		    =  model(SCREAMING_SNAKE_CASE__							)[0]

             __SCREAMING_SNAKE_CASE       :Optional[int]		    =  torch.Size((1, 6, 7_68)							)
             self.assertEqual(output.shape		,SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE       :int		    =  torch.tensor(
                 [[[-0.4_5_3_2, -0.0_9_3_6, 0.5_1_3_7], [-0.2_6_7_6, 0.0_6_2_8, 0.6_1_8_6], [-0.3_6_2_9, -0.1_7_2_6, 0.4_7_1_6]]]							)

             self.assertTrue(torch.allclose(output[:, :3, :3]		,SCREAMING_SNAKE_CASE__		,atol=1E-4							)							)




       @slow
       def    _UpperCamelCase					(						self							)      ->				Optional[int]:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  '''the [MASK] of Belgium is Brussels'''

             __SCREAMING_SNAKE_CASE       :Optional[Any]		    =  AutoTokenizer.from_pretrained('''uw-madison/nystromformer-512'''							)
             __SCREAMING_SNAKE_CASE       :str		    =  NystromformerForMaskedLM.from_pretrained('''uw-madison/nystromformer-512'''							)

             __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  tokenizer(SCREAMING_SNAKE_CASE__		,return_tensors='''pt'''							)

             with torch.no_grad():
                   __SCREAMING_SNAKE_CASE       :Union[str, Any]		    =  model(encoding.input_ids							).logits

             __SCREAMING_SNAKE_CASE       :List[str]		    =  token_logits[:, 2, :].argmax(-1							)[0]

             self.assertEqual(tokenizer.decode(SCREAMING_SNAKE_CASE__							)		,'''capital'''							)