



'''simple docstring'''





from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
     TFTransfoXLForSequenceClassification,
     TFTransfoXLLMHeadModel,
     TFTransfoXLModel,
 )






class 	lowerCAmelCase_    :
  '''simple docstring'''




  def __init__(   self  ,__UpperCAmelCase  ,)   ->		Optional[int]:
      lowerCAmelCase__			:  Tuple											= parent
      lowerCAmelCase__			:  Dict											= 13
      lowerCAmelCase__			:  Union[str, Any]											= 7
      lowerCAmelCase__			:  Optional[int]											= 30
      lowerCAmelCase__			:  List[Any]											= self.seq_length + self.mem_len
      lowerCAmelCase__			:  str											= 15
      lowerCAmelCase__			:  str											= True
      lowerCAmelCase__			:  Any											= True
      lowerCAmelCase__			:  Optional[Any]											= 99
      lowerCAmelCase__			:  List[Any]											= [10, 50, 80]
      lowerCAmelCase__			:  Optional[int]											= 32
      lowerCAmelCase__			:  Dict											= 32
      lowerCAmelCase__			:  Dict											= 4
      lowerCAmelCase__			:  Tuple											= 8
      lowerCAmelCase__			:  Union[str, Any]											= 128
      lowerCAmelCase__			:  Union[str, Any]											= 2
      lowerCAmelCase__			:  int											= 2
      lowerCAmelCase__			:  Union[str, Any]											= None
      lowerCAmelCase__			:  List[Any]											= 1
      lowerCAmelCase__			:  Any											= 0
      lowerCAmelCase__			:  Dict											= 3
      lowerCAmelCase__			:  Union[str, Any]											= self.vocab_size - 1
      lowerCAmelCase__			:  Optional[Any]											= 0.0_1




  def 							UpperCAmelCase_     (   self				)   ->		Union[str, Any]:
      lowerCAmelCase__			:  int											= ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size				)
      lowerCAmelCase__			:  Optional[int]											= ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size				)

      lowerCAmelCase__			:  List[Any]											= None
      if self.use_labels:
          lowerCAmelCase__			:  List[str]											= ids_tensor([self.batch_size, self.seq_length]  ,self.vocab_size				)

      lowerCAmelCase__			:  List[Any]											= TransfoXLConfig(
          vocab_size=self.vocab_size  ,mem_len=self.mem_len  ,clamp_len=self.clamp_len  ,cutoffs=self.cutoffs  ,d_model=self.hidden_size  ,d_embed=self.d_embed  ,n_head=self.num_attention_heads  ,d_head=self.d_head  ,d_inner=self.d_inner  ,div_val=self.div_val  ,n_layer=self.num_hidden_layers  ,eos_token_id=self.eos_token_id  ,pad_token_id=self.vocab_size - 1  ,init_range=self.init_range  ,num_labels=self.num_labels  ,)

      return (config, input_ids_a, input_ids_a, lm_labels)




  def 							UpperCAmelCase_     (   self				)   ->		Union[str, Any]:
      random.seed(self.seed				)
      tf.random.set_seed(self.seed				)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		List[str]:
      lowerCAmelCase__			:  Any											= TFTransfoXLModel(__UpperCAmelCase				)

      lowerCAmelCase__			:  Tuple											= model(__UpperCAmelCase				).to_tuple()

      lowerCAmelCase__			:  List[str]											= {"""input_ids""": input_ids_a, """mems""": mems_a}

      lowerCAmelCase__			:  Dict											= model(__UpperCAmelCase				).to_tuple()

      self.parent.assertEqual(hidden_states_a.shape  ,(self.batch_size, self.seq_length, self.hidden_size)				)
      self.parent.assertEqual(hidden_states_a.shape  ,(self.batch_size, self.seq_length, self.hidden_size)				)
      self.parent.assertListEqual(
          [mem.shape for mem in mems_a]  ,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  ,)
      self.parent.assertListEqual(
          [mem.shape for mem in mems_a]  ,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  ,)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Any:
      lowerCAmelCase__			:  Union[str, Any]											= TFTransfoXLLMHeadModel(__UpperCAmelCase				)

      lowerCAmelCase__			:  List[str]											= model(__UpperCAmelCase				).to_tuple()

      lowerCAmelCase__			:  Optional[int]											= {"""input_ids""": input_ids_a, """labels""": lm_labels}
      lowerCAmelCase__			:  Union[str, Any]											= model(__UpperCAmelCase				).to_tuple()

      lowerCAmelCase__			:  str											= model([input_ids_a, mems_a]				).to_tuple()

      lowerCAmelCase__			:  List[str]											= {"""input_ids""": input_ids_a, """mems""": mems_a, """labels""": lm_labels}

      lowerCAmelCase__			:  Any											= model(__UpperCAmelCase				).to_tuple()

      self.parent.assertEqual(lm_logits_a.shape  ,(self.batch_size, self.seq_length, self.vocab_size)				)
      self.parent.assertListEqual(
          [mem.shape for mem in mems_a]  ,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  ,)

      self.parent.assertEqual(lm_logits_a.shape  ,(self.batch_size, self.seq_length, self.vocab_size)				)
      self.parent.assertListEqual(
          [mem.shape for mem in mems_a]  ,[(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers  ,)




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		Tuple:
      lowerCAmelCase__			:  Any											= TFTransfoXLForSequenceClassification(__UpperCAmelCase				)
      lowerCAmelCase__			:  List[str]											= model(__UpperCAmelCase				)
      self.parent.assertEqual(result.logits.shape  ,(self.batch_size, self.num_labels)				)






  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      lowerCAmelCase__			:  int											= self.prepare_config_and_inputs()
      (lowerCAmelCase__)			:  Dict											= config_and_inputs
      lowerCAmelCase__			:  Dict											= {"""input_ids""": input_ids_a}
      return config, inputs_dict







@require_tf
class 	lowerCAmelCase_    (    SCREAMING_SNAKE_CASE_						,			SCREAMING_SNAKE_CASE_						,			unittest.TestCase					):
  '''simple docstring'''




  __lowercase							:				List[str]           =				(
      (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
  )
  __lowercase							:				List[Any]           =				() if is_tf_available() else ()
  __lowercase							:				Optional[Any]           =				(
      {
          '''feature-extraction''': TFTransfoXLModel,
          '''text-classification''': TFTransfoXLForSequenceClassification,
          '''text-generation''': TFTransfoXLLMHeadModel,
          '''zero-shot''': TFTransfoXLForSequenceClassification,
      }
      if is_tf_available()
      else {}
  )
  # TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
  __lowercase							:				Optional[Any]           =				False
  __lowercase							:				List[str]           =				False
  __lowercase							:				Union[str, Any]           =				False
  __lowercase							:				int           =				False




  def 							UpperCAmelCase_     (   self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase				)   ->		List[str]:
      if pipeline_test_casse_name == "TextGenerationPipelineTests":
          # Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
          # `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
          # tokenizer.
          return True

      return False




  def 							UpperCAmelCase_     (   self				)   ->		str:
      lowerCAmelCase__			:  int											= TFTransfoXLModelTester(self				)
      lowerCAmelCase__			:  Optional[int]											= ConfigTester(self  ,config_class=__UpperCAmelCase  ,d_embed=37				)




  def 							UpperCAmelCase_     (   self				)   ->		Union[str, Any]:
      self.config_tester.run_common_tests()




  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      self.model_tester.set_seed()
      lowerCAmelCase__			:  Optional[Any]											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_transfo_xl_model(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		int:
      self.model_tester.set_seed()
      lowerCAmelCase__			:  int											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_transfo_xl_lm_head(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		List[Any]:
      lowerCAmelCase__			:  Tuple											= self.model_tester.prepare_config_and_inputs()
      self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*__UpperCAmelCase				)




  def 							UpperCAmelCase_     (   self				)   ->		List[Any]:
      lowerCAmelCase__			:  List[str]											= self.model_tester.prepare_config_and_inputs_for_common()
      lowerCAmelCase__			:  Dict											= [TFTransfoXLForSequenceClassification]

      for model_class in self.all_model_classes:
          lowerCAmelCase__			:  str											= model_class(__UpperCAmelCase				)
          assert isinstance(model.get_input_embeddings()  ,tf.keras.layers.Layer				)
          if model_class in list_other_models_with_output_ebd:
              lowerCAmelCase__			:  Any											= model.get_output_embeddings()
              assert isinstance(__UpperCAmelCase  ,tf.keras.layers.Layer				)
              lowerCAmelCase__			:  int											= model.get_bias()
              assert name is None
          else:
              lowerCAmelCase__			:  List[Any]											= model.get_output_embeddings()
              assert x is None
              lowerCAmelCase__			:  Any											= model.get_bias()
              assert name is None




  def 							UpperCAmelCase_     (   self				)   ->		Optional[Any]:
      # TODO JP: Make TransfoXL XLA compliant
      pass




  @slow
  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
          lowerCAmelCase__			:  Optional[int]											= TFTransfoXLModel.from_pretrained(__UpperCAmelCase				)
          self.assertIsNotNone(__UpperCAmelCase				)






  @unittest.skip(reason="""This model doesn't play well with fit() due to not returning a single loss."""				)
  def 							UpperCAmelCase_     (   self				)   ->		List[Any]:
      pass







@require_tf
class 	lowerCAmelCase_    (    unittest.TestCase					):
  '''simple docstring'''




  @unittest.skip("""Skip test until #12651 is resolved."""				)
  @slow
  def 							UpperCAmelCase_     (   self				)   ->		Tuple:
      lowerCAmelCase__			:  Union[str, Any]											= TFTransfoXLLMHeadModel.from_pretrained("""transfo-xl-wt103"""				)
      # fmt: off
      lowerCAmelCase__			:  Union[str, Any]											= tf.convert_to_tensor([[33,1297,2,1,1009,4,1109,1_1739,4762,358,5,25,245,22,1706,17,2_0098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,7_1477,2_0098,10_4447,2,2_0961,1,2604,4,1,329,3,6224,831,1_6002,2,8,603,7_8967,2_9546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,2_9546,54,8,3609,5,5_7211,49,4,1,277,18,8,1755,1_5691,3,341,25,416,693,4_2573,71,17,401,94,31,1_7919,2,2_9546,7873,18,1,435,23,1_1011,755,5,5167,3,7983,98,84,2,2_9546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,2_9546,824,1400,1868,2,19,160,2,311,8,5496,2,2_0920,17,25,1_5097,3,24,24,0]]  ,dtype=tf.intaa				)  # noqa: E231
      # fmt: on
      #  In 1991 , the remains of Russian Tsar Nicholas II and his family
      #  ( except for Alexei and Maria ) are discovered .
      #  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
      #  remainder of the story . 1883 Western Siberia ,
      #  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
      #  Rasputin has a vision and denounces one of the men as a horse thief . Although his
      #  father initially slaps him for making such an accusation , Rasputin watches as the
      #  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
      #  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
      #  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

      # fmt: off
      lowerCAmelCase__			:  Optional[Any]											= [33,1297,2,1,1009,4,1109,1_1739,4762,358,5,25,245,22,1706,17,2_0098,5,3215,21,37,1110,3,13,1041,4,24,603,490,2,7_1477,2_0098,10_4447,2,2_0961,1,2604,4,1,329,3,6224,831,1_6002,2,8,603,7_8967,2_9546,23,803,20,25,416,5,8,232,4,277,6,1855,4601,3,2_9546,54,8,3609,5,5_7211,49,4,1,277,18,8,1755,1_5691,3,341,25,416,693,4_2573,71,17,401,94,31,1_7919,2,2_9546,7873,18,1,435,23,1_1011,755,5,5167,3,7983,98,84,2,2_9546,3267,8,3609,4,1,4865,1075,2,6087,71,6,346,8,5854,3,2_9546,824,1400,1868,2,19,160,2,311,8,5496,2,2_0920,17,25,1_5097,3,24,24,0,33,1,1857,2,1,1009,4,1109,1_1739,4762,358,5,25,245,28,1110,3,13,1041,4,24,603,490,2,7_1477,2_0098,10_4447,2,2_0961,1,2604,4,1,329,3,0]  # noqa: E231
      # fmt: on
      #  In 1991, the remains of Russian Tsar Nicholas II and his family (
      #  except for Alexei and Maria ) are discovered. The voice of young son,
      #  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
      #  1883 Western Siberia, a young Grigori Rasputin is asked by his father
      #  and a group of men to perform magic. Rasputin has a vision and
      #  denounces one of the men as a horse thief. Although his father initially
      #  slaps him for making such an accusation, Rasputin watches as the man
      #  is chased outside and beaten. Twenty years later, Rasputin sees a vision
      #  of the Virgin Mary, prompting him to become a priest.
      #  Rasputin quickly becomes famous, with people, even a bishop, begging for
      #  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
      # Nicholas II and his family were discovered. The voice of <unk> young son,
      # Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

      lowerCAmelCase__			:  Dict											= model.generate(__UpperCAmelCase  ,max_length=200  ,do_sample=__UpperCAmelCase				)
      self.assertListEqual(output_ids[0].numpy().tolist()  ,__UpperCAmelCase				)







'''simple docstring'''





import json
import os
import shutil
import tempfile
from unittest import TestCase

from transformers import BartTokenizer, BartTokenizerFast, DPRQuestionEncoderTokenizer, DPRQuestionEncoderTokenizerFast
from transformers.models.bart.configuration_bart import BartConfig
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES as DPR_VOCAB_FILES_NAMES
from transformers.models.dpr.configuration_dpr import DPRConfig
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES as BART_VOCAB_FILES_NAMES
from transformers.testing_utils import require_faiss, require_tokenizers, require_torch, slow
from transformers.utils import is_datasets_available, is_faiss_available, is_torch_available


if is_torch_available() and is_datasets_available() and is_faiss_available():
 from transformers.models.rag.configuration_rag import RagConfig
 from transformers.models.rag.tokenization_rag import RagTokenizer






@require_faiss
@require_torch
class 	lowerCAmelCase_(    SCREAMING_SNAKE_CASE_					):
  '''simple docstring'''




  def 							UpperCAmelCase_     (   self				)   ->		Dict:
      lowerCAmelCase__			:  str											= tempfile.mkdtemp()
      lowerCAmelCase__			:  List[Any]											= 8

      # DPR tok
      lowerCAmelCase__			:  int											= [
          """[UNK]""",
          """[CLS]""",
          """[SEP]""",
          """[PAD]""",
          """[MASK]""",
          """want""",
          """##want""",
          """##ed""",
          """wa""",
          """un""",
          """runn""",
          """##ing""",
          """,""",
          """low""",
          """lowest""",
      ]
      lowerCAmelCase__			:  List[Any]											= os.path.join(self.tmpdirname  ,"""dpr_tokenizer"""				)
      os.makedirs(__UpperCAmelCase  ,exist_ok=__UpperCAmelCase				)
      lowerCAmelCase__			:  Dict											= os.path.join(__UpperCAmelCase  ,DPR_VOCAB_FILES_NAMES["""vocab_file"""]				)
      with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""				) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]				)				)

      # BART tok
      lowerCAmelCase__			:  str											= [
          """l""",
          """o""",
          """w""",
          """e""",
          """r""",
          """s""",
          """t""",
          """i""",
          """d""",
          """n""",
          """\u0120""",
          """\u0120l""",
          """\u0120n""",
          """\u0120lo""",
          """\u0120low""",
          """er""",
          """\u0120lowest""",
          """\u0120newer""",
          """\u0120wider""",
          """<unk>""",
      ]
      lowerCAmelCase__			:  List[Any]											= dict(zip(__UpperCAmelCase  ,range(len(__UpperCAmelCase				)				)				)				)
      lowerCAmelCase__			:  Optional[Any]											= ["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
      lowerCAmelCase__			:  Any											= {"""unk_token""": """<unk>"""}

      lowerCAmelCase__			:  str											= os.path.join(self.tmpdirname  ,"""bart_tokenizer"""				)
      os.makedirs(__UpperCAmelCase  ,exist_ok=__UpperCAmelCase				)
      lowerCAmelCase__			:  Any											= os.path.join(__UpperCAmelCase  ,BART_VOCAB_FILES_NAMES["""vocab_file"""]				)
      lowerCAmelCase__			:  Dict											= os.path.join(__UpperCAmelCase  ,BART_VOCAB_FILES_NAMES["""merges_file"""]				)
      with open(self.vocab_file  ,"""w"""  ,encoding="""utf-8"""				) as fp:
          fp.write(json.dumps(__UpperCAmelCase				) + """\n"""				)
      with open(self.merges_file  ,"""w"""  ,encoding="""utf-8"""				) as fp:
          fp.write("""\n""".join(__UpperCAmelCase				)				)




  def 							UpperCAmelCase_     (   self				)   ->		DPRQuestionEncoderTokenizer:
      return DPRQuestionEncoderTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,"""dpr_tokenizer"""				)				)




  def 							UpperCAmelCase_     (   self				)   ->		BartTokenizer:
      return BartTokenizer.from_pretrained(os.path.join(self.tmpdirname  ,"""bart_tokenizer"""				)				)




  def 							UpperCAmelCase_     (   self				)   ->		Any:
      shutil.rmtree(self.tmpdirname				)




  @require_tokenizers
  def 							UpperCAmelCase_     (   self				)   ->		int:
      lowerCAmelCase__			:  Any											= os.path.join(self.tmpdirname  ,"""rag_tokenizer"""				)
      lowerCAmelCase__			:  Any											= RagConfig(question_encoder=DPRConfig().to_dict()  ,generator=BartConfig().to_dict()				)
      lowerCAmelCase__			:  str											= RagTokenizer(question_encoder=self.get_dpr_tokenizer()  ,generator=self.get_bart_tokenizer()				)
      rag_config.save_pretrained(__UpperCAmelCase				)
      rag_tokenizer.save_pretrained(__UpperCAmelCase				)
      lowerCAmelCase__			:  Any											= RagTokenizer.from_pretrained(__UpperCAmelCase  ,config=__UpperCAmelCase				)
      self.assertIsInstance(new_rag_tokenizer.question_encoder  ,__UpperCAmelCase				)
      self.assertEqual(new_rag_tokenizer.question_encoder.get_vocab()  ,rag_tokenizer.question_encoder.get_vocab()				)
      self.assertIsInstance(new_rag_tokenizer.generator  ,__UpperCAmelCase				)
      self.assertEqual(new_rag_tokenizer.generator.get_vocab()  ,rag_tokenizer.generator.get_vocab()				)




  @slow
  def 							UpperCAmelCase_     (   self				)   ->		Union[str, Any]:
      lowerCAmelCase__			:  List[str]											= RagTokenizer.from_pretrained("""facebook/rag-token-nq"""				)
      lowerCAmelCase__			:  Optional[Any]											= [
          """who got the first nobel prize in physics""",
          """when is the next deadpool movie being released""",
          """which mode is used for short wave broadcast service""",
          """who is the owner of reading football club""",
          """when is the next scandal episode coming out""",
          """when is the last time the philadelphia won the superbowl""",
          """what is the most current adobe flash player version""",
          """how many episodes are there in dragon ball z""",
          """what is the first step in the evolution of the eye""",
          """where is gall bladder situated in human body""",
          """what is the main mineral in lithium batteries""",
          """who is the president of usa right now""",
          """where do the greasers live in the outsiders""",
          """panda is a national animal of which country""",
          """what is the name of manchester united stadium""",
      ]
      lowerCAmelCase__			:  Dict											= tokenizer(__UpperCAmelCase				)
      self.assertIsNotNone(__UpperCAmelCase				)






  @slow
  def 							UpperCAmelCase_     (   self				)   ->		Union[str, Any]:
      lowerCAmelCase__			:  str											= RagTokenizer.from_pretrained("""facebook/rag-sequence-nq"""				)
      lowerCAmelCase__			:  str											= [
          """who got the first nobel prize in physics""",
          """when is the next deadpool movie being released""",
          """which mode is used for short wave broadcast service""",
          """who is the owner of reading football club""",
          """when is the next scandal episode coming out""",
          """when is the last time the philadelphia won the superbowl""",
          """what is the most current adobe flash player version""",
          """how many episodes are there in dragon ball z""",
          """what is the first step in the evolution of the eye""",
          """where is gall bladder situated in human body""",
          """what is the main mineral in lithium batteries""",
          """who is the president of usa right now""",
          """where do the greasers live in the outsiders""",
          """panda is a national animal of which country""",
          """what is the name of manchester united stadium""",
      ]
      lowerCAmelCase__			:  Tuple											= tokenizer(__UpperCAmelCase				)
      self.assertIsNotNone(__UpperCAmelCase				)


