




import unittest

from transformers import AutoTokenizer, FalconConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
     import torch

     from transformers import (
         FalconForCausalLM,
         FalconForQuestionAnswering,
         FalconForSequenceClassification,
         FalconForTokenClassification,
         FalconModel,
     )

class   __a:






    """simple docstring"""





    def __init__(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE=3       ,_SCREAMING_SNAKE_CASE=7       ,_SCREAMING_SNAKE_CASE=True       ,_SCREAMING_SNAKE_CASE=True       ,_SCREAMING_SNAKE_CASE=False       ,_SCREAMING_SNAKE_CASE=True       ,_SCREAMING_SNAKE_CASE=99       ,_SCREAMING_SNAKE_CASE=32       ,_SCREAMING_SNAKE_CASE=5       ,_SCREAMING_SNAKE_CASE=4       ,_SCREAMING_SNAKE_CASE=37       ,_SCREAMING_SNAKE_CASE="gelu"       ,_SCREAMING_SNAKE_CASE=0.1       ,_SCREAMING_SNAKE_CASE=0.1       ,_SCREAMING_SNAKE_CASE=512       ,_SCREAMING_SNAKE_CASE=16       ,_SCREAMING_SNAKE_CASE=2       ,_SCREAMING_SNAKE_CASE=0.02       ,_SCREAMING_SNAKE_CASE=3       ,_SCREAMING_SNAKE_CASE=4       ,_SCREAMING_SNAKE_CASE=None       ,)			->		Dict:
         UpperCAmelCase_      :							str       =      parent
         UpperCAmelCase_      :							List[str]       =      batch_size
         UpperCAmelCase_      :							Union[str, Any]       =      seq_length
         UpperCAmelCase_      :							Optional[int]       =      is_training
         UpperCAmelCase_      :							str       =      use_input_mask
         UpperCAmelCase_      :							Union[str, Any]       =      use_token_type_ids
         UpperCAmelCase_      :							Tuple       =      use_labels
         UpperCAmelCase_      :							Union[str, Any]       =      vocab_size
         UpperCAmelCase_      :							List[str]       =      hidden_size
         UpperCAmelCase_      :							Optional[Any]       =      num_hidden_layers
         UpperCAmelCase_      :							Dict       =      num_attention_heads
         UpperCAmelCase_      :							int       =      intermediate_size
         UpperCAmelCase_      :							Any       =      hidden_act
         UpperCAmelCase_      :							Tuple       =      hidden_dropout_prob
         UpperCAmelCase_      :							Union[str, Any]       =      attention_probs_dropout_prob
         UpperCAmelCase_      :							str       =      max_position_embeddings
         UpperCAmelCase_      :							Optional[Any]       =      type_vocab_size
         UpperCAmelCase_      :							Any       =      type_sequence_label_size
         UpperCAmelCase_      :							Dict       =      initializer_range
         UpperCAmelCase_      :							Optional[Any]       =      num_labels
         UpperCAmelCase_      :							int       =      num_choices
         UpperCAmelCase_      :							str       =      scope





    def 							a__				(				self  )			->		List[Any]:
         UpperCAmelCase_      :							str       =      ids_tensor([self.batch_size, self.seq_length]       ,self.vocab_size  )

         UpperCAmelCase_      :							List[str]       =      None
         if self.use_input_mask:
              UpperCAmelCase_      :							List[Any]       =      random_attention_mask([self.batch_size, self.seq_length]  )

         UpperCAmelCase_      :							List[Any]       =      None

         UpperCAmelCase_      :							Optional[Any]       =      None
         UpperCAmelCase_      :							int       =      None
         UpperCAmelCase_      :							Any       =      None
         if self.use_labels:
              UpperCAmelCase_      :							List[str]       =      ids_tensor([self.batch_size]       ,self.type_sequence_label_size  )
              UpperCAmelCase_      :							List[str]       =      ids_tensor([self.batch_size, self.seq_length]       ,self.num_labels  )
              UpperCAmelCase_      :							int       =      ids_tensor([self.batch_size]       ,self.num_choices  )

         UpperCAmelCase_      :							Optional[int]       =      self.get_config()

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





    def 							a__				(				self  )			->		Tuple:
         return FalconConfig(
             vocab_size=self.vocab_size       ,hidden_size=self.hidden_size       ,num_hidden_layers=self.num_hidden_layers       ,num_attention_heads=self.num_attention_heads       ,intermediate_size=self.intermediate_size       ,hidden_act=self.hidden_act       ,hidden_dropout_prob=self.hidden_dropout_prob       ,attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,max_position_embeddings=self.max_position_embeddings       ,type_vocab_size=self.type_vocab_size       ,is_decoder=_SCREAMING_SNAKE_CASE       ,initializer_range=self.initializer_range       ,pad_token_id=1       ,new_decoder_architecture=_SCREAMING_SNAKE_CASE       ,)





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )			->		int:
         UpperCAmelCase_      :							Any       =      FalconModel(config=_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Any       =      model(_SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							Any       =      model(_SCREAMING_SNAKE_CASE  )
         self.parent.assertEqual(result.last_hidden_state.shape       ,(self.batch_size, self.seq_length, self.hidden_size)  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,)			->		Dict:
         UpperCAmelCase_      :							List[Any]       =      True
         UpperCAmelCase_      :							Any       =      FalconModel(_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Optional[int]       =      model(
             _SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,encoder_hidden_states=_SCREAMING_SNAKE_CASE       ,encoder_attention_mask=_SCREAMING_SNAKE_CASE       ,)
         UpperCAmelCase_      :							Optional[int]       =      model(
             _SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,encoder_hidden_states=_SCREAMING_SNAKE_CASE       ,)
         UpperCAmelCase_      :							Optional[Any]       =      model(_SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE  )
         self.parent.assertEqual(result.last_hidden_state.shape       ,(self.batch_size, self.seq_length, self.hidden_size)  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,)			->		Union[str, Any]:
         UpperCAmelCase_      :							List[str]       =      FalconForCausalLM(config=_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Tuple       =      model(_SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,labels=_SCREAMING_SNAKE_CASE  )
         self.parent.assertEqual(result.logits.shape       ,(self.batch_size, self.seq_length, self.vocab_size)  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,)			->		Tuple:
         UpperCAmelCase_      :							List[Any]       =      True
         UpperCAmelCase_      :							Optional[int]       =      True
         UpperCAmelCase_      :							int       =      FalconForCausalLM(config=_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()

         # first forward pass
         UpperCAmelCase_      :							Optional[int]       =      model(
             _SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,encoder_hidden_states=_SCREAMING_SNAKE_CASE       ,encoder_attention_mask=_SCREAMING_SNAKE_CASE       ,use_cache=_SCREAMING_SNAKE_CASE       ,)
         UpperCAmelCase_      :							List[Any]       =      outputs.past_key_values

         # create hypothetical multiple next token and extent to next_input_ids
         UpperCAmelCase_      :							List[str]       =      ids_tensor((self.batch_size, 3)       ,config.vocab_size  )
         UpperCAmelCase_      :							Dict       =      ids_tensor((self.batch_size, 3)       ,vocab_size=2  )

         # append to next input_ids and
         UpperCAmelCase_      :							int       =      torch.cat([input_ids, next_tokens]       ,dim=-1  )
         UpperCAmelCase_      :							Optional[int]       =      torch.cat([input_mask, next_mask]       ,dim=-1  )

         UpperCAmelCase_      :							Dict       =      model(
             _SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,encoder_hidden_states=_SCREAMING_SNAKE_CASE       ,encoder_attention_mask=_SCREAMING_SNAKE_CASE       ,output_hidden_states=_SCREAMING_SNAKE_CASE       ,)['''hidden_states'''][0]
         UpperCAmelCase_      :							int       =      model(
             _SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,encoder_hidden_states=_SCREAMING_SNAKE_CASE       ,encoder_attention_mask=_SCREAMING_SNAKE_CASE       ,past_key_values=_SCREAMING_SNAKE_CASE       ,output_hidden_states=_SCREAMING_SNAKE_CASE       ,)['''hidden_states'''][0]

         # select random slice
         UpperCAmelCase_      :							List[str]       =      ids_tensor((1,)       ,output_from_past.shape[-1]  ).item()
         UpperCAmelCase_      :							Union[str, Any]       =      output_from_no_past[:, -3:, random_slice_idx].detach()
         UpperCAmelCase_      :							str       =      output_from_past[:, :, random_slice_idx].detach()

         self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

         # test that outputs are equal for slice
         self.parent.assertTrue(torch.allclose(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,atol=1e-3  )  )





    def 							a__				(				self  )			->		Any:
         UpperCAmelCase_      :							List[str]       =      self.prepare_config_and_inputs()
         (
             (
             UpperCAmelCase_
         ),   (
             UpperCAmelCase_
         ),   (
             UpperCAmelCase_
         ),   (
             UpperCAmelCase_
         ),   (
             UpperCAmelCase_
         ),   (
             UpperCAmelCase_
         ),   (
             UpperCAmelCase_
         ),   
         )      :							List[Any]       =      config_and_inputs
         UpperCAmelCase_      :							str       =      {'''input_ids''': input_ids, '''attention_mask''': input_mask}
         return config, inputs_dict


@require_torch
class   __a(					_a    ,						_a    ,						_a    ,						unittest.TestCase       ):






    """simple docstring"""





    lowerCAmelCase        =				(
        (
            FalconModel,
            FalconForCausalLM,
            FalconForSequenceClassification,
            FalconForTokenClassification,
            FalconForQuestionAnswering,
        )
        if is_torch_available()
        else ()
    )
    lowerCAmelCase        =				(FalconForCausalLM,) if is_torch_available() else ()
    lowerCAmelCase        =				(
        {
            '''feature-extraction''': FalconModel,
            '''text-classification''': FalconForSequenceClassification,
            '''text-generation''': FalconForCausalLM,
            '''question-answering''': FalconForQuestionAnswering,
            '''token-classification''': FalconForTokenClassification,
            '''zero-shot''': FalconForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowerCAmelCase        =				False
    lowerCAmelCase        =				False





    def 							a__				(				self  )			->		Dict:
         UpperCAmelCase_      :							List[Any]       =      FalconModelTester(self  )
         UpperCAmelCase_      :							Optional[int]       =      ConfigTester(self       ,config_class=_SCREAMING_SNAKE_CASE       ,hidden_size=37  )





    def 							a__				(				self  )			->		List[Any]:
         self.config_tester.run_common_tests()





    def 							a__				(				self  )			->		Tuple:
         UpperCAmelCase_      :							int       =      self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self  )			->		List[Any]:
         UpperCAmelCase_,   *UpperCAmelCase_      :							int       =      self.model_tester.prepare_config_and_inputs()
         for alibi in [True, False]:
              UpperCAmelCase_      :							str       =      alibi
              self.model_tester.create_and_check_model(_SCREAMING_SNAKE_CASE       ,*_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self  )			->		Union[str, Any]:
         UpperCAmelCase_,   UpperCAmelCase_      :							Tuple       =      self.model_tester.prepare_config_and_inputs_for_common()
         UpperCAmelCase_      :							str       =      3
         UpperCAmelCase_      :							int       =      input_dict['''input_ids''']
         UpperCAmelCase_      :							Any       =      input_ids.ne(1  ).to(_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							str       =      ids_tensor([self.model_tester.batch_size]       ,self.model_tester.type_sequence_label_size  )
         UpperCAmelCase_      :							Union[str, Any]       =      FalconForSequenceClassification(_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Optional[int]       =      model(_SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,labels=_SCREAMING_SNAKE_CASE  )
         self.assertEqual(result.logits.shape       ,(self.model_tester.batch_size, self.model_tester.num_labels)  )





    def 							a__				(				self  )			->		Dict:
         UpperCAmelCase_,   UpperCAmelCase_      :							Dict       =      self.model_tester.prepare_config_and_inputs_for_common()
         UpperCAmelCase_      :							Any       =      3
         UpperCAmelCase_      :							Union[str, Any]       =      '''single_label_classification'''
         UpperCAmelCase_      :							int       =      input_dict['''input_ids''']
         UpperCAmelCase_      :							int       =      input_ids.ne(1  ).to(_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							Union[str, Any]       =      ids_tensor([self.model_tester.batch_size]       ,self.model_tester.type_sequence_label_size  )
         UpperCAmelCase_      :							Dict       =      FalconForSequenceClassification(_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Any       =      model(_SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,labels=_SCREAMING_SNAKE_CASE  )
         self.assertEqual(result.logits.shape       ,(self.model_tester.batch_size, self.model_tester.num_labels)  )





    def 							a__				(				self  )			->		Dict:
         UpperCAmelCase_,   UpperCAmelCase_      :							Any       =      self.model_tester.prepare_config_and_inputs_for_common()
         UpperCAmelCase_      :							Any       =      input_dict['''input_ids''']
         UpperCAmelCase_      :							int       =      FalconForCausalLM(_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Any       =      model(_SCREAMING_SNAKE_CASE       ,use_cache=_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							List[Any]       =      input_ids.shape[0]
         UpperCAmelCase_      :							List[Any]       =      model._convert_to_rw_cache(result.past_key_values  )
         UpperCAmelCase_      :							str       =      model._convert_cache_to_standard_format(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )
         for layer in range(len(_SCREAMING_SNAKE_CASE  )  ):
              for tensor_idx in range(2  ):
                   self.assertTrue(rw_cache[layer][tensor_idx].ndim == 3  )
                   self.assertTrue(result.past_key_values[layer][tensor_idx].ndim == 4  )
                   self.assertTrue(
                       torch.all(result.past_key_values[layer][tensor_idx] == standard_cache[layer][tensor_idx]  )  )





    def 							a__				(				self  )			->		Any:
         UpperCAmelCase_,   UpperCAmelCase_      :							Any       =      self.model_tester.prepare_config_and_inputs_for_common()
         UpperCAmelCase_      :							Any       =      3
         UpperCAmelCase_      :							str       =      '''multi_label_classification'''
         UpperCAmelCase_      :							List[str]       =      input_dict['''input_ids''']
         UpperCAmelCase_      :							str       =      input_ids.ne(1  ).to(_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							Optional[Any]       =      ids_tensor(
             [self.model_tester.batch_size, config.num_labels]       ,self.model_tester.type_sequence_label_size  ).to(torch.float  )
         UpperCAmelCase_      :							Optional[int]       =      FalconForSequenceClassification(_SCREAMING_SNAKE_CASE  )
         model.to(_SCREAMING_SNAKE_CASE  )
         model.eval()
         UpperCAmelCase_      :							Any       =      model(_SCREAMING_SNAKE_CASE       ,attention_mask=_SCREAMING_SNAKE_CASE       ,labels=_SCREAMING_SNAKE_CASE  )
         self.assertEqual(result.logits.shape       ,(self.model_tester.batch_size, self.model_tester.num_labels)  )





    def 							a__				(				self  )			->		Optional[int]:
         # Falcon can have different numbers of KV-heads than the number of query heads, so we need
         # to override this test to use the right head counts.
         for model_class in self.all_generative_model_classes:
              UpperCAmelCase_,   UpperCAmelCase_      :							Union[str, Any]       =      self.model_tester.prepare_config_and_inputs_for_common()

              # If it doesn't support cache, pass the test
              if not hasattr(_SCREAMING_SNAKE_CASE       ,'''use_cache'''  ):
                   return

              UpperCAmelCase_      :							Any       =      model_class(_SCREAMING_SNAKE_CASE  ).to(_SCREAMING_SNAKE_CASE  )
              if "use_cache" not in inputs:
                   UpperCAmelCase_      :							Dict       =      True
              UpperCAmelCase_      :							Tuple       =      model(**_SCREAMING_SNAKE_CASE  )

              # If "past_key_values" is not returned, pass the test (e.g. RWKV uses a different cache name and format)
              if "past_key_values" not in outputs:
                   return

              UpperCAmelCase_      :							Union[str, Any]       =      (
                  getattr(_SCREAMING_SNAKE_CASE       ,'''decoder_layers'''       ,_SCREAMING_SNAKE_CASE  )
                  or getattr(_SCREAMING_SNAKE_CASE       ,'''num_decoder_layers'''       ,_SCREAMING_SNAKE_CASE  )
                  or config.num_hidden_layers
              )
              UpperCAmelCase_      :							Tuple       =      getattr(_SCREAMING_SNAKE_CASE       ,'''num_kv_heads'''       ,config.num_attention_heads  )
              UpperCAmelCase_      :							Optional[int]       =      getattr(_SCREAMING_SNAKE_CASE       ,'''d_model'''       ,config.hidden_size  )
              UpperCAmelCase_      :							Tuple       =      embed_dim // num_attention_heads

              UpperCAmelCase_      :							Optional[int]       =      outputs['''past_key_values''']
              self.assertEqual(len(_SCREAMING_SNAKE_CASE  )       ,_SCREAMING_SNAKE_CASE  )

              UpperCAmelCase_,   UpperCAmelCase_      :							Tuple       =      inputs['''input_ids'''].shape
              for i in range(_SCREAMING_SNAKE_CASE  ):
                   if config.new_decoder_architecture:
                        UpperCAmelCase_      :							int       =      config.num_attention_heads
                   elif config.multi_query:
                        UpperCAmelCase_      :							Optional[Any]       =      1
                   self.assertEqual(len(past_kv[0]  )       ,2  )  # K V for the decoder = 2
                   self.assertEqual(
                       past_kv[i][0].shape       ,(batch_size, num_attention_heads, seq_length, per_head_embed_dim)  )
                   self.assertEqual(
                       past_kv[i][1].shape       ,(batch_size, num_attention_heads, seq_length, per_head_embed_dim)  )


@require_torch
class   __a(					unittest.TestCase       ):






    """simple docstring"""





    @slow
    def 							a__				(				self  )			->		str:
         UpperCAmelCase_      :							Optional[int]       =      AutoTokenizer.from_pretrained('''Rocketknight1/falcon-rw-1b'''  )
         UpperCAmelCase_      :							Tuple       =      FalconForCausalLM.from_pretrained('''Rocketknight1/falcon-rw-1b'''  )
         model.eval()
         model.to(_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							Union[str, Any]       =      tokenizer('''My favorite food is'''       ,return_tensors='''pt'''  ).to(_SCREAMING_SNAKE_CASE  )

         UpperCAmelCase_      :							Any       =      (
             '''My favorite food is pizza. I love it so much that I have a pizza party every year for my birthday.'''
         )

         UpperCAmelCase_      :							Optional[int]       =      model.generate(**_SCREAMING_SNAKE_CASE       ,do_sample=_SCREAMING_SNAKE_CASE       ,max_new_tokens=19  )
         UpperCAmelCase_      :							Union[str, Any]       =      tokenizer.batch_decode(_SCREAMING_SNAKE_CASE  )[0]

         self.assertEqual(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )





    @slow
    def 							a__				(				self  )			->		str:
         # The big models are way too big for the CI, so we use tiny random models that resemble their
         # architectures but with much smaller and fewer layers
         for repo in ["Rocketknight1/tiny-random-falcon-7b", "Rocketknight1/tiny-random-falcon-40b"]:
              UpperCAmelCase_      :							List[Any]       =      AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  )
              UpperCAmelCase_      :							Any       =      FalconForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE  )
              model.eval()
              model.to(_SCREAMING_SNAKE_CASE  )
              UpperCAmelCase_      :							Any       =      tokenizer('''My favorite food is'''       ,return_tensors='''pt'''  ).to(_SCREAMING_SNAKE_CASE  )

              # We just test that these run without errors - the models are randomly initialized
              # and so the actual text outputs will be garbage
              model.generate(**_SCREAMING_SNAKE_CASE       ,do_sample=_SCREAMING_SNAKE_CASE       ,max_new_tokens=4  )
              model.generate(**_SCREAMING_SNAKE_CASE       ,do_sample=_SCREAMING_SNAKE_CASE       ,max_new_tokens=4  )
              model.generate(**_SCREAMING_SNAKE_CASE       ,num_beams=2       ,max_new_tokens=4  )





    @slow
    def 							a__				(				self  )			->		List[str]:
         # The big models are way too big for the CI, so we use tiny random models that resemble their
         # architectures but with much smaller and fewer layers
         with torch.no_grad():
              for repo in [
                  "Rocketknight1/falcon-rw-1b",
                  "Rocketknight1/tiny-random-falcon-7b",
                  "Rocketknight1/tiny-random-falcon-40b",
              ]:
                   UpperCAmelCase_      :							List[Any]       =      AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  )
                   UpperCAmelCase_      :							Optional[Any]       =      FalconForCausalLM.from_pretrained(_SCREAMING_SNAKE_CASE  )
                   model.eval()
                   model.to(device=_SCREAMING_SNAKE_CASE  )
                   UpperCAmelCase_      :							Optional[Any]       =      tokenizer('''My favorite food is'''       ,return_tensors='''pt'''  ).to(_SCREAMING_SNAKE_CASE  )

                   # Test results are the same with and without cache
                   UpperCAmelCase_      :							List[Any]       =      model.generate(**_SCREAMING_SNAKE_CASE       ,do_sample=_SCREAMING_SNAKE_CASE       ,max_new_tokens=20       ,use_cache=_SCREAMING_SNAKE_CASE  )
                   UpperCAmelCase_      :							int       =      model.generate(**_SCREAMING_SNAKE_CASE       ,do_sample=_SCREAMING_SNAKE_CASE       ,max_new_tokens=20       ,use_cache=_SCREAMING_SNAKE_CASE  )
                   self.assertTrue((outputs_cache - outputs_no_cache).sum().item() == 0  )





from dataclasses import dataclass, field
from typing import Optional

@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Model name or path of model to be trained.'''}       )
    lowerCAmelCase        =				field(
        default='''./'''    ,						metadata={'''help''': '''Save dir where model repo is cloned and models updates are saved to.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-train'''    ,						metadata={'''help''': '''Name or path of training dataset.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-valid'''    ,						metadata={'''help''': '''Name or path of validation dataset.'''}       )
    lowerCAmelCase        =				field(default=2    ,						metadata={'''help''': '''Batch size for training.'''}       )
    lowerCAmelCase        =				field(default=2    ,						metadata={'''help''': '''Batch size for evaluation.'''}       )
    lowerCAmelCase        =				field(default=0.1    ,						metadata={'''help''': '''Value of weight decay.'''}       )
    lowerCAmelCase        =				field(
        default=1_0000    ,						metadata={'''help''': '''Size of buffer used to shuffle streaming dataset.'''}       )
    lowerCAmelCase        =				field(default=2E-4    ,						metadata={'''help''': '''Learning rate fo training.'''}       )
    lowerCAmelCase        =				field(default='''cosine'''    ,						metadata={'''help''': '''Learning rate.'''}       )
    lowerCAmelCase        =				field(
        default=750    ,						metadata={'''help''': '''Number of warmup steps in the learning rate schedule.'''}       )
    lowerCAmelCase        =				field(
        default=16    ,						metadata={'''help''': '''Number of gradient accumulation steps.'''}       )
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''Use gradient checkpointing to reduce memory footprint.'''}       )
    lowerCAmelCase        =				field(default=5_0000    ,						metadata={'''help''': '''Maximum number of training steps.'''}       )
    lowerCAmelCase        =				field(
        default=-1    ,						metadata={'''help''': '''Maximum number of evaluation steps. If -1 the full dataset is evaluated.'''}       )
    lowerCAmelCase        =				field(default=1024    ,						metadata={'''help''': '''Sequence lengths used for training.'''}       )
    lowerCAmelCase        =				field(default=1    ,						metadata={'''help''': '''Training seed.'''}       )
    lowerCAmelCase        =				field(
        default=1024    ,						metadata={'''help''': '''Interval to save checkpoints. Measured as number of forward passes not training steps.'''}    ,						)
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''States path if the training should continue from a checkpoint folder.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''If True the data is pretokenized.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Model name or path of model to be evaluated.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-valid'''    ,						metadata={'''help''': '''Name or path of validation dataset.'''}       )
    lowerCAmelCase        =				field(default=2    ,						metadata={'''help''': '''Batch size used for evaluation.'''}       )
    lowerCAmelCase        =				field(
        default=-1    ,						metadata={'''help''': '''Maximum number of evaluation steps. If -1 the full dataset is evaluated.'''}       )
    lowerCAmelCase        =				field(default=1024    ,						metadata={'''help''': '''Length of sequences to be evaluated.'''}       )
    lowerCAmelCase        =				field(default=1    ,						metadata={'''help''': '''Random seed used for evaluation.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Model name or path of model to be evaluated.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Number of workers used for code evaluation.'''}       )
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''The number of human-eval tasks to run. If not included all tasks are evaluated.'''}    ,						)
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''Sample from the language model\'s output distribution.'''}       )
    lowerCAmelCase        =				field(default=0.2    ,						metadata={'''help''': '''Sampling temperature used for generation.'''}       )
    lowerCAmelCase        =				field(default=256    ,						metadata={'''help''': '''Maximum number of newly generated tokens.'''}       )
    lowerCAmelCase        =				field(default=0    ,						metadata={'''help''': '''Top-k parameter used for generation.'''}       )
    lowerCAmelCase        =				field(default=0.95    ,						metadata={'''help''': '''Top-p parameter used for nucleus sampling.'''}       )
    lowerCAmelCase        =				field(default=10    ,						metadata={'''help''': '''Number of generations to run in parallel.'''}       )
    lowerCAmelCase        =				field(
        default=200    ,						metadata={'''help''': '''Number of completions to generate for each sample.'''}       )
    lowerCAmelCase        =				field(default=1    ,						metadata={'''help''': '''Random seed used for evaluation.'''}       )
    lowerCAmelCase        =				field(
        default='''eval_results.json'''    ,						metadata={'''help''': '''Random seed used for evaluation.'''}       )
    lowerCAmelCase        =				field(
        default='''0'''    ,						metadata={'''help''': '''Allow `code_eval` to execute Python code on machine'''}       )
    lowerCAmelCase        =				field(
        default=-1    ,						metadata={
            '''help''': (
                '''Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive'''
                ''' number corresponds to which GPU device id to run on.'''
            )
        }    ,						)


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default=_a    ,						metadata={
            '''help''': '''The number of CPU cores to use for parallel preprocessing. Default uses the maximum available.'''
        }    ,						)
    lowerCAmelCase        =				field(
        default='''transformersbook/codeparrot'''    ,						metadata={'''help''': '''Folder or name of dataset to process.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot-clean'''    ,						metadata={'''help''': '''Folder to save processed processed dataset.'''}       )
    lowerCAmelCase        =				field(
        default=10_0000    ,						metadata={'''help''': '''Number of files to save per JSON output file.'''}       )
    lowerCAmelCase        =				field(default='''content'''    ,						metadata={'''help''': '''Column containing text data to process.'''}       )
    lowerCAmelCase        =				field(
        default=1000    ,						metadata={'''help''': '''Maximum line length in file, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=100    ,						metadata={'''help''': '''Maximum mean line length in file, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=0.25    ,						metadata={'''help''': '''Maximum fraction of non-alphanumeric characters, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=1.5    ,						metadata={'''help''': '''Minimum character token ratio for the file, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=0.7    ,						metadata={'''help''': '''Probability for filtering config, test and uncommon files.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Name or path to the tokenizer.'''}    ,						)
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''If True, near-duplicate samples are removed.'''}       )
    lowerCAmelCase        =				field(
        default=0.85    ,						metadata={'''help''': '''Jaccard threshold for near-duplicate samples.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''gpt2'''    ,						metadata={'''help''': '''Base tokenizer to build new tokenizer from.'''}       )
    lowerCAmelCase        =				field(
        default='''transformersbook/codeparrot-train'''    ,						metadata={'''help''': '''Dataset to train tokenizer on.'''}       )
    lowerCAmelCase        =				field(default='''content'''    ,						metadata={'''help''': '''Column containing text data to process.'''}       )
    lowerCAmelCase        =				field(default=20_0000    ,						metadata={'''help''': '''Number of examples to train tokenizer on.'''}       )
    lowerCAmelCase        =				field(
        default=3_2768    ,						metadata={'''help''': '''Number of examples to train the tokenizer on.'''}       )
    lowerCAmelCase        =				field(default='''codeparrot'''    ,						metadata={'''help''': '''Name of new tokenizer.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Push saved tokenizer to the hub.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Name or path to the tokenizer.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-train'''    ,						metadata={'''help''': '''Name or path to the dataset to pretokenize.'''}       )
    lowerCAmelCase        =				field(
        default='''tokenized-codeparrot-train'''    ,						metadata={'''help''': '''Repo name of the pretokenized data.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Number of workers used for code evaluation.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''gpt2-large'''    ,						metadata={'''help''': '''Configuration to use for model initialization.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Tokenizer attached to model.'''}       )
    lowerCAmelCase        =				field(default='''codeparrot'''    ,						metadata={'''help''': '''Name of the created model.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Push saved tokenizer to the hub.'''}       )