





import unittest

from transformers import XLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      XLMForMultipleChoice,
      XLMForQuestionAnswering,
      XLMForQuestionAnsweringSimple,
      XLMForSequenceClassification,
      XLMForTokenClassification,
      XLMModel,
      XLMWithLMHeadModel,
  )
  from transformers.models.xlm.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_LIST





class  lowerCAmelCase							:







  def __init__(      self  :List[Any]   ,							_lowercase  :List[str]   ,							_lowercase  :Optional[int]=13   ,							_lowercase  :List[str]=7   ,							_lowercase  :Dict=True   ,							_lowercase  :int=True   ,							_lowercase  :str=True   ,							_lowercase  :Optional[Any]=True   ,							_lowercase  :Dict=True   ,							_lowercase  :Any=False   ,							_lowercase  :Dict=False   ,							_lowercase  :Tuple=False   ,							_lowercase  :Optional[int]=2   ,							_lowercase  :str=99   ,							_lowercase  :Dict=0   ,							_lowercase  :List[str]=32   ,							_lowercase  :Dict=5   ,							_lowercase  :Optional[Any]=4   ,							_lowercase  :Union[str, Any]=0.1   ,							_lowercase  :str=0.1   ,							_lowercase  :Dict=5_12   ,							_lowercase  :List[Any]=2   ,							_lowercase  :str=0.02   ,							_lowercase  :Any=2   ,							_lowercase  :str=4   ,							_lowercase  :Tuple="last"   ,							_lowercase  :str=True   ,							_lowercase  :Tuple=None   ,							_lowercase  :List[Any]=0   ,							):






     '''simple docstring'''





     lowercase__			       =  parent
     lowercase__			       =  batch_size
     lowercase__			       =  seq_length
     lowercase__			       =  is_training
     lowercase__			       =  use_input_lengths
     lowercase__			       =  use_token_type_ids
     lowercase__			       =  use_labels
     lowercase__			       =  gelu_activation
     lowercase__			       =  sinusoidal_embeddings
     lowercase__			       =  causal
     lowercase__			       =  asm
     lowercase__			       =  n_langs
     lowercase__			       =  vocab_size
     lowercase__			       =  n_special
     lowercase__			       =  hidden_size
     lowercase__			       =  num_hidden_layers
     lowercase__			       =  num_attention_heads
     lowercase__			       =  hidden_dropout_prob
     lowercase__			       =  attention_probs_dropout_prob
     lowercase__			       =  max_position_embeddings
     lowercase__			       =  type_sequence_label_size
     lowercase__			       =  initializer_range
     lowercase__			       =  num_labels
     lowercase__			       =  num_choices
     lowercase__			       =  summary_type
     lowercase__			       =  use_proj
     lowercase__			       =  scope
     lowercase__			       =  bos_token_id







  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     lowercase__			       =  ids_tensor([self.batch_size, self.seq_length]   ,							self.vocab_size )
     lowercase__			       =  random_attention_mask([self.batch_size, self.seq_length] )

     lowercase__			       =  None
     if self.use_input_lengths:
        lowercase__			       =  (
            ids_tensor([self.batch_size]   ,							vocab_size=2 ) + self.seq_length - 2
        )  # small variation of seq_length

     lowercase__			       =  None
     if self.use_token_type_ids:
        lowercase__			       =  ids_tensor([self.batch_size, self.seq_length]   ,							self.n_langs )

     lowercase__			       =  None
     lowercase__			       =  None
     lowercase__			       =  None
     if self.use_labels:
        lowercase__			       =  ids_tensor([self.batch_size]   ,							self.type_sequence_label_size )
        lowercase__			       =  ids_tensor([self.batch_size, self.seq_length]   ,							self.num_labels )
        lowercase__			       =  ids_tensor([self.batch_size]   ,							2 ).float()
        lowercase__			       =  ids_tensor([self.batch_size]   ,							self.num_choices )

     lowercase__			       =  self.get_config()

     return (
         config,
         input_ids,
         token_type_ids,
         input_lengths,
         sequence_labels,
         token_labels,
         is_impossible_labels,
         choice_labels,
         input_mask,
     )







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     return XLMConfig(
         vocab_size=self.vocab_size   ,							n_special=self.n_special   ,							emb_dim=self.hidden_size   ,							n_layers=self.num_hidden_layers   ,							n_heads=self.num_attention_heads   ,							dropout=self.hidden_dropout_prob   ,							attention_dropout=self.attention_probs_dropout_prob   ,							gelu_activation=self.gelu_activation   ,							sinusoidal_embeddings=self.sinusoidal_embeddings   ,							asm=self.asm   ,							causal=self.causal   ,							n_langs=self.n_langs   ,							max_position_embeddings=self.max_position_embeddings   ,							initializer_range=self.initializer_range   ,							summary_type=self.summary_type   ,							use_proj=self.use_proj   ,							num_labels=self.num_labels   ,							bos_token_id=self.bos_token_id   ,							)







  def   UpperCAmelCase					(      self  :Any   ,							_lowercase  :str   ,							_lowercase  :str   ,							_lowercase  :Optional[int]   ,							_lowercase  :Tuple   ,							_lowercase  :Optional[int]   ,							_lowercase  :Tuple   ,							_lowercase  :int   ,							_lowercase  :Any   ,							_lowercase  :Tuple   ,							):






     '''simple docstring'''





     lowercase__			       =  XLMModel(config=_lowercase )
     model.to(_lowercase )
     model.eval()
     lowercase__			       =  model(_lowercase   ,							lengths=_lowercase   ,							langs=_lowercase )
     lowercase__			       =  model(_lowercase   ,							langs=_lowercase )
     lowercase__			       =  model(_lowercase )
     self.parent.assertEqual(result.last_hidden_state.shape   ,							(self.batch_size, self.seq_length, self.hidden_size) )







  def   UpperCAmelCase					(      self  :Optional[Any]   ,							_lowercase  :Dict   ,							_lowercase  :str   ,							_lowercase  :Optional[Any]   ,							_lowercase  :int   ,							_lowercase  :int   ,							_lowercase  :Tuple   ,							_lowercase  :List[Any]   ,							_lowercase  :Optional[int]   ,							_lowercase  :List[Any]   ,							):






     '''simple docstring'''





     lowercase__			       =  XLMWithLMHeadModel(_lowercase )
     model.to(_lowercase )
     model.eval()

     lowercase__			       =  model(_lowercase   ,							token_type_ids=_lowercase   ,							labels=_lowercase )
     self.parent.assertEqual(result.loss.shape   ,							() )
     self.parent.assertEqual(result.logits.shape   ,							(self.batch_size, self.seq_length, self.vocab_size) )







  def   UpperCAmelCase					(      self  :Union[str, Any]   ,							_lowercase  :List[Any]   ,							_lowercase  :Any   ,							_lowercase  :Optional[int]   ,							_lowercase  :Optional[int]   ,							_lowercase  :Dict   ,							_lowercase  :str   ,							_lowercase  :List[str]   ,							_lowercase  :Optional[int]   ,							_lowercase  :Optional[int]   ,							):






     '''simple docstring'''





     lowercase__			       =  XLMForQuestionAnsweringSimple(_lowercase )
     model.to(_lowercase )
     model.eval()

     lowercase__			       =  model(_lowercase )

     lowercase__			       =  model(_lowercase   ,							start_positions=_lowercase   ,							end_positions=_lowercase )
     lowercase__			       =  outputs
     self.parent.assertEqual(result.start_logits.shape   ,							(self.batch_size, self.seq_length) )
     self.parent.assertEqual(result.end_logits.shape   ,							(self.batch_size, self.seq_length) )







  def   UpperCAmelCase					(      self  :Optional[int]   ,							_lowercase  :Tuple   ,							_lowercase  :List[str]   ,							_lowercase  :Tuple   ,							_lowercase  :Optional[int]   ,							_lowercase  :Optional[Any]   ,							_lowercase  :str   ,							_lowercase  :str   ,							_lowercase  :int   ,							_lowercase  :List[str]   ,							):






     '''simple docstring'''





     lowercase__			       =  XLMForQuestionAnswering(_lowercase )
     model.to(_lowercase )
     model.eval()

     lowercase__			       =  model(_lowercase )

     lowercase__			       =  model(
         _lowercase   ,							start_positions=_lowercase   ,							end_positions=_lowercase   ,							cls_index=_lowercase   ,							is_impossible=_lowercase   ,							p_mask=_lowercase   ,							)

     lowercase__			       =  model(
         _lowercase   ,							start_positions=_lowercase   ,							end_positions=_lowercase   ,							cls_index=_lowercase   ,							is_impossible=_lowercase   ,							)

     ((lowercase__)       ,							)			       =  result_with_labels.to_tuple()

     lowercase__			       =  model(_lowercase   ,							start_positions=_lowercase   ,							end_positions=_lowercase )

     ((lowercase__)       ,							)			       =  result_with_labels.to_tuple()

     self.parent.assertEqual(result_with_labels.loss.shape   ,							() )
     self.parent.assertEqual(result.start_top_log_probs.shape   ,							(self.batch_size, model.config.start_n_top) )
     self.parent.assertEqual(result.start_top_index.shape   ,							(self.batch_size, model.config.start_n_top) )
     self.parent.assertEqual(
         result.end_top_log_probs.shape   ,							(self.batch_size, model.config.start_n_top * model.config.end_n_top) )
     self.parent.assertEqual(
         result.end_top_index.shape   ,							(self.batch_size, model.config.start_n_top * model.config.end_n_top) )
     self.parent.assertEqual(result.cls_logits.shape   ,							(self.batch_size,) )







  def   UpperCAmelCase					(      self  :Tuple   ,							_lowercase  :Tuple   ,							_lowercase  :Dict   ,							_lowercase  :Optional[int]   ,							_lowercase  :Dict   ,							_lowercase  :Dict   ,							_lowercase  :int   ,							_lowercase  :Optional[Any]   ,							_lowercase  :Dict   ,							_lowercase  :str   ,							):






     '''simple docstring'''





     lowercase__			       =  XLMForSequenceClassification(_lowercase )
     model.to(_lowercase )
     model.eval()

     lowercase__			       =  model(_lowercase )
     lowercase__			       =  model(_lowercase   ,							labels=_lowercase )
     self.parent.assertEqual(result.loss.shape   ,							() )
     self.parent.assertEqual(result.logits.shape   ,							(self.batch_size, self.type_sequence_label_size) )







  def   UpperCAmelCase					(      self  :str   ,							_lowercase  :Union[str, Any]   ,							_lowercase  :Optional[int]   ,							_lowercase  :Any   ,							_lowercase  :Optional[Any]   ,							_lowercase  :Tuple   ,							_lowercase  :int   ,							_lowercase  :Tuple   ,							_lowercase  :Any   ,							_lowercase  :Union[str, Any]   ,							):






     '''simple docstring'''





     lowercase__			       =  self.num_labels
     lowercase__			       =  XLMForTokenClassification(_lowercase )
     model.to(_lowercase )
     model.eval()

     lowercase__			       =  model(_lowercase   ,							attention_mask=_lowercase   ,							labels=_lowercase )
     self.parent.assertEqual(result.logits.shape   ,							(self.batch_size, self.seq_length, self.num_labels) )







  def   UpperCAmelCase					(      self  :Dict   ,							_lowercase  :Optional[int]   ,							_lowercase  :Union[str, Any]   ,							_lowercase  :Optional[Any]   ,							_lowercase  :List[str]   ,							_lowercase  :str   ,							_lowercase  :int   ,							_lowercase  :int   ,							_lowercase  :Optional[int]   ,							_lowercase  :List[Any]   ,							):






     '''simple docstring'''





     lowercase__			       =  self.num_choices
     lowercase__			       =  XLMForMultipleChoice(config=_lowercase )
     model.to(_lowercase )
     model.eval()
     lowercase__			       =  input_ids.unsqueeze(1 ).expand(-1   ,							self.num_choices   ,							-1 ).contiguous()
     lowercase__			       =  token_type_ids.unsqueeze(1 ).expand(-1   ,							self.num_choices   ,							-1 ).contiguous()
     lowercase__			       =  input_mask.unsqueeze(1 ).expand(-1   ,							self.num_choices   ,							-1 ).contiguous()
     lowercase__			       =  model(
         _lowercase   ,							attention_mask=_lowercase   ,							token_type_ids=_lowercase   ,							labels=_lowercase   ,							)
     self.parent.assertEqual(result.logits.shape   ,							(self.batch_size, self.num_choices) )







  def   UpperCAmelCase					(      self  :Optional[int] ):






     '''simple docstring'''





     lowercase__			       =  self.prepare_config_and_inputs()
     (
         (
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							(
         lowercase__
     )       ,							
     )			       =  config_and_inputs
     lowercase__			       =  {"input_ids": input_ids, "token_type_ids": token_type_ids, "lengths": input_lengths}
     return config, inputs_dict





@require_torch
class  lowerCAmelCase							(       lowercase_							,							lowercase_							,							lowercase_							,							unittest.TestCase ):
  __lowerCamelCase		=      (
      (
          XLMModel,
          XLMWithLMHeadModel,
          XLMForQuestionAnswering,
          XLMForSequenceClassification,
          XLMForQuestionAnsweringSimple,
          XLMForTokenClassification,
          XLMForMultipleChoice,
      )
      if is_torch_available()
      else ()
  )
  __lowerCamelCase		=      (
      (XLMWithLMHeadModel,) if is_torch_available() else ()
  )  # TODO (PVP): Check other models whether language generation is also applicable
  __lowerCamelCase		=      (
      {
          'feature-extraction': XLMModel,
          'fill-mask': XLMWithLMHeadModel,
          'question-answering': XLMForQuestionAnsweringSimple,
          'text-classification': XLMForSequenceClassification,
          'text-generation': XLMWithLMHeadModel,
          'token-classification': XLMForTokenClassification,
          'zero-shot': XLMForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )







  def   UpperCAmelCase					(      self  :List[str]   ,							_lowercase  :Optional[int]   ,							_lowercase  :Union[str, Any]   ,							_lowercase  :List[Any]   ,							_lowercase  :List[str]   ,							_lowercase  :int ):






     '''simple docstring'''





     if (
         pipeline_test_casse_name == "QAPipelineTests"
         and tokenizer_name is not None
         and not tokenizer_name.endswith("Fast" )
     ):
        # `QAPipelineTests` fails for a few models when the slower tokenizer are used.
        # (The slower tokenizers were never used for pipeline tests before the pipeline testing rework)
        # TODO: check (and possibly fix) the `QAPipelineTests` with slower tokenizer
        return True

     return False







  def   UpperCAmelCase					(      self  :List[Any]   ,							_lowercase  :Optional[Any]   ,							_lowercase  :Optional[int]   ,							_lowercase  :Any=False ):






     '''simple docstring'''





     lowercase__			       =  super()._prepare_for_class(_lowercase   ,							_lowercase   ,							return_labels=_lowercase )

     if return_labels:
        if model_class.__name__ == "XLMForQuestionAnswering":
           lowercase__			       =  torch.zeros(
               self.model_tester.batch_size   ,							dtype=torch.long   ,							device=_lowercase )
           lowercase__			       =  torch.zeros(
               self.model_tester.batch_size   ,							dtype=torch.long   ,							device=_lowercase )

     return inputs_dict







  def   UpperCAmelCase					(      self  :Tuple ):






     '''simple docstring'''





     lowercase__			       =  XLMModelTester(self )
     lowercase__			       =  ConfigTester(self   ,							config_class=_lowercase   ,							emb_dim=37 )







  def   UpperCAmelCase					(      self  :int ):






     '''simple docstring'''





     self.config_tester.run_common_tests()







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_model(*_lowercase )







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_lm_head(*_lowercase )







  def   UpperCAmelCase					(      self  :Tuple ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_simple_qa(*_lowercase )







  def   UpperCAmelCase					(      self  :List[str] ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_qa(*_lowercase )







  def   UpperCAmelCase					(      self  :Dict ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_sequence_classif(*_lowercase )







  def   UpperCAmelCase					(      self  :Union[str, Any] ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_token_classif(*_lowercase )







  def   UpperCAmelCase					(      self  :str ):






     '''simple docstring'''





     lowercase__			       =  self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_xlm_for_multiple_choice(*_lowercase )







  def   UpperCAmelCase					(      self  :Tuple   ,							_lowercase  :int   ,							_lowercase  :Optional[Any]   ,							_lowercase  :int   ,							_lowercase  :Any   ,							_lowercase  :Optional[int]   ,							_lowercase  :List[Any]=False   ,							_lowercase  :Tuple=1 ):






     '''simple docstring'''





     self.assertIsInstance(_lowercase   ,							_lowercase )
     self.assertListEqual(
         [isinstance(_lowercase   ,							_lowercase ) for iter_attentions in attentions]   ,							[True] * len(_lowercase ) )
     self.assertEqual(len(_lowercase )   ,							(max_length - min_length) * num_beam_groups )

     for idx, iter_attentions in enumerate(_lowercase ):
        # adds PAD dummy token
        lowercase__			       =  min_length + idx + 1
        lowercase__			       =  min_length + idx + 1

        lowercase__			       =  (
            batch_size * num_beam_groups,
            config.num_attention_heads,
            tgt_len,
            src_len,
        )
        # check attn size
        self.assertListEqual(
            [layer_attention.shape for layer_attention in iter_attentions]   ,							[expected_shape] * len(_lowercase ) )







  def   UpperCAmelCase					(      self  :str   ,							_lowercase  :Tuple   ,							_lowercase  :Any   ,							_lowercase  :List[str]   ,							_lowercase  :Optional[int]   ,							_lowercase  :Any   ,							_lowercase  :Any=False   ,							_lowercase  :List[Any]=1 ):






     '''simple docstring'''





     self.assertIsInstance(_lowercase   ,							_lowercase )
     self.assertListEqual(
         [isinstance(_lowercase   ,							_lowercase ) for iter_hidden_states in hidden_states]   ,							[True] * len(_lowercase )   ,							)
     self.assertEqual(len(_lowercase )   ,							(max_length - min_length) * num_beam_groups )

     for idx, iter_hidden_states in enumerate(_lowercase ):
        # adds PAD dummy token
        lowercase__			       =  min_length + idx + 1
        lowercase__			       =  (batch_size * num_beam_groups, seq_len, config.hidden_size)
        # check hidden size
        self.assertListEqual(
            [layer_hidden_states.shape for layer_hidden_states in iter_hidden_states]   ,							[expected_shape] * len(_lowercase )   ,							)
     pass







  @slow
  def   UpperCAmelCase					(      self  :Tuple ):






     '''simple docstring'''





     for model_name in XLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
        lowercase__			       =  XLMModel.from_pretrained(_lowercase )
        self.assertIsNotNone(_lowercase )





@require_torch
class  lowerCAmelCase							(       unittest.TestCase ):







  @slow
  def   UpperCAmelCase					(      self  :Optional[Any] ):






     '''simple docstring'''





     lowercase__			       =  XLMWithLMHeadModel.from_pretrained("xlm-mlm-en-2048" )
     model.to(_lowercase )
     lowercase__			       =  torch.tensor([[14, 4_47]]   ,							dtype=torch.long   ,							device=_lowercase )  # the president
     lowercase__			       =  [
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
         14,
         4_47,
     ]  # the president the president the president the president the president the president the president the president the president the president
     # TODO(PVP): this and other input_ids I tried for generation give pretty bad results. Not sure why. Model might just not be made for auto-regressive inference
     lowercase__			       =  model.generate(_lowercase   ,							do_sample=_lowercase )
     self.assertListEqual(output_ids[0].cpu().numpy().tolist()   ,							_lowercase )













from abc import ABC, abstractmethod
from typing import Optional, Union

from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit
from ..utils.typing import NestedDataStructureLike, PathLike





class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :str   ,							_lowercase  :Optional[NestedDataStructureLike[PathLike]] = None   ,							_lowercase  :Optional[NamedSplit] = None   ,							_lowercase  :Optional[Features] = None   ,							_lowercase  :str = None   ,							_lowercase  :bool = False   ,							_lowercase  :bool = False   ,							_lowercase  :Optional[int] = None   ,							**_lowercase  :Tuple   ,							):






     '''simple docstring'''





     lowercase__			       =  path_or_paths
     lowercase__			       =  split if split or isinstance(_lowercase   ,							_lowercase ) else "train"
     lowercase__			       =  features
     lowercase__			       =  cache_dir
     lowercase__			       =  keep_in_memory
     lowercase__			       =  streaming
     lowercase__			       =  num_proc
     lowercase__			       =  kwargs







  @abstractmethod
  def   UpperCAmelCase					(      self  :Any ):






     '''simple docstring'''





     pass





class  lowerCAmelCase							(       lowercase_ ):







  def __init__(      self  :List[Any]   ,							_lowercase  :Optional[Features] = None   ,							_lowercase  :str = None   ,							_lowercase  :bool = False   ,							_lowercase  :bool = False   ,							_lowercase  :Optional[int] = None   ,							**_lowercase  :Optional[int]   ,							):






     '''simple docstring'''





     lowercase__			       =  features
     lowercase__			       =  cache_dir
     lowercase__			       =  keep_in_memory
     lowercase__			       =  streaming
     lowercase__			       =  num_proc
     lowercase__			       =  kwargs







  @abstractmethod
  def   UpperCAmelCase					(      self  :int ):






     '''simple docstring'''





     pass






