





from __future__ import annotations

import os
import tempfile
import unittest

from transformers import ConvBertConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers import (
        TFConvBertForMaskedLM,
        TFConvBertForMultipleChoice,
        TFConvBertForQuestionAnswering,
        TFConvBertForSequenceClassification,
        TFConvBertForTokenClassification,
        TFConvBertModel,
    )





class 			A__   :




   """simple docstring"""



   def __init__( self ,  lowercase ,  lowercase=13 ,  lowercase=7 ,  lowercase=True ,  lowercase=True ,  lowercase=True ,  lowercase=True ,  lowercase=99 ,  lowercase=32 ,  lowercase=2 ,  lowercase=4 ,  lowercase=37 ,  lowercase="gelu" ,  lowercase=0.1 ,  lowercase=0.1 ,  lowercase=512 ,  lowercase=16 ,  lowercase=2 ,  lowercase=0.02 ,  lowercase=3 ,  lowercase=4 ,  lowercase=None ,  )      ->						int:







          '''simple docstring'''

          a__						:     Optional[Any]			=    parent
          a__						:     Tuple			=    13
          a__						:     Optional[int]			=    7
          a__						:     Optional[Any]			=    True
          a__						:     List[str]			=    True
          a__						:     Union[str, Any]			=    True
          a__						:     Optional[Any]			=    True
          a__						:     Dict			=    99
          a__						:     Tuple			=    384
          a__						:     List[Any]			=    2
          a__						:     Union[str, Any]			=    4
          a__						:     Dict			=    37
          a__						:     Tuple			=    'gelu'
          a__						:     str			=    0.1
          a__						:     Tuple			=    0.1
          a__						:     Union[str, Any]			=    512
          a__						:     Tuple			=    16
          a__						:     Tuple			=    2
          a__						:     Dict			=    0.02
          a__						:     List[str]			=    3
          a__						:     int			=    4
          a__						:     Optional[int]			=    128
          a__						:     List[str]			=    2
          a__						:     Optional[Any]			=    9
          a__						:     Optional[Any]			=    1
          a__						:     Tuple			=    None

   def 							__lowercase		( self)      ->						List[str]:







          '''simple docstring'''

          a__						:     Optional[int]			=    ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size)

          a__						:     int			=    None
          if self.use_input_mask:
                 a__						:     Optional[int]			=    random_attention_mask([self.batch_size, self.seq_length])

          a__						:     Tuple			=    None
          if self.use_token_type_ids:
                 a__						:     Optional[int]			=    ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size)

          a__						:     List[Any]			=    None
          a__						:     List[str]			=    None
          a__						:     Optional[Any]			=    None
          if self.use_labels:
                 a__						:     str			=    ids_tensor([self.batch_size] ,  self.type_sequence_label_size)
                 a__						:     int			=    ids_tensor([self.batch_size, self.seq_length] ,  self.num_labels)
                 a__						:     List[str]			=    ids_tensor([self.batch_size] ,  self.num_choices)

          a__						:     Optional[Any]			=    ConvBertConfig(
              vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  initializer_range=self.initializer_range ,  return_dict=lowercase ,  )

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase)      ->						int:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    TFConvBertModel(config=lowercase)
          a__						:     Union[str, Any]			=    {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

          a__						:     Optional[Any]			=    [input_ids, input_mask]
          a__						:     Tuple			=    model(lowercase)

          a__						:     List[Any]			=    model(lowercase)

          self.parent.assertEqual(result.last_hidden_state.shape ,  (self.batch_size, self.seq_length, self.hidden_size))

   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase)      ->						Union[str, Any]:







          '''simple docstring'''

          a__						:     int			=    TFConvBertForMaskedLM(config=lowercase)
          a__						:     int			=    {
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          a__						:     Union[str, Any]			=    model(lowercase)
          self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.vocab_size))

   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase)      ->						str:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    self.num_labels
          a__						:     Union[str, Any]			=    TFConvBertForSequenceClassification(config=lowercase)
          a__						:     Optional[int]			=    {
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          a__						:     Optional[Any]			=    model(lowercase)
          self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_labels))

   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase)      ->						Any:







          '''simple docstring'''

          a__						:     Tuple			=    self.num_choices
          a__						:     Optional[Any]			=    TFConvBertForMultipleChoice(config=lowercase)
          a__						:     Any			=    tf.tile(tf.expand_dims(lowercase ,  1) ,  (1, self.num_choices, 1))
          a__						:     List[str]			=    tf.tile(tf.expand_dims(lowercase ,  1) ,  (1, self.num_choices, 1))
          a__						:     Dict			=    tf.tile(tf.expand_dims(lowercase ,  1) ,  (1, self.num_choices, 1))
          a__						:     int			=    {
              'input_ids': multiple_choice_inputs_ids,
              'attention_mask': multiple_choice_input_mask,
              'token_type_ids': multiple_choice_token_type_ids,
          }
          a__						:     Optional[int]			=    model(lowercase)
          self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.num_choices))

   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     Dict			=    self.num_labels
          a__						:     str			=    TFConvBertForTokenClassification(config=lowercase)
          a__						:     List[str]			=    {
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }
          a__						:     Union[str, Any]			=    model(lowercase)
          self.parent.assertEqual(result.logits.shape ,  (self.batch_size, self.seq_length, self.num_labels))

   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase ,  lowercase)      ->						Any:







          '''simple docstring'''

          a__						:     Optional[Any]			=    TFConvBertForQuestionAnswering(config=lowercase)
          a__						:     Optional[int]			=    {
              'input_ids': input_ids,
              'attention_mask': input_mask,
              'token_type_ids': token_type_ids,
          }

          a__						:     int			=    model(lowercase)
          self.parent.assertEqual(result.start_logits.shape ,  (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape ,  (self.batch_size, self.seq_length))

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     str			=    self.prepare_config_and_inputs()
          (
              a__							
          )						:     str			=    config_and_inputs
          a__						:     List[str]			=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
          return config, inputs_dict





@require_tf
class 			A__   (							__UpperCamelCase			,  __UpperCamelCase			,  unittest.TestCase     ):




   """simple docstring"""



   __A       :    Dict	  =  (
       (
           TFConvBertModel,
           TFConvBertForMaskedLM,
           TFConvBertForQuestionAnswering,
           TFConvBertForSequenceClassification,
           TFConvBertForTokenClassification,
           TFConvBertForMultipleChoice,
       )
       if is_tf_available()
       else ()
   )
   __A       :    int	  =  (
       {
           '''feature-extraction''': TFConvBertModel,
           '''fill-mask''': TFConvBertForMaskedLM,
           '''question-answering''': TFConvBertForQuestionAnswering,
           '''text-classification''': TFConvBertForSequenceClassification,
           '''token-classification''': TFConvBertForTokenClassification,
           '''zero-shot''': TFConvBertForSequenceClassification,
       }
       if is_tf_available()
       else {}
   )
   __A       :    Optional[Any]	  =  False
   __A       :    List[str]	  =  False
   __A       :    Union[str, Any]	  =  False

   def 							__lowercase		( self)      ->						Any:







          '''simple docstring'''

          a__						:     str			=    TFConvBertModelTester(self)
          a__						:     Union[str, Any]			=    ConfigTester(self ,  config_class=lowercase ,  hidden_size=37)

   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          self.config_tester.run_common_tests()

   def 							__lowercase		( self)      ->						str:







          '''simple docstring'''

          a__						:     Tuple			=    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*lowercase)

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     Tuple			=    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*lowercase)

   def 							__lowercase		( self)      ->						int:







          '''simple docstring'''

          a__						:     Dict			=    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*lowercase)

   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          a__						:     str			=    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*lowercase)

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     List[str]			=    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*lowercase)

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     str			=    self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*lowercase)

   @slow
   def 							__lowercase		( self)      ->						int:







          '''simple docstring'''

          a__						:     List[Any]			=    self.model_tester.prepare_config_and_inputs_for_common()
          a__						:     List[Any]			=    True
          a__						:     Any			=    True

          if hasattr(lowercase ,  'use_cache'):
                 a__						:     Dict			=    True

          a__						:     Optional[int]			=    getattr(self.model_tester ,  'encoder_seq_length' ,  self.model_tester.seq_length)
          a__						:     Optional[Any]			=    getattr(self.model_tester ,  'key_length' ,  lowercase)

          for model_class in self.all_model_classes:
                 a__						:     Union[str, Any]			=    self._prepare_for_class(lowercase ,  lowercase)
                 a__						:     List[str]			=    model_class(lowercase)
                 a__						:     List[Any]			=    len(model(lowercase))

                 with tempfile.TemporaryDirectory() as tmpdirname:
                        model.save_pretrained(lowercase ,  saved_model=lowercase)
                        a__						:     Any			=    os.path.join(lowercase ,  'saved_model' ,  '1')
                        a__						:     int			=    tf.keras.models.load_model(lowercase)
                        a__						:     Dict			=    model(lowercase)

                        if self.is_encoder_decoder:
                               a__						:     Any			=    outputs['encoder_hidden_states']
                               a__						:     Optional[Any]			=    outputs['encoder_attentions']
                        else:
                               a__						:     int			=    outputs['hidden_states']
                               a__						:     Dict			=    outputs['attentions']

                        self.assertEqual(len(lowercase) ,  lowercase)

                        a__						:     List[str]			=    getattr(
                            self.model_tester ,  'expected_num_hidden_layers' ,  self.model_tester.num_hidden_layers + 1)

                        self.assertEqual(len(lowercase) ,  lowercase)
                        self.assertListEqual(
                            list(output_hidden_states[0].shape[-2:]) ,  [self.model_tester.seq_length, self.model_tester.hidden_size] ,  )

                        self.assertEqual(len(lowercase) ,  self.model_tester.num_hidden_layers)
                        self.assertListEqual(
                            list(output_attentions[0].shape[-3:]) ,  [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length] ,  )

   @slow
   def 							__lowercase		( self)      ->						int:







          '''simple docstring'''

          a__						:     Tuple			=    TFConvBertModel.from_pretrained('YituTech/conv-bert-base')
          self.assertIsNotNone(lowercase)

   def 							__lowercase		( self)      ->						Dict:







          '''simple docstring'''

          a__						:     Tuple			=    self.model_tester.prepare_config_and_inputs_for_common()
          a__						:     Tuple			=    True
          a__						:     Dict			=    getattr(self.model_tester ,  'decoder_seq_length' ,  self.model_tester.seq_length)
          a__						:     Optional[int]			=    getattr(self.model_tester ,  'encoder_seq_length' ,  self.model_tester.seq_length)
          a__						:     int			=    getattr(self.model_tester ,  'key_length' ,  lowercase)
          a__						:     Optional[int]			=    getattr(self.model_tester ,  'key_length' ,  lowercase)

          def check_decoder_attentions_output(lowercase):
                 a__						:     Optional[Any]			=    len(lowercase)
                 self.assertEqual(out_len % 2 ,  0)
                 a__						:     Any			=    outputs.decoder_attentions
                 self.assertEqual(len(lowercase) ,  self.model_tester.num_hidden_layers)
                 self.assertListEqual(
                     list(decoder_attentions[0].shape[-3:]) ,  [self.model_tester.num_attention_heads / 2, decoder_seq_length, decoder_key_length] ,  )

          def check_encoder_attentions_output(lowercase):
                 a__						:     int			=    [
                     t.numpy() for t in (outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions)
                 ]
                 self.assertEqual(len(lowercase) ,  self.model_tester.num_hidden_layers)
                 self.assertListEqual(
                     list(attentions[0].shape[-3:]) ,  [self.model_tester.num_attention_heads / 2, encoder_seq_length, encoder_key_length] ,  )

          for model_class in self.all_model_classes:
                 a__						:     str			=    True
                 a__						:     List[str]			=    False
                 a__						:     str			=    model_class(lowercase)
                 a__						:     Any			=    model(self._prepare_for_class(lowercase ,  lowercase))
                 a__						:     List[Any]			=    len(lowercase)
                 self.assertEqual(config.output_hidden_states ,  lowercase)
                 check_encoder_attentions_output(lowercase)

                 if self.is_encoder_decoder:
                        a__						:     int			=    model_class(lowercase)
                        a__						:     Dict			=    model(self._prepare_for_class(lowercase ,  lowercase))
                        self.assertEqual(config.output_hidden_states ,  lowercase)
                        check_decoder_attentions_output(lowercase)

                 # Check that output attentions can also be changed via the config
                 del inputs_dict["output_attentions"]
                 a__						:     Union[str, Any]			=    True
                 a__						:     int			=    model_class(lowercase)
                 a__						:     Dict			=    model(self._prepare_for_class(lowercase ,  lowercase))
                 self.assertEqual(config.output_hidden_states ,  lowercase)
                 check_encoder_attentions_output(lowercase)

                 # Check attention is always last and order is fine
                 a__						:     str			=    True
                 a__						:     List[Any]			=    True
                 a__						:     int			=    model_class(lowercase)
                 a__						:     Tuple			=    model(self._prepare_for_class(lowercase ,  lowercase))

                 self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1) ,  len(lowercase))
                 self.assertEqual(model.config.output_hidden_states ,  lowercase)
                 check_encoder_attentions_output(lowercase)





@require_tf
class 			A__   (							unittest.TestCase     ):




   """simple docstring"""



   @slow
   def 							__lowercase		( self)      ->						str:







          '''simple docstring'''

          a__						:     Tuple			=    TFConvBertModel.from_pretrained('YituTech/conv-bert-base')
          a__						:     List[Any]			=    tf.constant([[0, 1, 2, 3, 4, 5]])
          a__						:     Any			=    model(lowercase)[0]

          a__						:     Dict			=    [1, 6, 768]
          self.assertEqual(output.shape ,  lowercase)

          a__						:     Tuple			=    tf.constant(
              [
                  [
                      [-0.03_47_54_93, -0.4_68_60_34, -0.30_63_88_32],
                      [0.22_63_72_48, -0.26_98_86_46, -0.7_42_34_24],
                      [0.10_32_48_68, -0.45_01_35_08, -0.58_28_07_84],
                  ]
              ])
          tf.debugging.assert_near(output[:, :3, :3] ,  lowercase ,  atol=1e-4)









import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.eager import context
    from tensorflow.python.framework import ops

    from transformers import GradientAccumulator, create_optimizer





@require_tf
class 			A__   (							unittest.TestCase     ):




   """simple docstring"""



   def 							__lowercase		( self ,  lowercase ,  lowercase ,  lowercase)      ->						List[str]:







          '''simple docstring'''

          self.assertEqual(len(lowercase) ,  len(lowercase))
          for a, b in zip(lowercase ,  lowercase):
                 self.assertAlmostEqual(lowercase ,  lowercase ,  delta=lowercase)

   def 							__lowercase		( self)      ->						Any:







          '''simple docstring'''

          a__						:     Tuple			=    GradientAccumulator()
          accumulator([tf.constant([1.0, 2.0])])
          accumulator([tf.constant([-2.0, 1.0])])
          accumulator([tf.constant([-1.0, 2.0])])
          with self.assertRaises(lowercase):
                 accumulator([tf.constant([1.0, 1.0]), tf.constant([2.0, 2.0])])
          self.assertEqual(accumulator.step ,  3)
          self.assertEqual(len(accumulator.gradients) ,  1)
          self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist() ,  [-2.0, 5.0] ,  tol=1e-2)
          accumulator.reset()
          self.assertEqual(accumulator.step ,  0)
          self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist() ,  [0.0, 0.0] ,  tol=1e-2)

   def 							__lowercase		( self)      ->						List[Any]:







          '''simple docstring'''

          a__						:     Tuple			=    None
          ops.enable_eager_execution_internal()
          a__						:     Optional[int]			=    tf.config.list_physical_devices('CPU')
          if len(lowercase) == 1:
                 tf.config.set_logical_device_configuration(
                     physical_devices[0] ,  [tf.config.LogicalDeviceConfiguration(), tf.config.LogicalDeviceConfiguration()])
          a__						:     int			=    tf.config.list_logical_devices(device_type='CPU')
          a__						:     Union[str, Any]			=    tf.distribute.MirroredStrategy(devices=devices[:2])

          with strategy.scope():
                 a__						:     str			=    GradientAccumulator()
                 a__						:     Tuple			=    tf.Variable([4.0, 3.0])
                 a__			,							a__						:     Tuple			=    create_optimizer(5e-5 ,  10 ,  5)
                 a__						:     Tuple			=    tf.Variable([0.0, 0.0] ,  trainable=lowercase)

          def accumulate_on_replica(lowercase):
                 accumulator([gradient])

          def apply_on_replica():
                 optimizer.apply_gradients(list(zip(accumulator.gradients ,  [variable])))

          @tf.function
          def accumulate(lowercase ,  lowercase):
                 with strategy.scope():
                        a__						:     Union[str, Any]			=    strategy.experimental_local_results(lowercase)
                        local_variables[0].assign(lowercase)
                        local_variables[1].assign(lowercase)
                        strategy.run(lowercase ,  args=(gradient_placeholder,))

          @tf.function
          def apply_grad():
                 with strategy.scope():
                        strategy.run(lowercase)

          def _check_local_values(lowercase ,  lowercase):
                 a__						:     Union[str, Any]			=    strategy.experimental_local_results(accumulator._gradients[0])
                 self.assertListAlmostEqual(values[0].value() ,  lowercase ,  tol=1e-2)
                 self.assertListAlmostEqual(values[1].value() ,  lowercase ,  tol=1e-2)

          accumulate([1.0, 2.0] ,  [-1.0, 1.0])
          accumulate([3.0, -1.0] ,  [-1.0, -1.0])
          accumulate([-2.0, 2.0] ,  [3.0, -2.0])
          self.assertEqual(accumulator.step ,  3)
          _check_local_values([2.0, 3.0] ,  [1.0, -2.0])
          apply_grad()
          self.assertListAlmostEqual(variable.value() ,  [4.0, 3.0] ,  tol=1e-2)
          accumulator.reset()
          self.assertEqual(accumulator.step ,  0)
          _check_local_values([0.0, 0.0] ,  [0.0, 0.0])


