





'''simple docstring'''




from __future__ import annotations

import copy
import inspect
import unittest

import numpy as np

from transformers import is_tf_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
      TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
      TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
      TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
      TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
      LayoutLMvaConfig,
      TFLayoutLMvaForQuestionAnswering,
      TFLayoutLMvaForSequenceClassification,
      TFLayoutLMvaForTokenClassification,
      TFLayoutLMvaModel,
  )

if is_vision_available():
  from PIL import Image

  from transformers import LayoutLMvaImageProcessor


class 			SCREAMING_SNAKE_CASE   :


     def __init__(						self						,     _UpperCAmelCase						,     _UpperCAmelCase=2						,     _UpperCAmelCase=3						,     _UpperCAmelCase=4						,     _UpperCAmelCase=2						,     _UpperCAmelCase=7						,     _UpperCAmelCase=True						,     _UpperCAmelCase=True						,     _UpperCAmelCase=True						,     _UpperCAmelCase=True						,     _UpperCAmelCase=99						,     _UpperCAmelCase=36						,     _UpperCAmelCase=2						,     _UpperCAmelCase=4						,     _UpperCAmelCase=37						,     _UpperCAmelCase="gelu"						,     _UpperCAmelCase=0.1						,     _UpperCAmelCase=0.1						,     _UpperCAmelCase=512						,     _UpperCAmelCase=16						,     _UpperCAmelCase=2						,     _UpperCAmelCase=0.02						,     _UpperCAmelCase=6						,     _UpperCAmelCase=6						,     _UpperCAmelCase=3						,     _UpperCAmelCase=4						,     _UpperCAmelCase=None						,     _UpperCAmelCase=1000						,     ):

          '''simple docstring'''






          __A						:			Dict								=	parent
          __A						:			Optional[int]								=	batch_size
          __A						:			List[str]								=	num_channels
          __A						:			Optional[Any]								=	image_size
          __A						:			List[str]								=	patch_size
          __A						:			Dict								=	is_training
          __A						:			Optional[Any]								=	use_input_mask
          __A						:			str								=	use_token_type_ids
          __A						:			Any								=	use_labels
          __A						:			str								=	vocab_size
          __A						:			List[str]								=	hidden_size
          __A						:			Any								=	num_hidden_layers
          __A						:			Union[str, Any]								=	num_attention_heads
          __A						:			int								=	intermediate_size
          __A						:			str								=	hidden_act
          __A						:			int								=	hidden_dropout_prob
          __A						:			List[Any]								=	attention_probs_dropout_prob
          __A						:			List[Any]								=	max_position_embeddings
          __A						:			Union[str, Any]								=	type_vocab_size
          __A						:			Union[str, Any]								=	type_sequence_label_size
          __A						:			Optional[int]								=	initializer_range
          __A						:			List[Any]								=	coordinate_size
          __A						:			Any								=	shape_size
          __A						:			Any								=	num_labels
          __A						:			List[Any]								=	num_choices
          __A						:			Any								=	scope
          __A						:			Dict								=	range_bbox

          # LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
          __A						:			Optional[int]								=	text_seq_length
          __A						:			List[Any]								=	(image_size // patch_size) ** 2 + 1
          __A						:			Dict								=	self.text_seq_length + self.image_seq_length


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			int								=	ids_tensor([self.batch_size, self.text_seq_length]						,     self.vocab_size)

          __A						:			Tuple								=	ids_tensor([self.batch_size, self.text_seq_length, 4]						,     self.range_bbox)
          __A						:			str								=	bbox.numpy()
          # Ensure that bbox is legal
          for i in range(bbox.shape[0]):
               for j in range(bbox.shape[1]):
                    if bbox[i, j, 3] < bbox[i, j, 1]:
                         __A						:			str								=	bbox[i, j, 3]
                         __A						:			List[str]								=	bbox[i, j, 1]
                         __A						:			List[str]								=	tmp_coordinate
                    if bbox[i, j, 2] < bbox[i, j, 0]:
                         __A						:			Dict								=	bbox[i, j, 2]
                         __A						:			Optional[int]								=	bbox[i, j, 0]
                         __A						:			Optional[int]								=	tmp_coordinate
          __A						:			List[str]								=	tf.constant(_UpperCAmelCase)

          __A						:			int								=	floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size])

          __A						:			str								=	None
          if self.use_input_mask:
               __A						:			Optional[Any]								=	random_attention_mask([self.batch_size, self.text_seq_length])

          __A						:			str								=	None
          if self.use_token_type_ids:
               __A						:			Optional[int]								=	ids_tensor([self.batch_size, self.text_seq_length]						,     self.type_vocab_size)

          __A						:			Optional[Any]								=	None
          __A						:			str								=	None
          if self.use_labels:
               __A						:			List[Any]								=	ids_tensor([self.batch_size]						,     self.type_sequence_label_size)
               __A						:			str								=	ids_tensor([self.batch_size, self.text_seq_length]						,     self.num_labels)

          __A						:			Dict								=	LayoutLMvaConfig(
              vocab_size=self.vocab_size						,     hidden_size=self.hidden_size						,     num_hidden_layers=self.num_hidden_layers						,     num_attention_heads=self.num_attention_heads						,     intermediate_size=self.intermediate_size						,     hidden_act=self.hidden_act						,     hidden_dropout_prob=self.hidden_dropout_prob						,     attention_probs_dropout_prob=self.attention_probs_dropout_prob						,     max_position_embeddings=self.max_position_embeddings						,     type_vocab_size=self.type_vocab_size						,     initializer_range=self.initializer_range						,     coordinate_size=self.coordinate_size						,     shape_size=self.shape_size						,     input_size=self.image_size						,     patch_size=self.patch_size						,     )

          return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	TFLayoutLMvaModel(config=_UpperCAmelCase)

          # text + image
          __A						:			Any								=	model(_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     training=_UpperCAmelCase)
          __A						:			Any								=	model(
              _UpperCAmelCase						,     bbox=_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     token_type_ids=_UpperCAmelCase						,     training=_UpperCAmelCase						,     )
          __A						:			Optional[int]								=	model(_UpperCAmelCase						,     bbox=_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     training=_UpperCAmelCase)

          self.parent.assertEqual(result.last_hidden_state.shape						,     (self.batch_size, self.seq_length, self.hidden_size))

          # text only
          __A						:			Dict								=	model(_UpperCAmelCase						,     training=_UpperCAmelCase)
          self.parent.assertEqual(
              result.last_hidden_state.shape						,     (self.batch_size, self.text_seq_length, self.hidden_size))

          # image only
          __A						:			Optional[int]								=	model({'pixel_values': pixel_values}						,     training=_UpperCAmelCase)
          self.parent.assertEqual(
              result.last_hidden_state.shape						,     (self.batch_size, self.image_seq_length, self.hidden_size))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	self.num_labels
          __A						:			Union[str, Any]								=	TFLayoutLMvaForSequenceClassification(config=_UpperCAmelCase)
          __A						:			Optional[Any]								=	model(
              _UpperCAmelCase						,     bbox=_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     token_type_ids=_UpperCAmelCase						,     labels=_UpperCAmelCase						,     training=_UpperCAmelCase						,     )
          self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.num_labels))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Optional[Any]								=	self.num_labels
          __A						:			Optional[Any]								=	TFLayoutLMvaForTokenClassification(config=_UpperCAmelCase)
          __A						:			int								=	model(
              _UpperCAmelCase						,     bbox=_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     token_type_ids=_UpperCAmelCase						,     labels=_UpperCAmelCase						,     training=_UpperCAmelCase						,     )
          self.parent.assertEqual(result.logits.shape						,     (self.batch_size, self.text_seq_length, self.num_labels))


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          __A						:			Any								=	2
          __A						:			Optional[int]								=	TFLayoutLMvaForQuestionAnswering(config=_UpperCAmelCase)
          __A						:			Tuple								=	model(
              _UpperCAmelCase						,     bbox=_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     attention_mask=_UpperCAmelCase						,     token_type_ids=_UpperCAmelCase						,     start_positions=_UpperCAmelCase						,     end_positions=_UpperCAmelCase						,     training=_UpperCAmelCase						,     )
          self.parent.assertEqual(result.start_logits.shape						,     (self.batch_size, self.seq_length))
          self.parent.assertEqual(result.end_logits.shape						,     (self.batch_size, self.seq_length))


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	self.prepare_config_and_inputs()
          ((__A)			,(__A)			,(__A)			,(__A)			,(__A)			,(__A)			,(__A)			,(__A))						:			List[str]								=	config_and_inputs
          __A						:			Union[str, Any]								=	{
              'input_ids': input_ids,
              'bbox': bbox,
              'pixel_values': pixel_values,
              'token_type_ids': token_type_ids,
              'attention_mask': input_mask,
          }
          return config, inputs_dict




@require_tf
class 			SCREAMING_SNAKE_CASE   (a__ ,		a__ ,		unittest.TestCase							):
     lowerCAmelCase		      =	(
         (
             TFLayoutLMvaModel,
             TFLayoutLMvaForQuestionAnswering,
             TFLayoutLMvaForSequenceClassification,
             TFLayoutLMvaForTokenClassification,
         )
         if is_tf_available()
         else ()
     )
     lowerCAmelCase		      =	(
         {'''document-question-answering''': TFLayoutLMvaForQuestionAnswering, '''feature-extraction''': TFLayoutLMvaModel}
         if is_tf_available()
         else {}
     )

     lowerCAmelCase		      =	False
     lowerCAmelCase		      =	False
     lowerCAmelCase		      =	False


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase):

          '''simple docstring'''






          return True


     def 							SCREAMING_SNAKE_CASE  (						self						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase=False):

          '''simple docstring'''






          __A						:			Dict								=	copy.deepcopy(_UpperCAmelCase)

          if model_class in get_values(_UpperCAmelCase):
               __A						:			Any								=	{
                   k: tf.tile(tf.expand_dims(_UpperCAmelCase						,     1)						,     (1, self.model_tester.num_choices) + (1,) * (v.ndim - 1))
                   if isinstance(_UpperCAmelCase						,     tf.Tensor) and v.ndim > 0
                   else v
                   for k, v in inputs_dict.items()
               }

          if return_labels:
               if model_class in get_values(_UpperCAmelCase):
                    __A						:			Optional[int]								=	tf.ones(self.model_tester.batch_size						,     dtype=tf.intaa)
               elif model_class in get_values(_UpperCAmelCase):
                    __A						:			Any								=	tf.zeros(self.model_tester.batch_size						,     dtype=tf.intaa)
                    __A						:			List[str]								=	tf.zeros(self.model_tester.batch_size						,     dtype=tf.intaa)
               elif model_class in get_values(_UpperCAmelCase):
                    __A						:			Union[str, Any]								=	tf.zeros(self.model_tester.batch_size						,     dtype=tf.intaa)
               elif model_class in get_values(_UpperCAmelCase):
                    __A						:			Optional[Any]								=	tf.zeros(
                        (self.model_tester.batch_size, self.model_tester.text_seq_length)						,     dtype=tf.intaa)

          return inputs_dict


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Tuple								=	TFLayoutLMvaModelTester(self)
          __A						:			Any								=	ConfigTester(self						,     config_class=_UpperCAmelCase						,     hidden_size=37)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.config_tester.run_common_tests()


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A			,__A						:			Optional[Any]								=	self.model_tester.prepare_config_and_inputs_for_common()
          for model_class in self.all_model_classes:
               __A						:			List[Any]								=	model_class(_UpperCAmelCase)
               if getattr(_UpperCAmelCase						,     'hf_compute_loss'						,     _UpperCAmelCase):
                    # The number of elements in the loss should be the same as the number of elements in the label
                    __A						:			List[Any]								=	self._prepare_for_class(inputs_dict.copy()						,     _UpperCAmelCase						,     return_labels=_UpperCAmelCase)
                    __A						:			Tuple								=	prepared_for_class[
                        sorted(prepared_for_class.keys() - inputs_dict.keys()						,     reverse=_UpperCAmelCase)[0]
                    ]
                    __A						:			Optional[int]								=	added_label.shape.as_list()[:1]

                    # Test that model correctly compute the loss with kwargs
                    __A						:			List[Any]								=	self._prepare_for_class(inputs_dict.copy()						,     _UpperCAmelCase						,     return_labels=_UpperCAmelCase)
                    __A						:			Optional[Any]								=	prepared_for_class.pop('input_ids')

                    __A						:			Any								=	model(_UpperCAmelCase						,     **_UpperCAmelCase)[0]
                    self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])

                    # Test that model correctly compute the loss when we mask some positions
                    __A						:			Union[str, Any]								=	self._prepare_for_class(inputs_dict.copy()						,     _UpperCAmelCase						,     return_labels=_UpperCAmelCase)
                    __A						:			int								=	prepared_for_class.pop('input_ids')
                    if "labels" in prepared_for_class:
                         __A						:			int								=	prepared_for_class['labels'].numpy()
                         if len(labels.shape) > 1 and labels.shape[1] != 1:
                              __A						:			Dict								=	-100
                              __A						:			Dict								=	tf.convert_to_tensor(_UpperCAmelCase)
                              __A						:			List[Any]								=	model(_UpperCAmelCase						,     **_UpperCAmelCase)[0]
                              self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])
                              self.assertTrue(not np.any(np.isnan(loss.numpy())))

                # Test that model correctly compute the loss with a dict
                    __A						:			Optional[int]								=	self._prepare_for_class(inputs_dict.copy()						,     _UpperCAmelCase						,     return_labels=_UpperCAmelCase)
                    __A						:			Union[str, Any]								=	model(_UpperCAmelCase)[0]
                    self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])

                    # Test that model correctly compute the loss with a tuple
                    __A						:			Tuple								=	self._prepare_for_class(inputs_dict.copy()						,     _UpperCAmelCase						,     return_labels=_UpperCAmelCase)

                    # Get keys that were added with the _prepare_for_class function
                    __A						:			Optional[Any]								=	prepared_for_class.keys() - inputs_dict.keys()
                    __A						:			Optional[int]								=	inspect.signature(model.call).parameters
                    __A						:			Dict								=	list(signature.keys())

                    # Create a dictionary holding the location of the tensors in the tuple
                    __A						:			List[str]								=	{0: 'input_ids'}
                    for label_key in label_keys:
                         __A						:			Tuple								=	signature_names.index(_UpperCAmelCase)
                         __A						:			Dict								=	label_key
                    __A						:			Union[str, Any]								=	sorted(tuple_index_mapping.items())
                    # Initialize a list with their default values, update the values and convert to a tuple
                    __A						:			str								=	[]

                    for name in signature_names:
                         if name != "kwargs":
                              list_input.append(signature[name].default)

                    for index, value in sorted_tuple_index_mapping:
                         __A						:			Union[str, Any]								=	prepared_for_class[value]

                    __A						:			str								=	tuple(_UpperCAmelCase)

                    # Send to model
                    __A						:			Optional[Any]								=	model(tuple_input[:-1])[0]

                    self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1])


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          (
              (
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,
          )						:			int								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(_UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          (
              (
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,
          )						:			int								=	self.model_tester.prepare_config_and_inputs()
          for type in ["absolute", "relative_key", "relative_key_query"]:
               __A						:			Any								=	type
               self.model_tester.create_and_check_model(_UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          (
              (
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,
          )						:			Dict								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(
              _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          (
              (
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,
          )						:			Union[str, Any]								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(
              _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          (
              (
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,(
              __A
          )			,
          )						:			str								=	self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(
              _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase						,     _UpperCAmelCase)


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for model_name in TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               __A						:			str								=	TFLayoutLMvaModel.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)






def 	_lowerCAmelCase      (  )					->					Any:
     __A						:			int								=	Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'      )
     return image







@require_tf
class 			SCREAMING_SNAKE_CASE   (unittest.TestCase							):


     @cached_property
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return LayoutLMvaImageProcessor(apply_ocr=_UpperCAmelCase) if is_vision_available() else None


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	TFLayoutLMvaModel.from_pretrained('microsoft/layoutlmv3-base')

          __A						:			str								=	self.default_image_processor
          __A						:			Optional[Any]								=	prepare_img()
          __A						:			Dict								=	image_processor(images=_UpperCAmelCase						,     return_tensors='tf').pixel_values

          __A						:			List[Any]								=	tf.constant([[1, 2]])
          __A						:			List[Any]								=	tf.expand_dims(tf.constant([[1, 2, 3, 4], [5, 6, 7, 8]])						,     axis=0)

          # forward pass
          __A						:			Any								=	model(input_ids=_UpperCAmelCase						,     bbox=_UpperCAmelCase						,     pixel_values=_UpperCAmelCase						,     training=_UpperCAmelCase)

          # verify the logits
          __A						:			str								=	(1, 199, 768)
          self.assertEqual(outputs.last_hidden_state.shape						,     _UpperCAmelCase)

          __A						:			Union[str, Any]								=	tf.constant(
              [[-0.0529, 0.3618, 0.1632], [-0.1587, -0.1667, -0.0400], [-0.1557, -0.1671, -0.0505]])

          self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]						,     _UpperCAmelCase						,     atol=1e-4))






'''simple docstring'''




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast


class 			SCREAMING_SNAKE_CASE   (unittest.TestCase							):


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	0


     @slow
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
               self.assertGreater(len(_UpperCAmelCase)						,     0)

          for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
               __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
               self.assertIsNotNone(_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     (GPTaTokenizer, GPTaTokenizerFast))
               self.assertGreater(len(_UpperCAmelCase)						,     0)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (RobertaTokenizer, RobertaTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     20)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Tuple								=	AutoConfig.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)
          # Check that tokenizer_type ≠ model_type
          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     config=_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          self.assertEqual(tokenizer.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.txt'						,     os.path.join(_UpperCAmelCase						,     'vocab.txt'))

               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='bert'						,     use_fast=_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.json'						,     os.path.join(_UpperCAmelCase						,     'vocab.json'))
               shutil.copy('./tests/fixtures/merges.txt'						,     os.path.join(_UpperCAmelCase						,     'merges.txt'))

               __A						:			str								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='gpt2'						,     use_fast=_UpperCAmelCase)
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.txt'						,     os.path.join(_UpperCAmelCase						,     'vocab.txt'))

               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='bert')
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy('./tests/fixtures/vocab.json'						,     os.path.join(_UpperCAmelCase						,     'vocab.json'))
               shutil.copy('./tests/fixtures/merges.txt'						,     os.path.join(_UpperCAmelCase						,     'merges.txt'))

               __A						:			List[str]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     tokenizer_type='gpt2')
               self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with pytest.raises(_UpperCAmelCase):
               AutoTokenizer.from_pretrained('./'						,     tokenizer_type='xxx')


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               __A						:			List[Any]								=	tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased')
               self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))

               if isinstance(_UpperCAmelCase						,     _UpperCAmelCase):
                    self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						,     _UpperCAmelCase)
               else:
                    self.assertEqual(tokenizer.do_lower_case						,     _UpperCAmelCase)

               self.assertEqual(tokenizer.model_max_length						,     512)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               with self.assertRaisesRegex(
                   _UpperCAmelCase						,     'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'						,     ):
                    __A						:			str								=	tokenizer_class.from_pretrained('julien-c/herlolip-not-exists')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Any								=	TOKENIZER_MAPPING.values()
          __A						:			Union[str, Any]								=	[]

          for slow_tok, fast_tok in tokenizers:
               if slow_tok is not None:
                    tokenizer_names.append(slow_tok.__name__)

               if fast_tok is not None:
                    tokenizer_names.append(fast_tok.__name__)

          for tokenizer_name in tokenizer_names:
               # must find the right class
               tokenizer_class_from_name(_UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'						,     use_fast=_UpperCAmelCase)						,     _UpperCAmelCase)
          self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased')						,     _UpperCAmelCase)


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[str]								=	AutoTokenizer.from_pretrained('distilbert-base-uncased'						,     do_lower_case=_UpperCAmelCase)
          __A						:			str								=	'Hello, world. How are you?'
          __A						:			List[str]								=	tokenizer.tokenize(_UpperCAmelCase)
          self.assertEqual('[UNK]'						,     tokens[0])

          __A						:			Dict								=	AutoTokenizer.from_pretrained('microsoft/mpnet-base'						,     do_lower_case=_UpperCAmelCase)
          __A						:			List[Any]								=	tokenizer.tokenize(_UpperCAmelCase)
          self.assertEqual('[UNK]'						,     tokens[0])


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config')
          self.assertEqual(type(_UpperCAmelCase)						,     _UpperCAmelCase)
          self.assertEqual(tokenizer.model_max_length						,     512)
          self.assertEqual(tokenizer.vocab_size						,     3_0000)
          self.assertEqual(tokenizer.unk_token						,     '[UNK]')
          self.assertEqual(tokenizer.padding_side						,     'right')
          self.assertEqual(tokenizer.truncation_side						,     'right')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          self.assertIsInstance(_UpperCAmelCase						,     (BertTokenizer, BertTokenizerFast))
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)

          self.assertIsInstance(_UpperCAmelCase						,     tokenizer.__class__)
          self.assertEqual(tokenizera.vocab_size						,     12)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Dict								=	AutoTokenizer.from_pretrained('ctrl')
          # There is no fast CTRL so this always gives us a slow tokenizer.
          self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			List[Any]								=	get_tokenizer_config('bert-base-cased')
          __A						:			Optional[int]								=	config.pop('_commit_hash'						,     _UpperCAmelCase)
          # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
          self.assertEqual(_UpperCAmelCase						,     {'do_lower_case': False})

          # This model does not have a tokenizer_config so we get back an empty dict.
          __A						:			Dict								=	get_tokenizer_config(_UpperCAmelCase)
          self.assertDictEqual(_UpperCAmelCase						,     {})

          # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
          __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Any								=	get_tokenizer_config(_UpperCAmelCase)

          # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
          self.assertEqual(config['tokenizer_class']						,     'BertTokenizer')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)

               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_UpperCAmelCase):
                    AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)

               __A						:			Optional[Any]								=	CustomTokenizer.from_pretrained(_UpperCAmelCase)
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)

                    __A						:			int								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)

               # Can register in two steps
               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, None))
               AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, CustomTokenizerFast))

               del TOKENIZER_MAPPING._extra_content[CustomConfig]
               # Can register in one step
               AutoTokenizer.register(
                   _UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						,     (CustomTokenizer, CustomTokenizerFast))

               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_UpperCAmelCase):
                    AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)

               # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
               # and that model does not have a tokenizer.json
               with tempfile.TemporaryDirectory() as tmp_dir:
                    __A						:			Optional[int]								=	BertTokenizerFast.from_pretrained(_UpperCAmelCase)
                    bert_tokenizer.save_pretrained(_UpperCAmelCase)
                    __A						:			Dict								=	CustomTokenizerFast.from_pretrained(_UpperCAmelCase)

               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)

                    __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

                    __A						:			Any								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
                    self.assertIsInstance(_UpperCAmelCase						,     _UpperCAmelCase)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaises(_UpperCAmelCase):
               __A						:			List[str]								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer')
          # If remote code is disabled, we can't load this config.
          with self.assertRaises(_UpperCAmelCase):
               __A						:			Dict								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)

          __A						:			str								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(tokenizer.special_attribute_present)
          # Test tokenizer can be reloaded.
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase)
               __A						:			Dict								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(reloaded_tokenizer.special_attribute_present)

          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizerFast')

               # Test we can also load the slow version
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               # Test tokenizer can be reloaded.
               with tempfile.TemporaryDirectory() as tmp_dir:
                    tokenizer.save_pretrained(_UpperCAmelCase)
                    __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertTrue(reloaded_tokenizer.special_attribute_present)
          else:
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertEqual(reloaded_tokenizer.__class__.__name__						,     'NewTokenizer')


     @require_tokenizers
     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          class 			SCREAMING_SNAKE_CASE   (a__							):
               lowerCAmelCase		      =	False




          class 			SCREAMING_SNAKE_CASE   (a__							):
               lowerCAmelCase		      =	NewTokenizer
               lowerCAmelCase		      =	False

          try:
               AutoConfig.register('custom'						,     _UpperCAmelCase)
               AutoTokenizer.register(_UpperCAmelCase						,     slow_tokenizer_class=_UpperCAmelCase)
               AutoTokenizer.register(_UpperCAmelCase						,     fast_tokenizer_class=_UpperCAmelCase)
               # If remote code is not set, the default is to use local
               __A						:			List[Any]								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer')
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertFalse(tokenizer.special_attribute_present)
               __A						:			Dict								=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote code is disabled, we load the local one.
               __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertFalse(tokenizer.special_attribute_present)
               __A						:			Any								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertFalse(tokenizer.special_attribute_present)

               # If remote is enabled, we load from the Hub
               __A						:			int								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')
               self.assertTrue(tokenizer.special_attribute_present)
               __A						:			Optional[Any]								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
               self.assertTrue(tokenizer.special_attribute_present)

          finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                    del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                    del TOKENIZER_MAPPING._extra_content[CustomConfig]


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			int								=	AutoTokenizer.from_pretrained(
              'hf-internal-testing/test_dynamic_tokenizer_legacy'						,     trust_remote_code=_UpperCAmelCase)
          self.assertTrue(tokenizer.special_attribute_present)
          if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizerFast')

               # Test we can also load the slow version
               __A						:			int								=	AutoTokenizer.from_pretrained(
                   'hf-internal-testing/test_dynamic_tokenizer_legacy'						,     trust_remote_code=_UpperCAmelCase						,     use_fast=_UpperCAmelCase)
               self.assertTrue(tokenizer.special_attribute_present)
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')
          else:
               self.assertEqual(tokenizer.__class__.__name__						,     'NewTokenizer')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaisesRegex(
              _UpperCAmelCase						,     'bert-base is not a local folder and is not a valid model identifier'):
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('bert-base')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          with self.assertRaisesRegex(
              _UpperCAmelCase						,     R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'):
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained(_UpperCAmelCase						,     revision='aaaaaa')


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          __A						:			Optional[int]								=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
          with RequestCounter() as counter:
               __A						:			Union[str, Any]								=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
               self.assertEqual(counter.get_request_count						,     0)
               self.assertEqual(counter.head_request_count						,     1)
               self.assertEqual(counter.other_request_count						,     0)