




"""simple docstring"""



import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import (
          LiltForQuestionAnswering,
          LiltForSequenceClassification,
          LiltForTokenClassification,
          LiltModel,
      )
      from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST




class      UpperCAmelCase_  :




    def __init__(   self						:	str     ,			snake_case_						:	Any     ,			snake_case_						:	List[Any]=13     ,			snake_case_						:	Optional[int]=7     ,			snake_case_						:	List[Any]=True     ,			snake_case_						:	Optional[Any]=True     ,			snake_case_						:	str=True     ,			snake_case_						:	List[Any]=True     ,			snake_case_						:	List[str]=99     ,			snake_case_						:	List[Any]=24     ,			snake_case_						:	List[str]=2     ,			snake_case_						:	Dict=6     ,			snake_case_						:	Optional[Any]=37     ,			snake_case_						:	Dict="gelu"     ,			snake_case_						:	Union[str, Any]=0.1     ,			snake_case_						:	List[Any]=0.1     ,			snake_case_						:	Tuple=512     ,			snake_case_						:	int=16     ,			snake_case_						:	Any=2     ,			snake_case_						:	Union[str, Any]=0.02     ,			snake_case_						:	Tuple=3     ,			snake_case_						:	Union[str, Any]=None     ,			snake_case_						:	str=1_000     ,			)				->							int:




     '''simple docstring'''

     A__	=    parent
     A__	=    batch_size
     A__	=    seq_length
     A__	=    is_training
     A__	=    use_input_mask
     A__	=    use_token_type_ids
     A__	=    use_labels
     A__	=    vocab_size
     A__	=    hidden_size
     A__	=    num_hidden_layers
     A__	=    num_attention_heads
     A__	=    intermediate_size
     A__	=    hidden_act
     A__	=    hidden_dropout_prob
     A__	=    attention_probs_dropout_prob
     A__	=    max_position_embeddings
     A__	=    type_vocab_size
     A__	=    type_sequence_label_size
     A__	=    initializer_range
     A__	=    num_labels
     A__	=    scope
     A__	=    range_bbox




    def 							__magic_name__							(   self						:	List[Any]				)				->							Dict:




     '''simple docstring'''

     A__	=    ids_tensor([self.batch_size, self.seq_length]     ,			self.vocab_size				)

     A__	=    ids_tensor([self.batch_size, self.seq_length, 4]     ,			self.range_bbox				)
     # Ensure that bbox is legal
     for i in range(bbox.shape[0]				):
      for j in range(bbox.shape[1]				):
       if bbox[i, j, 3] < bbox[i, j, 1]:
        A__	=    bbox[i, j, 3]
        A__	=    bbox[i, j, 1]
        A__	=    t
       if bbox[i, j, 2] < bbox[i, j, 0]:
        A__	=    bbox[i, j, 2]
        A__	=    bbox[i, j, 0]
        A__	=    t

     A__	=    None
     if self.use_input_mask:
      A__	=    ids_tensor([self.batch_size, self.seq_length]     ,			vocab_size=2				)

     A__	=    None
     if self.use_token_type_ids:
      A__	=    ids_tensor([self.batch_size, self.seq_length]     ,			self.type_vocab_size				)

     A__	=    None
     A__	=    None
     if self.use_labels:
      A__	=    ids_tensor([self.batch_size]     ,			self.type_sequence_label_size				)
      A__	=    ids_tensor([self.batch_size, self.seq_length]     ,			self.num_labels				)

     A__	=    self.get_config()

     return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels




    def 							__magic_name__							(   self						:	str				)				->							List[Any]:




     '''simple docstring'''

     return LiltConfig(
         vocab_size=self.vocab_size     ,			hidden_size=self.hidden_size     ,			num_hidden_layers=self.num_hidden_layers     ,			num_attention_heads=self.num_attention_heads     ,			intermediate_size=self.intermediate_size     ,			hidden_act=self.hidden_act     ,			hidden_dropout_prob=self.hidden_dropout_prob     ,			attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,			max_position_embeddings=self.max_position_embeddings     ,			type_vocab_size=self.type_vocab_size     ,			initializer_range=self.initializer_range     ,			)




    def 							__magic_name__							(   self						:	Optional[int]     ,			snake_case_						:	Union[str, Any]     ,			snake_case_						:	Any     ,			snake_case_						:	int     ,			snake_case_						:	Optional[Any]     ,			snake_case_						:	Dict     ,			snake_case_						:	List[str]     ,			snake_case_						:	str     ,			)				->							Optional[int]:




     '''simple docstring'''

     A__	=    LiltModel(config=snake_case_				)
     model.to(snake_case_				)
     model.eval()
     A__	=    model(snake_case_     ,			bbox=snake_case_     ,			attention_mask=snake_case_     ,			token_type_ids=snake_case_				)
     A__	=    model(snake_case_     ,			bbox=snake_case_     ,			token_type_ids=snake_case_				)
     A__	=    model(snake_case_     ,			bbox=snake_case_				)

     self.parent.assertEqual(result.last_hidden_state.shape     ,			(self.batch_size, self.seq_length, self.hidden_size)				)
     self.parent.assertEqual(result.pooler_output.shape     ,			(self.batch_size, self.hidden_size)				)




    def 							__magic_name__							(   self						:	Optional[int]     ,			snake_case_						:	int     ,			snake_case_						:	Dict     ,			snake_case_						:	Optional[int]     ,			snake_case_						:	Union[str, Any]     ,			snake_case_						:	Optional[int]     ,			snake_case_						:	Dict     ,			snake_case_						:	List[Any]     ,			)				->							List[Any]:




     '''simple docstring'''

     A__	=    self.num_labels
     A__	=    LiltForTokenClassification(config=snake_case_				)
     model.to(snake_case_				)
     model.eval()
     A__	=    model(
         snake_case_     ,			bbox=snake_case_     ,			attention_mask=snake_case_     ,			token_type_ids=snake_case_     ,			labels=snake_case_				)
     self.parent.assertEqual(result.logits.shape     ,			(self.batch_size, self.seq_length, self.num_labels)				)




    def 							__magic_name__							(   self						:	int     ,			snake_case_						:	Optional[Any]     ,			snake_case_						:	List[str]     ,			snake_case_						:	List[str]     ,			snake_case_						:	List[Any]     ,			snake_case_						:	List[str]     ,			snake_case_						:	str     ,			snake_case_						:	str     ,			)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    LiltForQuestionAnswering(config=snake_case_				)
     model.to(snake_case_				)
     model.eval()
     A__	=    model(
         snake_case_     ,			bbox=snake_case_     ,			attention_mask=snake_case_     ,			token_type_ids=snake_case_     ,			start_positions=snake_case_     ,			end_positions=snake_case_     ,			)
     self.parent.assertEqual(result.start_logits.shape     ,			(self.batch_size, self.seq_length)				)
     self.parent.assertEqual(result.end_logits.shape     ,			(self.batch_size, self.seq_length)				)




    def 							__magic_name__							(   self						:	Any				)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    self.prepare_config_and_inputs()
     (
         (
         A__
     ),				(
         A__
     ),				(
         A__
     ),				(
         A__
     ),				(
         A__
     ),				(
         A__
     ),				(
         A__
     ),				
     )	=    config_and_inputs
     A__	=    {
         "input_ids": input_ids,
         "bbox": bbox,
         "token_type_ids": token_type_ids,
         "attention_mask": input_mask,
     }
     return config, inputs_dict






@require_torch
class      UpperCAmelCase_  (						A_,       A_,       A_,       unittest.TestCase							):
    lowercase__						 =				(
        (
            LiltModel,
            LiltForSequenceClassification,
            LiltForTokenClassification,
            LiltForQuestionAnswering,
        )
        if is_torch_available()
        else ()
    )
    lowercase__						 =				(
        {
            '''feature-extraction''': LiltModel,
            '''question-answering''': LiltForQuestionAnswering,
            '''text-classification''': LiltForSequenceClassification,
            '''token-classification''': LiltForTokenClassification,
            '''zero-shot''': LiltForSequenceClassification,
        }
        if is_torch_available()
        else {}
    )
    lowercase__						 =				False
    lowercase__						 =				False




    def 							__magic_name__							(   self						:	Optional[int]     ,			snake_case_						:	int     ,			snake_case_						:	List[Any]     ,			snake_case_						:	Any     ,			snake_case_						:	List[str]     ,			snake_case_						:	List[Any]				)				->							Optional[int]:




     '''simple docstring'''

     return True




    def 							__magic_name__							(   self						:	List[str]				)				->							Tuple:




     '''simple docstring'''

     A__	=    LiltModelTester(self				)
     A__	=    ConfigTester(self     ,			config_class=snake_case_     ,			hidden_size=37				)




    def 							__magic_name__							(   self						:	Tuple				)				->							List[str]:




     '''simple docstring'''

     self.config_tester.run_common_tests()




    def 							__magic_name__							(   self						:	List[Any]				)				->							Union[str, Any]:




     '''simple docstring'''

     A__	=    self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_model(*snake_case_				)




    def 							__magic_name__							(   self						:	Union[str, Any]				)				->							List[str]:




     '''simple docstring'''

     A__	=    self.model_tester.prepare_config_and_inputs()
     for type in ["absolute", "relative_key", "relative_key_query"]:
      A__	=    type
      self.model_tester.create_and_check_model(*snake_case_				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_token_classification(*snake_case_				)




    def 							__magic_name__							(   self						:	int				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    self.model_tester.prepare_config_and_inputs()
     self.model_tester.create_and_check_for_question_answering(*snake_case_				)




    @slow
    def 							__magic_name__							(   self						:	Any				)				->							Optional[int]:




     '''simple docstring'''

     for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
      A__	=    LiltModel.from_pretrained(snake_case_				)
      self.assertIsNotNone(snake_case_				)






@require_torch
@slow
class      UpperCAmelCase_  (						unittest.TestCase							):




    def 							__magic_name__							(   self						:	Tuple				)				->							Any:




     '''simple docstring'''

     A__	=    LiltModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base"				).to(snake_case_				)

     A__	=    torch.tensor([[1, 2]]     ,			device=snake_case_				)
     A__	=    torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]     ,			device=snake_case_				)

     # forward pass
     with torch.no_grad():
      A__	=    model(input_ids=snake_case_     ,			bbox=snake_case_				)

     A__	=    torch.Size([1, 2, 768]				)
     A__	=    torch.tensor(
         [[-0.0653, 0.0950, -0.0061], [-0.0545, 0.0926, -0.0324]]     ,			device=snake_case_     ,			)

     self.assertTrue(outputs.last_hidden_state.shape     ,			snake_case_				)
     self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]     ,			snake_case_     ,			atol=1e-3				)				)











"""simple docstring"""



import argparse
import os

import transformers

from .convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS
from .utils import logging


logging.set_verbosity_info()

SCREAMING_SNAKE_CASE    =	logging.get_logger(__name__)


SCREAMING_SNAKE_CASE    =	{name: getattr(transformers, name + "Fast") for name in SLOW_TO_FAST_CONVERTERS}







def   _SCREAMING_SNAKE_CASE       (							lowercase_      ,			lowercase_      ,			lowercase_      ,			lowercase_						)						->		Any:
 if tokenizer_name is not None and tokenizer_name not in TOKENIZER_CLASSES:
  raise ValueError(f"""Unrecognized tokenizer name, should be one of {list(TOKENIZER_CLASSES.keys()						)}."""						)

 if tokenizer_name is None:
  A__	=    TOKENIZER_CLASSES
 else:
  A__	=    {tokenizer_name: getattr(lowercase_      ,			tokenizer_name + "Fast"						)}

 logger.info(f"""Loading tokenizer classes: {tokenizer_names}"""						)

 for tokenizer_name in tokenizer_names:
  A__	=    TOKENIZER_CLASSES[tokenizer_name]

  A__	=    True
  if checkpoint_name is None:
   A__	=    list(tokenizer_class.max_model_input_sizes.keys()						)
  else:
   A__	=    [checkpoint_name]

  logger.info(f"""For tokenizer {tokenizer_class.__class__.__name__} loading checkpoints: {checkpoint_names}"""						)

  for checkpoint in checkpoint_names:
   logger.info(f"""Loading {tokenizer_class.__class__.__name__} {checkpoint}"""						)

   # Load tokenizer
   A__	=    tokenizer_class.from_pretrained(lowercase_      ,			force_download=lowercase_						)

   # Save fast tokenizer
   logger.info(f"""Save fast tokenizer to {dump_path} with prefix {checkpoint} add_prefix {add_prefix}"""						)

   # For organization names we create sub-directories
   if "/" in checkpoint:
    A__,				A__	=    checkpoint.split("/"						)
    A__	=    os.path.join(lowercase_      ,			lowercase_						)
   elif add_prefix:
    A__	=    checkpoint
    A__	=    dump_path
   else:
    A__	=    None
    A__	=    dump_path

   logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""						)

   if checkpoint in list(tokenizer.pretrained_vocab_files_map.values()						)[0]:
    A__	=    list(tokenizer.pretrained_vocab_files_map.values()						)[0][checkpoint]
    A__	=    file_path.split(lowercase_						)[-1][0]
    if next_char == "/":
     A__	=    os.path.join(lowercase_      ,			lowercase_						)
     A__	=    None

    logger.info(f"""=> {dump_path_full} with prefix {checkpoint_prefix_name}, add_prefix {add_prefix}"""						)

   A__	=    tokenizer.save_pretrained(
       lowercase_      ,			legacy_format=lowercase_      ,			filename_prefix=lowercase_						)
   logger.info(f"""=> File names {file_names}"""						)

   for file_name in file_names:
    if not file_name.endswith("tokenizer.json"						):
     os.remove(lowercase_						)
     logger.info(f"""=> removing {file_name}"""						)


if __name__ == "__main__":
      SCREAMING_SNAKE_CASE    =	argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          "--dump_path", default=None, type=str, required=True, help="Path to output generated fast tokenizer files."
      )
      parser.add_argument(
          "--tokenizer_name",
          default=None,
          type=str,
          help=(
              f'Optional tokenizer type selected in the list of {list(TOKENIZER_CLASSES.keys())}. If not given, will '
              "download and convert all the checkpoints from AWS."
          ),
      )
      parser.add_argument(
          "--checkpoint_name",
          default=None,
          type=str,
          help="Optional checkpoint name. If not given, will download and convert the canonical checkpoints from AWS.",
      )
      parser.add_argument(
          "--force_download",
          action="store_true",
          help="Re-download checkpoints.",
      )
      SCREAMING_SNAKE_CASE    =	parser.parse_args()

      convert_slow_checkpoint_to_fast(args.tokenizer_name, args.checkpoint_name, args.dump_path, args.force_download)





