




"""simple docstring"""



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english




@require_tokenizers
class      UpperCAmelCase_  (						A_,       unittest.TestCase							):
    lowercase__						 =				BertTokenizer
    lowercase__						 =				BertTokenizerFast
    lowercase__						 =				True
    lowercase__						 =				True
    lowercase__						 =				filter_non_english




    def 							__magic_name__							(   self						:	List[Any]				)				->							Dict:




     '''simple docstring'''

     super().setUp()

     A__	=    [
         "[UNK]",
         "[CLS]",
         "[SEP]",
         "[PAD]",
         "[MASK]",
         "want",
         "##want",
         "##ed",
         "wa",
         "un",
         "runn",
         "##ing",
         ",",
         "low",
         "lowest",
     ]
     A__	=    os.path.join(self.tmpdirname     ,			VOCAB_FILES_NAMES["vocab_file"]				)
     with open(self.vocab_file     ,			"w"     ,			encoding="utf-8"				) as vocab_writer:
      vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)




    def 							__magic_name__							(   self						:	Any     ,			snake_case_						:	List[str]				)				->							Union[str, Any]:




     '''simple docstring'''

     A__	=    "UNwant\u00E9d,running"
     A__	=    "unwanted, running"
     return input_text, output_text




    def 							__magic_name__							(   self						:	str				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    self.tokenizer_class(self.vocab_file				)

     A__	=    tokenizer.tokenize("UNwant\u00E9d,running"				)
     self.assertListEqual(snake_case_     ,			["un", "##want", "##ed", ",", "runn", "##ing"]				)
     self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_				)     ,			[9, 6, 7, 12, 10, 11]				)




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							str:




     '''simple docstring'''

     if not self.test_rust_tokenizer:
      return

     A__	=    self.get_tokenizer()
     A__	=    self.get_rust_tokenizer()

     A__	=    "UNwant\u00E9d,running"

     A__	=    tokenizer.tokenize(snake_case_				)
     A__	=    rust_tokenizer.tokenize(snake_case_				)
     self.assertListEqual(snake_case_     ,			snake_case_				)

     A__	=    tokenizer.encode(snake_case_     ,			add_special_tokens=snake_case_				)
     A__	=    rust_tokenizer.encode(snake_case_     ,			add_special_tokens=snake_case_				)
     self.assertListEqual(snake_case_     ,			snake_case_				)

     A__	=    self.get_rust_tokenizer()
     A__	=    tokenizer.encode(snake_case_				)
     A__	=    rust_tokenizer.encode(snake_case_				)
     self.assertListEqual(snake_case_     ,			snake_case_				)

     # With lower casing
     A__	=    self.get_tokenizer(do_lower_case=snake_case_				)
     A__	=    self.get_rust_tokenizer(do_lower_case=snake_case_				)

     A__	=    "UNwant\u00E9d,running"

     A__	=    tokenizer.tokenize(snake_case_				)
     A__	=    rust_tokenizer.tokenize(snake_case_				)
     self.assertListEqual(snake_case_     ,			snake_case_				)

     A__	=    tokenizer.encode(snake_case_     ,			add_special_tokens=snake_case_				)
     A__	=    rust_tokenizer.encode(snake_case_     ,			add_special_tokens=snake_case_				)
     self.assertListEqual(snake_case_     ,			snake_case_				)

     A__	=    self.get_rust_tokenizer()
     A__	=    tokenizer.encode(snake_case_				)
     A__	=    rust_tokenizer.encode(snake_case_				)
     self.assertListEqual(snake_case_     ,			snake_case_				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    BasicTokenizer()

     self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"				)     ,			["ah", "\u535A", "\u63A8", "zz"]				)




    def 							__magic_name__							(   self						:	int				)				->							str:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)     ,			["hello", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["hello"]				)




    def 							__magic_name__							(   self						:	Dict				)				->							List[Any]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_     ,			strip_accents=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["hällo", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["h\u00E9llo"]				)




    def 							__magic_name__							(   self						:	Optional[int]				)				->							int:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_     ,			strip_accents=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["hallo", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["hello"]				)




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							Union[str, Any]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["hallo", "!", "how", "are", "you", "?"]				)
     self.assertListEqual(tokenizer.tokenize("H\u00E9llo"				)     ,			["hello"]				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							List[str]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "				)     ,			["HeLLo", "!", "how", "Are", "yoU", "?"]				)




    def 							__magic_name__							(   self						:	Union[str, Any]				)				->							Dict:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_     ,			strip_accents=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["HäLLo", "!", "how", "Are", "yoU", "?"]				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							Optional[int]:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_     ,			strip_accents=snake_case_				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "				)     ,			["HaLLo", "!", "how", "Are", "yoU", "?"]				)




    def 							__magic_name__							(   self						:	Optional[int]				)				->							Dict:




     '''simple docstring'''

     A__	=    BasicTokenizer(do_lower_case=snake_case_     ,			never_split=["[UNK]"]				)

     self.assertListEqual(
         tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"				)     ,			["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]				)




    def 							__magic_name__							(   self						:	int				)				->							Optional[Any]:




     '''simple docstring'''

     A__	=    BasicTokenizer()
     A__	=    "a\n'll !!to?'d of, can't."
     A__	=    ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
     self.assertListEqual(tokenizer.tokenize(snake_case_				)     ,			snake_case_				)




    def 							__magic_name__							(   self						:	Dict				)				->							Union[str, Any]:




     '''simple docstring'''

     A__	=    ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

     A__	=    {}
     for i, token in enumerate(snake_case_				):
      A__	=    i
     A__	=    WordpieceTokenizer(vocab=snake_case_     ,			unk_token="[UNK]"				)

     self.assertListEqual(tokenizer.tokenize(""				)     ,			[]				)

     self.assertListEqual(tokenizer.tokenize("unwanted running"				)     ,			["un", "##want", "##ed", "runn", "##ing"]				)

     self.assertListEqual(tokenizer.tokenize("unwantedX running"				)     ,			["[UNK]", "runn", "##ing"]				)




    def 							__magic_name__							(   self						:	List[Any]				)				->							Tuple:




     '''simple docstring'''

     self.assertTrue(_is_whitespace(" "				)				)
     self.assertTrue(_is_whitespace("\t"				)				)
     self.assertTrue(_is_whitespace("\r"				)				)
     self.assertTrue(_is_whitespace("\n"				)				)
     self.assertTrue(_is_whitespace("\u00A0"				)				)

     self.assertFalse(_is_whitespace("A"				)				)
     self.assertFalse(_is_whitespace("-"				)				)




    def 							__magic_name__							(   self						:	Tuple				)				->							str:




     '''simple docstring'''

     self.assertTrue(_is_control("\u0005"				)				)

     self.assertFalse(_is_control("A"				)				)
     self.assertFalse(_is_control(" "				)				)
     self.assertFalse(_is_control("\t"				)				)
     self.assertFalse(_is_control("\r"				)				)




    def 							__magic_name__							(   self						:	List[str]				)				->							Optional[int]:




     '''simple docstring'''

     self.assertTrue(_is_punctuation("-"				)				)
     self.assertTrue(_is_punctuation("$"				)				)
     self.assertTrue(_is_punctuation("`"				)				)
     self.assertTrue(_is_punctuation("."				)				)

     self.assertFalse(_is_punctuation("A"				)				)
     self.assertFalse(_is_punctuation(" "				)				)




    def 							__magic_name__							(   self						:	str				)				->							Tuple:




     '''simple docstring'''

     A__	=    self.get_tokenizer()
     A__	=    self.get_rust_tokenizer()

     # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
     self.assertListEqual([tokenizer.tokenize(snake_case_				) for t in ["Test", "\xad", "test"]]     ,			[["[UNK]"], [], ["[UNK]"]]				)

     self.assertListEqual(
         [rust_tokenizer.tokenize(snake_case_				) for t in ["Test", "\xad", "test"]]     ,			[["[UNK]"], [], ["[UNK]"]]				)




    @slow
    def 							__magic_name__							(   self						:	Any				)				->							int:




     '''simple docstring'''

     A__	=    self.tokenizer_class.from_pretrained("bert-base-uncased"				)

     A__	=    tokenizer.encode("sequence builders"     ,			add_special_tokens=snake_case_				)
     A__	=    tokenizer.encode("multi-sequence build"     ,			add_special_tokens=snake_case_				)

     A__	=    tokenizer.build_inputs_with_special_tokens(snake_case_				)
     A__	=    tokenizer.build_inputs_with_special_tokens(snake_case_     ,			snake_case_				)

     assert encoded_sentence == [101] + text + [102]
     assert encoded_pair == [101] + text + [102] + text_a + [102]




    def 							__magic_name__							(   self						:	str				)				->							str:




     '''simple docstring'''

     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
       A__	=    self.rust_tokenizer_class.from_pretrained(snake_case_     ,			**snake_case_				)

       A__	=    F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
       A__	=    tokenizer_r.encode_plus(
           snake_case_     ,			return_attention_mask=snake_case_     ,			return_token_type_ids=snake_case_     ,			return_offsets_mapping=snake_case_     ,			add_special_tokens=snake_case_     ,			)

       A__	=    tokenizer_r.do_lower_case if hasattr(snake_case_     ,			"do_lower_case"				) else False
       A__	=    (
           [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "A"),
               ((1, 2), ","),
               ((3, 5), "na"),
               ((5, 6), "##ï"),
               ((6, 8), "##ve"),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), "Allen"),
               ((21, 23), "##NL"),
               ((23, 24), "##P"),
               ((25, 33), "sentence"),
               ((33, 34), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
           if not do_lower_case
           else [
               ((0, 0), tokenizer_r.cls_token),
               ((0, 1), "a"),
               ((1, 2), ","),
               ((3, 8), "naive"),
               ((9, 15), tokenizer_r.mask_token),
               ((16, 21), "allen"),
               ((21, 23), "##nl"),
               ((23, 24), "##p"),
               ((25, 33), "sentence"),
               ((33, 34), "."),
               ((0, 0), tokenizer_r.sep_token),
           ]
       )

       self.assertEqual(
           [e[1] for e in expected_results]     ,			tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]				)				)
       self.assertEqual([e[0] for e in expected_results]     ,			tokens["offset_mapping"]				)




    def 							__magic_name__							(   self						:	Optional[Any]				)				->							int:




     '''simple docstring'''

     A__	=    ["的", "人", "有"]
     A__	=    "".join(snake_case_				)
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""				):
       A__	=    True
       A__	=    self.tokenizer_class.from_pretrained(snake_case_     ,			**snake_case_				)
       A__	=    self.rust_tokenizer_class.from_pretrained(snake_case_     ,			**snake_case_				)

       A__	=    tokenizer_p.encode(snake_case_     ,			add_special_tokens=snake_case_				)
       A__	=    tokenizer_r.encode(snake_case_     ,			add_special_tokens=snake_case_				)

       A__	=    tokenizer_r.convert_ids_to_tokens(snake_case_				)
       A__	=    tokenizer_p.convert_ids_to_tokens(snake_case_				)

       # it is expected that each Chinese character is not preceded by "##"
       self.assertListEqual(snake_case_     ,			snake_case_				)
       self.assertListEqual(snake_case_     ,			snake_case_				)

       A__	=    False
       A__	=    self.rust_tokenizer_class.from_pretrained(snake_case_     ,			**snake_case_				)
       A__	=    self.tokenizer_class.from_pretrained(snake_case_     ,			**snake_case_				)

       A__	=    tokenizer_r.encode(snake_case_     ,			add_special_tokens=snake_case_				)
       A__	=    tokenizer_p.encode(snake_case_     ,			add_special_tokens=snake_case_				)

       A__	=    tokenizer_r.convert_ids_to_tokens(snake_case_				)
       A__	=    tokenizer_p.convert_ids_to_tokens(snake_case_				)

       # it is expected that only the first Chinese character is not preceded by "##".
       A__	=    [
           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(snake_case_				)
       ]
       self.assertListEqual(snake_case_     ,			snake_case_				)
       self.assertListEqual(snake_case_     ,			snake_case_				)











"""simple docstring"""



from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class      UpperCAmelCase_  (						A_							):
    lowercase__						 =				['''image_processor''', '''tokenizer''']
    lowercase__						 =				'''AutoImageProcessor'''
    lowercase__						 =				'''AutoTokenizer'''




    def __init__(   self						:	str     ,			snake_case_						:	Dict     ,			snake_case_						:	List[str]				)				->							str:




     '''simple docstring'''

     super().__init__(snake_case_     ,			snake_case_				)
     A__	=    self.image_processor




    def __call__(   self						:	int     ,			snake_case_						:	Any=None     ,			snake_case_						:	Any=None     ,			snake_case_						:	Union[str, Any]=None     ,			**snake_case_						:	Optional[int]				)				->							Optional[int]:




     '''simple docstring'''

     if text is None and images is None:
      raise ValueError("You have to specify either text or images. Both cannot be none."				)

     if text is not None:
      A__	=    self.tokenizer(snake_case_     ,			return_tensors=snake_case_     ,			**snake_case_				)

     if images is not None:
      A__	=    self.image_processor(snake_case_     ,			return_tensors=snake_case_     ,			**snake_case_				)

     if text is not None and images is not None:
      A__	=    image_features.pixel_values
      return encoding
     elif text is not None:
      return encoding
     else:
      return BatchEncoding(data=dict(**snake_case_				)     ,			tensor_type=snake_case_				)




    def 							__magic_name__							(   self						:	Optional[int]     ,			*snake_case_						:	Union[str, Any]     ,			**snake_case_						:	List[Any]				)				->							int:




     '''simple docstring'''

     return self.tokenizer.batch_decode(*snake_case_     ,			**snake_case_				)




    def 							__magic_name__							(   self						:	List[str]     ,			*snake_case_						:	List[str]     ,			**snake_case_						:	Optional[int]				)				->							Tuple:




     '''simple docstring'''

     return self.tokenizer.decode(*snake_case_     ,			**snake_case_				)




    @property
    def 							__magic_name__							(   self						:	List[Any]				)				->							List[Any]:




     '''simple docstring'''

     return ["input_ids", "attention_mask", "pixel_values"]





