




import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class     UpperCamelCase_   (			UpperCAmelCase__		, unittest.TestCase					):

    '''simple docstring'''






    UpperCAmelCase__							  =				CLIPTokenizer
    UpperCAmelCase__							  =				CLIPTokenizerFast
    UpperCAmelCase__							  =				True
    UpperCAmelCase__							  =				{}
    UpperCAmelCase__							  =				False


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[str])						->Tuple:





           '''simple docstring'''

           super().setUp()

           # fmt: off
           A__							=   ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
           # fmt: on
           A__							=   dict(zip(UpperCAmelCase__ ,			range(len(UpperCAmelCase__))))
           A__							=   ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
           A__							=   {'''unk_token''': '''<unk>'''}

           A__							=   os.path.join(self.tmpdirname ,			VOCAB_FILES_NAMES['''vocab_file'''])
           A__							=   os.path.join(self.tmpdirname ,			VOCAB_FILES_NAMES['''merges_file'''])
           with open(self.vocab_file ,			'''w''' ,			encoding='''utf-8''') as fp:
                  fp.write(json.dumps(UpperCAmelCase__) + '''\n''')
           with open(self.merges_file ,			'''w''' ,			encoding='''utf-8''') as fp:
                  fp.write('''\n'''.join(UpperCAmelCase__))


    def 	SCREAMING_SNAKE_CASE							(      self				:     int ,			**UpperCAmelCase__				:     Any)						->Any:





           '''simple docstring'''

           kwargs.update(self.special_tokens_map)
           return CLIPTokenizer.from_pretrained(self.tmpdirname ,			**UpperCAmelCase__)


    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[Any] ,			**UpperCAmelCase__				:     Union[str, Any])						->Union[str, Any]:





           '''simple docstring'''

           kwargs.update(self.special_tokens_map)
           return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,			**UpperCAmelCase__)


    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any] ,			UpperCAmelCase__				:     Optional[int])						->int:





           '''simple docstring'''

           A__							=   '''lower newer'''
           A__							=   '''lower newer'''
           return input_text, output_text


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[Any])						->Any:





           '''simple docstring'''

           A__							=   CLIPTokenizer(self.vocab_file ,			self.merges_file ,			**self.special_tokens_map)
           A__							=   '''lower newer'''
           A__							=   ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
           A__							=   tokenizer.tokenize(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__ ,			UpperCAmelCase__)

           A__							=   tokens + [tokenizer.unk_token]
           A__							=   [10, 2, 16, 9, 3, 2, 16, 20]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__) ,			UpperCAmelCase__)


    @require_ftfy
    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[int])						->int:





           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                         A__							=   self.tokenizer_class.from_pretrained(UpperCAmelCase__ ,			**UpperCAmelCase__)
                         A__							=   self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__ ,			**UpperCAmelCase__)

                         A__							=   '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
                         A__							=   tokenizer_s.tokenize(UpperCAmelCase__)
                         A__							=   tokenizer_r.tokenize(UpperCAmelCase__)

                         self.assertListEqual(UpperCAmelCase__ ,			UpperCAmelCase__)

                         # Test that the tokenization is identical on an example containing a character (Latin Small Letter A
                         # with Tilde) encoded in 2 different ways
                         A__							=   '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
                         A__							=   tokenizer_s.tokenize(UpperCAmelCase__)
                         A__							=   tokenizer_r.tokenize(UpperCAmelCase__)

                         self.assertListEqual(UpperCAmelCase__ ,			UpperCAmelCase__)

                         # Test that the tokenization is identical on unicode of space type
                         A__							=   [
                             '''\u0009''',  # (horizontal tab, '\t')
                             '''\u000B''',  # (vertical tab)
                             '''\u000C''',  # (form feed)
                             '''\u0020''',  # (space, ' ')
                             '''\u200E''',  # (left-to-right mark):w
                             '''\u200F''',  # (right-to-left mark)
                         ]
                         for unicode_seq in spaces_unicodes:
                                A__							=   tokenizer_s.tokenize(UpperCAmelCase__)
                                A__							=   tokenizer_r.tokenize(UpperCAmelCase__)

                                self.assertListEqual(UpperCAmelCase__ ,			UpperCAmelCase__)

                         # Test that the tokenization is identical on unicode of line break type
                         A__							=   [
                             '''\u000A''',  # (line feed, '\n')
                             '''\r\n''',  # (carriage return and line feed, '\r\n')
                             '''\u000D''',  # (carriage return, '\r')
                             '''\r''',  # (carriage return, '\r')
                             '''\u000D''',  # (carriage return, '\r')
                             '''\u2028''',  # (line separator)
                             '''\u2029''',  # (paragraph separator)
                             # "\u0085", # (next line)
                         ]

                         # The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
                         # it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
                         # space (and thus into an empty list).

                         for unicode_seq in line_break_unicodes:
                                A__							=   tokenizer_s.tokenize(UpperCAmelCase__)
                                A__							=   tokenizer_r.tokenize(UpperCAmelCase__)

                                self.assertListEqual(UpperCAmelCase__ ,			UpperCAmelCase__)


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[Any])						->int:





           '''simple docstring'''

           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
                         A__							=   '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
                         A__							=   f"""{text_of_1_token} {text_of_1_token}"""

                         A__							=   self.rust_tokenizer_class.from_pretrained(
                             UpperCAmelCase__ ,			use_fast=UpperCAmelCase__ ,			)
                         A__							=   tokenizer_r(UpperCAmelCase__ ,			return_offsets_mapping=UpperCAmelCase__ ,			add_special_tokens=UpperCAmelCase__)
                         self.assertEqual(encoding.offset_mapping[0] ,			(0, len(UpperCAmelCase__)))
                         self.assertEqual(
                             encoding.offset_mapping[1] ,			(len(UpperCAmelCase__) + 1, len(UpperCAmelCase__) + 1 + len(UpperCAmelCase__)) ,			)

                         A__							=   f""" {text}"""

                         A__							=   self.rust_tokenizer_class.from_pretrained(
                             UpperCAmelCase__ ,			use_fast=UpperCAmelCase__ ,			)
                         A__							=   tokenizer_r(UpperCAmelCase__ ,			return_offsets_mapping=UpperCAmelCase__ ,			add_special_tokens=UpperCAmelCase__)
                         self.assertEqual(encoding.offset_mapping[0] ,			(1, 1 + len(UpperCAmelCase__)))
                         self.assertEqual(
                             encoding.offset_mapping[1] ,			(1 + len(UpperCAmelCase__) + 1, 1 + len(UpperCAmelCase__) + 1 + len(UpperCAmelCase__)) ,			)


    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any])						->Any:





           '''simple docstring'''

           with self.assertRaises(UpperCAmelCase__) as context:
                  self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer''')

           self.assertTrue(
               context.exception.args[0].startswith(
                   '''The `backend_tokenizer` provided does not match the expected format.'''))


    @require_ftfy
    def 	SCREAMING_SNAKE_CASE							(      self				:     List[str])						->Any:





           '''simple docstring'''

           super().test_tokenization_python_rust_equals()





    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[Any])						->Dict:





           '''simple docstring'''

           pass








class     UpperCamelCase_   :

    '''simple docstring'''






    def __init__(      self				:     List[str] ,			UpperCAmelCase__				:     int)						->Union[str, Any]:





           '''simple docstring'''

           A__							=   n
           A__							=   [None] * self.n
           A__							=   0  # index of the first element
           A__							=   0
           A__							=   0


    def __len__(      self				:     List[Any])						->int:





           '''simple docstring'''

           return self.size


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[Any])						->bool:





           '''simple docstring'''

           return self.size == 0


    def 	SCREAMING_SNAKE_CASE							(      self				:     str)						->List[Any]:





           '''simple docstring'''

           return False if self.is_empty() else self.array[self.front]


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[str] ,			UpperCAmelCase__				:     str)						->Any:





           '''simple docstring'''

           if self.size >= self.n:
                  raise Exception('''QUEUE IS FULL''')

           A__							=   data
           A__							=   (self.rear + 1) % self.n
           self.size += 1
           return self





    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any])						->Any:





           '''simple docstring'''

           if self.size == 0:
                  raise Exception('''UNDERFLOW''')

           A__							=   self.array[self.front]
           A__							=   None
           A__							=   (self.front + 1) % self.n
           self.size -= 1
           return temp


