


'''simple docstring'''






import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 		a__    (							a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       Optional[int]     						=				GPTaTokenizer
							lowercase__      :       int     						=				GPTaTokenizerFast
							lowercase__      :       str     						=				True
							lowercase__      :       int     						=				{"add_prefix_space": True}
							lowercase__      :       int     						=				False







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							int:
														super().setUp()

														# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
														lowerCAmelCase__	       =     [
														    '''l''',
														    '''o''',
														    '''w''',
														    '''e''',
														    '''r''',
														    '''s''',
														    '''t''',
														    '''i''',
														    '''d''',
														    '''n''',
														    '''\u0120''',
														    '''\u0120l''',
														    '''\u0120n''',
														    '''\u0120lo''',
														    '''\u0120low''',
														    '''er''',
														    '''\u0120lowest''',
														    '''\u0120newer''',
														    '''\u0120wider''',
														    '''<unk>''',
														    '''<|endoftext|>''',
														]
														lowerCAmelCase__	       =     dict(zip(lowerCamelCase_ ,   range(len(lowerCamelCase_       )       )       )       )
														lowerCAmelCase__	       =     ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
														lowerCAmelCase__	       =     {'''unk_token''': '''<unk>'''}

														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''vocab_file''']       )
														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''merges_file''']       )
														with open(self.vocab_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write(json.dumps(lowerCamelCase_       ) + '''\n'''       )
														with open(self.merges_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write('''\n'''.join(lowerCamelCase_       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							List[str]:
														kwargs.update(self.special_tokens_map       )
														return GPTaTokenizer.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							Optional[int]:
														kwargs.update(self.special_tokens_map       )
														return GPTaTokenizerFast.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							int:
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     '''lower newer'''
														return input_text, output_text







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														lowerCAmelCase__	       =     GPTaTokenizer(self.vocab_file ,   self.merges_file ,   **self.special_tokens_map       )
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     ['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokens + [tokenizer.unk_token]
														lowerCAmelCase__	       =     [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														if not self.test_rust_tokenizer:
																					return

														lowerCAmelCase__	       =     self.get_tokenizer()
														lowerCAmelCase__	       =     self.get_rust_tokenizer(add_prefix_space=lowerCamelCase_       )

														lowerCAmelCase__	       =     '''lower newer'''

														# Testing tokenization
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# Testing conversion to ids without special tokens
														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# Testing conversion to ids with special tokens
														lowerCAmelCase__	       =     self.get_rust_tokenizer(add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_prefix_space=lowerCamelCase_       )
														lowerCAmelCase__	       =     rust_tokenizer.encode(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														# Testing the unknown token
														lowerCAmelCase__	       =     tokens + [rust_tokenizer.unk_token]
														lowerCAmelCase__	       =     [14, 15, 10, 9, 3, 2, 15, 19]
														self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   *lowerCamelCase_ ,   **lowerCamelCase_       )	->							int:
														# It's very difficult to mix/test pretokenization with byte-level
														# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
														pass







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_=15       )	->							str:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												# Simple input
																												lowerCAmelCase__	       =     '''This is a simple input'''
																												lowerCAmelCase__	       =     ['''This is a simple input 1''', '''This is a simple input 2''']
																												lowerCAmelCase__	       =     ('''This is a simple input''', '''This is a pair''')
																												lowerCAmelCase__	       =     [
																												    ('''This is a simple input 1''', '''This is a simple input 2'''),
																												    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																												]

																												# Simple input tests
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Simple input
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Simple input
																												self.assertRaises(
																												    lowerCamelCase_ ,   tokenizer_r.batch_encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length''' ,   )

																												# Pair input
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Pair input
																												self.assertRaises(lowerCamelCase_ ,   tokenizer_r.encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length'''       )

																												# Pair input
																												self.assertRaises(
																												    lowerCamelCase_ ,   tokenizer_r.batch_encode_plus ,   lowerCamelCase_ ,   max_length=lowerCamelCase_ ,   padding='''max_length''' ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														lowerCAmelCase__	       =     GPTaTokenizer.from_pretrained(self.tmpdirname ,   pad_token='''<pad>'''       )

														# Simple input
														lowerCAmelCase__	       =     '''This is a simple input'''
														lowerCAmelCase__	       =     ['''This is a simple input looooooooong''', '''This is a simple input''']
														lowerCAmelCase__	       =     ('''This is a simple input''', '''This is a pair''')
														lowerCAmelCase__	       =     [
														    ('''This is a simple input loooooong''', '''This is a simple input'''),
														    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
														]

														lowerCAmelCase__	       =     tokenizer.pad_token_id

														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   padding='''max_length''' ,   max_length=30 ,   return_tensors='''np'''       )
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   padding=lowerCamelCase_ ,   truncate=lowerCamelCase_ ,   return_tensors='''np'''       )
														lowerCAmelCase__	       =     tokenizer(*lowerCamelCase_ ,   padding='''max_length''' ,   max_length=60 ,   return_tensors='''np'''       )
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_ ,   padding=lowerCamelCase_ ,   truncate=lowerCamelCase_ ,   return_tensors='''np'''       )

														# s
														# test single string max_length padding
														self.assertEqual(out_s['''input_ids'''].shape[-1] ,   30       )
														self.assertTrue(pad_token_id in out_s['''input_ids''']       )
														self.assertTrue(0 in out_s['''attention_mask''']       )

														# s2
														# test automatic padding
														self.assertEqual(out_sa['''input_ids'''].shape[-1] ,   33       )
														# long slice doesn't have padding
														self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]       )
														self.assertFalse(0 in out_sa['''attention_mask'''][0]       )
														# short slice does have padding
														self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]       )
														self.assertTrue(0 in out_sa['''attention_mask'''][1]       )

														# p
														# test single pair max_length padding
														self.assertEqual(out_p['''input_ids'''].shape[-1] ,   60       )
														self.assertTrue(pad_token_id in out_p['''input_ids''']       )
														self.assertTrue(0 in out_p['''attention_mask''']       )

														# p2
														# test automatic padding pair
														self.assertEqual(out_pa['''input_ids'''].shape[-1] ,   52       )
														# long slice pair doesn't have padding
														self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]       )
														self.assertFalse(0 in out_pa['''attention_mask'''][0]       )
														# short slice pair does have padding
														self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]       )
														self.assertTrue(0 in out_pa['''attention_mask'''][1]       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														lowerCAmelCase__	       =     '''$$$'''
														lowerCAmelCase__	       =     GPTaTokenizer.from_pretrained(self.tmpdirname ,   bos_token=lowerCamelCase_ ,   add_bos_token=lowerCamelCase_       )

														lowerCAmelCase__	       =     '''This is a simple input'''
														lowerCAmelCase__	       =     ['''This is a simple input 1''', '''This is a simple input 2''']

														lowerCAmelCase__	       =     tokenizer.bos_token_id

														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_       )
														lowerCAmelCase__	       =     tokenizer(lowerCamelCase_       )

														self.assertEqual(out_s.input_ids[0] ,   lowerCamelCase_       )
														self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

														lowerCAmelCase__	       =     tokenizer.decode(out_s.input_ids       )
														lowerCAmelCase__	       =     tokenizer.batch_decode(out_sa.input_ids       )

														self.assertEqual(decode_s.split()[0] ,   lowerCamelCase_       )
														self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														pass







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														# TODO: change to self.get_tokenizers() when the fast version is implemented
														lowerCAmelCase__	       =     [self.get_tokenizer(do_lower_case=lowerCamelCase_ ,   add_bos_token=lowerCamelCase_       )]
														for tokenizer in tokenizers:
																					with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
																												lowerCAmelCase__	       =     '''Encode this.'''
																												lowerCAmelCase__	       =     '''This one too please.'''
																												lowerCAmelCase__	       =     tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												encoded_sequence += tokenizer.encode(lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer.encode_plus(
																												    lowerCamelCase_ ,   lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_ ,   return_special_tokens_mask=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     encoded_sequence_dict['''input_ids''']
																												lowerCAmelCase__	       =     encoded_sequence_dict['''special_tokens_mask''']
																												self.assertEqual(len(lowerCamelCase_       ) ,   len(lowerCamelCase_       )       )

																												lowerCAmelCase__	       =     [
																												    (x if not special_tokens_mask[i] else None) for i, x in enumerate(lowerCamelCase_       )
																												]
																												lowerCAmelCase__	       =     [x for x in filtered_sequence if x is not None]
																												self.assertEqual(lowerCamelCase_ ,   lowerCamelCase_       )




@require_tokenizers
class 		a__    (							unittest.TestCase  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														# More context:
														# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
														# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
														# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

														lowerCAmelCase__	       =     AutoTokenizer.from_pretrained('''facebook/opt-350m''' ,   from_slow=lowerCamelCase_       )
														lowerCAmelCase__	       =     '''A photo of a cat'''

														lowerCAmelCase__	       =     tokenizer.encode(
														    lowerCamelCase_ ,   )
														self.assertEqual(lowerCamelCase_ ,   [2, 2_50, 13_45, 9, 10, 47_58]       )
														tokenizer.save_pretrained('''test_opt'''       )

														lowerCAmelCase__	       =     AutoTokenizer.from_pretrained('''./test_opt'''       )
														lowerCAmelCase__	       =     tokenizer.encode(
														    lowerCamelCase_ ,   )
														self.assertEqual(lowerCamelCase_ ,   [2, 2_50, 13_45, 9, 10, 47_58]       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[int]:
														lowerCAmelCase__	       =     AutoTokenizer.from_pretrained('''facebook/opt-350m''' ,   use_slow=lowerCamelCase_       )
														lowerCAmelCase__	       =     '''A photo of a cat'''

														lowerCAmelCase__	       =     tokenizer.encode(
														    lowerCamelCase_ ,   )
														# Same as above
														self.assertEqual(lowerCamelCase_ ,   [2, 2_50, 13_45, 9, 10, 47_58]       )







							@unittest.skip('''This test is failing because of a bug in the fast tokenizer'''       )
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     AutoTokenizer.from_pretrained('''facebook/opt-350m''' ,   from_slow=lowerCamelCase_       )

														lowerCAmelCase__	       =     '''bos'''
														lowerCAmelCase__	       =     tokenizer.get_vocab()['''bos''']

														lowerCAmelCase__	       =     '''A photo of a cat'''
														lowerCAmelCase__	       =     tokenizer.encode(
														    lowerCamelCase_ ,   )
														# We changed the bos token
														self.assertEqual(lowerCamelCase_ ,   [3_19_57, 2_50, 13_45, 9, 10, 47_58]       )
														tokenizer.save_pretrained('''./tok'''       )
														lowerCAmelCase__	       =     AutoTokenizer.from_pretrained('''./tok'''       )
														self.assertTrue(tokenizer.is_fast       )
														lowerCAmelCase__	       =     tokenizer.encode(
														    lowerCamelCase_ ,   )
														self.assertEqual(lowerCamelCase_ ,   [3_19_57, 2_50, 13_45, 9, 10, 47_58]       )



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


__UpperCAmelCase       =							logging.get_logger(__name__)

__UpperCAmelCase       =							{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCAmelCase       =							{
    '''vocab_file''': {
        '''junnyu/roformer_chinese_small''': '''https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_base''': '''https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt''',
        '''junnyu/roformer_chinese_char_small''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_chinese_char_base''': (
            '''https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_discriminator''': (
            '''https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'''
        ),
        '''junnyu/roformer_small_generator''': (
            '''https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'''
        ),
    }
}

__UpperCAmelCase       =							{
    '''junnyu/roformer_chinese_small''': 1_536,
    '''junnyu/roformer_chinese_base''': 1_536,
    '''junnyu/roformer_chinese_char_small''': 512,
    '''junnyu/roformer_chinese_char_base''': 512,
    '''junnyu/roformer_small_discriminator''': 128,
    '''junnyu/roformer_small_generator''': 128,
}


__UpperCAmelCase       =							{
    '''junnyu/roformer_chinese_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_small''': {'''do_lower_case''': True},
    '''junnyu/roformer_chinese_char_base''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_discriminator''': {'''do_lower_case''': True},
    '''junnyu/roformer_small_generator''': {'''do_lower_case''': True},
}




class 		a__    (							a__  ):





							'''simple docstring'''






							lowercase__      :       int     						=				VOCAB_FILES_NAMES
							lowercase__      :       Optional[int]     						=				PRETRAINED_VOCAB_FILES_MAP
							lowercase__      :       Any     						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__      :       Union[str, Any]     						=				PRETRAINED_INIT_CONFIGURATION
							lowercase__      :       Tuple     						=				RoFormerTokenizer







							def __init__(		self ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   lowerCamelCase_=True ,   lowerCamelCase_="[UNK]" ,   lowerCamelCase_="[SEP]" ,   lowerCamelCase_="[PAD]" ,   lowerCamelCase_="[CLS]" ,   lowerCamelCase_="[MASK]" ,   lowerCamelCase_=True ,   lowerCamelCase_=None ,   **lowerCamelCase_ ,   )	->							Tuple:
														super().__init__(
														    lowerCamelCase_ ,   tokenizer_file=lowerCamelCase_ ,   do_lower_case=lowerCamelCase_ ,   unk_token=lowerCamelCase_ ,   sep_token=lowerCamelCase_ ,   pad_token=lowerCamelCase_ ,   cls_token=lowerCamelCase_ ,   mask_token=lowerCamelCase_ ,   tokenize_chinese_chars=lowerCamelCase_ ,   strip_accents=lowerCamelCase_ ,   **lowerCamelCase_ ,   )

														lowerCAmelCase__	       =     json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
														if (
														    pre_tok_state.get('''lowercase''' ,   lowerCamelCase_       ) != do_lower_case
														    or pre_tok_state.get('''strip_accents''' ,   lowerCamelCase_       ) != strip_accents
														):
																					lowerCAmelCase__	       =     getattr(lowerCamelCase_ ,   pre_tok_state.pop('''type'''       )       )
																					lowerCAmelCase__	       =     do_lower_case
																					lowerCAmelCase__	       =     strip_accents
																					lowerCAmelCase__	       =     pre_tok_class(**lowerCamelCase_       )

														lowerCAmelCase__	       =     do_lower_case







							def __getstate__(		self       )	->							Any:
														lowerCAmelCase__	       =     self.__dict__.copy()
														lowerCAmelCase__	       =     BertPreTokenizer()
														return state







							def __setstate__(		self ,   lowerCamelCase_       )	->							List[Any]:
														lowerCAmelCase__	       =     d
														lowerCAmelCase__	       =     self.__dict__['''_tokenizer'''].get_vocab()
														lowerCAmelCase__	       =     PreTokenizer.custom(JiebaPreTokenizer(lowerCamelCase_       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_=None       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

														if token_ids_a:
																					output += token_ids_a + [self.sep_token_id]

														return output







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ = None       )	->							List[int]:
														lowerCAmelCase__	       =     [self.sep_token_id]
														lowerCAmelCase__	       =     [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep       ) * [0]
														return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ = None       )	->							Tuple[str]:
														lowerCAmelCase__	       =     self._tokenizer.model.save(lowerCamelCase_ ,   name=lowerCamelCase_       )
														return tuple(lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_=None ,   lowerCamelCase_=None ,   lowerCamelCase_=False ,   **lowerCamelCase_ ,   )	->							Union[str, Any]:
														lowerCAmelCase__	       =     BertPreTokenizer()
														return super().save_pretrained(lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   **lowerCamelCase_       )