


import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 		__a				(    UpperCAmelCase   ,				unittest.TestCase      ):
		_a     :  List[Any]           =     GPTaTokenizer
		_a     :  Optional[int]           =     GPTaTokenizerFast
		_a     :  List[Any]           =     True
		_a     :  Optional[int]           =     {'add_prefix_space': True}
		_a     :  Any           =     False
		def      UpperCAmelCase__    (      self						)			->	Any:




									"""simple docstring"""





									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									_UpperCAmelCase					=      [
									    'l',
									    'o',
									    'w',
									    'e',
									    'r',
									    's',
									    't',
									    'i',
									    'd',
									    'n',
									    '\u0120',
									    '\u0120l',
									    '\u0120n',
									    '\u0120lo',
									    '\u0120low',
									    'er',
									    '\u0120lowest',
									    '\u0120newer',
									    '\u0120wider',
									    '<unk>',
									    '<|endoftext|>',
									]
									_UpperCAmelCase					=      dict(zip(_SCREAMING_SNAKE_CASE      ,   range(len(_SCREAMING_SNAKE_CASE						)						)						)						)
									_UpperCAmelCase					=      ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
									_UpperCAmelCase					=      {'unk_token': '<unk>'}

									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['vocab_file']						)
									_UpperCAmelCase					=      os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES['merges_file']						)
									with open(self.vocab_file      ,   'w'      ,   encoding='utf-8'						) as fp:
																fp.write(json.dumps(_SCREAMING_SNAKE_CASE						) + '\n'						)
									with open(self.merges_file      ,   'w'      ,   encoding='utf-8'						) as fp:
																fp.write('\n'.join(_SCREAMING_SNAKE_CASE						)						)
		def      UpperCAmelCase__    (      self      ,   **_SCREAMING_SNAKE_CASE						)			->	List[Any]:




									"""simple docstring"""





									kwargs.update(self.special_tokens_map						)
									return GPTaTokenizer.from_pretrained(self.tmpdirname      ,   **_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   **_SCREAMING_SNAKE_CASE						)			->	Union[str, Any]:




									"""simple docstring"""





									kwargs.update(self.special_tokens_map						)
									return GPTaTokenizerFast.from_pretrained(self.tmpdirname      ,   **_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      'lower newer'
									_UpperCAmelCase					=      'lower newer'
									return input_text, output_text
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      GPTaTokenizer(self.vocab_file      ,   self.merges_file      ,   **self.special_tokens_map						)
									_UpperCAmelCase					=      'lower newer'
									_UpperCAmelCase					=      ['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE      ,   add_prefix_space=_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      tokens + [tokenizer.unk_token]
									_UpperCAmelCase					=      [14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   _SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									if not self.test_rust_tokenizer:
																return

									_UpperCAmelCase					=      self.get_tokenizer()
									_UpperCAmelCase					=      self.get_rust_tokenizer(add_prefix_space=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'lower newer'

									# Testing tokenization
									_UpperCAmelCase					=      tokenizer.tokenize(_SCREAMING_SNAKE_CASE      ,   add_prefix_space=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# Testing conversion to ids without special tokens
									_UpperCAmelCase					=      tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE      ,   add_prefix_space=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      rust_tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# Testing conversion to ids with special tokens
									_UpperCAmelCase					=      self.get_rust_tokenizer(add_prefix_space=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_prefix_space=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      rust_tokenizer.encode(_SCREAMING_SNAKE_CASE						)
									self.assertListEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)

									# Testing the unknown token
									_UpperCAmelCase					=      tokens + [rust_tokenizer.unk_token]
									_UpperCAmelCase					=      [14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE						)      ,   _SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   *_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)			->	List[Any]:




									"""simple docstring"""





									pass
		def      UpperCAmelCase__    (      self      ,   _SCREAMING_SNAKE_CASE=15						)			->	Union[str, Any]:




									"""simple docstring"""





									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''						):
																							_UpperCAmelCase					=      self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)

																							# Simple input
																							_UpperCAmelCase					=      'This is a simple input'
																							_UpperCAmelCase					=      ['This is a simple input 1', 'This is a simple input 2']
																							_UpperCAmelCase					=      ('This is a simple input', 'This is a pair')
																							_UpperCAmelCase					=      [
																							    ('This is a simple input 1', 'This is a simple input 2'),
																							    ('This is a simple pair 1', 'This is a simple pair 2'),
																							]

																							# Simple input tests
																							self.assertRaises(_SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode      ,   _SCREAMING_SNAKE_CASE      ,   max_length=_SCREAMING_SNAKE_CASE      ,   padding='max_length'						)

																							# Simple input
																							self.assertRaises(_SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode_plus      ,   _SCREAMING_SNAKE_CASE      ,   max_length=_SCREAMING_SNAKE_CASE      ,   padding='max_length'						)

																							# Simple input
																							self.assertRaises(
																							    _SCREAMING_SNAKE_CASE      ,   tokenizer_r.batch_encode_plus      ,   _SCREAMING_SNAKE_CASE      ,   max_length=_SCREAMING_SNAKE_CASE      ,   padding='max_length'      ,   )

																							# Pair input
																							self.assertRaises(_SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode      ,   _SCREAMING_SNAKE_CASE      ,   max_length=_SCREAMING_SNAKE_CASE      ,   padding='max_length'						)

																							# Pair input
																							self.assertRaises(_SCREAMING_SNAKE_CASE      ,   tokenizer_r.encode_plus      ,   _SCREAMING_SNAKE_CASE      ,   max_length=_SCREAMING_SNAKE_CASE      ,   padding='max_length'						)

																							# Pair input
																							self.assertRaises(
																							    _SCREAMING_SNAKE_CASE      ,   tokenizer_r.batch_encode_plus      ,   _SCREAMING_SNAKE_CASE      ,   max_length=_SCREAMING_SNAKE_CASE      ,   padding='max_length'      ,   )
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									_UpperCAmelCase					=      GPTaTokenizer.from_pretrained(self.tmpdirname      ,   pad_token='<pad>'						)

									# Simple input
									_UpperCAmelCase					=      'This is a simple input'
									_UpperCAmelCase					=      ['This is a simple input looooooooong', 'This is a simple input']
									_UpperCAmelCase					=      ('This is a simple input', 'This is a pair')
									_UpperCAmelCase					=      [
									    ('This is a simple input loooooong', 'This is a simple input'),
									    ('This is a simple pair loooooong', 'This is a simple pair'),
									]

									_UpperCAmelCase					=      tokenizer.pad_token_id

									_UpperCAmelCase					=      tokenizer(_SCREAMING_SNAKE_CASE      ,   padding='max_length'      ,   max_length=30      ,   return_tensors='np'						)
									_UpperCAmelCase					=      tokenizer(_SCREAMING_SNAKE_CASE      ,   padding=_SCREAMING_SNAKE_CASE      ,   truncate=_SCREAMING_SNAKE_CASE      ,   return_tensors='np'						)
									_UpperCAmelCase					=      tokenizer(*_SCREAMING_SNAKE_CASE      ,   padding='max_length'      ,   max_length=60      ,   return_tensors='np'						)
									_UpperCAmelCase					=      tokenizer(_SCREAMING_SNAKE_CASE      ,   padding=_SCREAMING_SNAKE_CASE      ,   truncate=_SCREAMING_SNAKE_CASE      ,   return_tensors='np'						)

									# s
									# test single string max_length padding
									self.assertEqual(out_s['input_ids'].shape[-1]      ,   30						)
									self.assertTrue(pad_token_id in out_s['input_ids']						)
									self.assertTrue(0 in out_s['attention_mask']						)

									# s2
									# test automatic padding
									self.assertEqual(out_sa['input_ids'].shape[-1]      ,   33						)
									# long slice doesn't have padding
									self.assertFalse(pad_token_id in out_sa['input_ids'][0]						)
									self.assertFalse(0 in out_sa['attention_mask'][0]						)
									# short slice does have padding
									self.assertTrue(pad_token_id in out_sa['input_ids'][1]						)
									self.assertTrue(0 in out_sa['attention_mask'][1]						)

									# p
									# test single pair max_length padding
									self.assertEqual(out_p['input_ids'].shape[-1]      ,   60						)
									self.assertTrue(pad_token_id in out_p['input_ids']						)
									self.assertTrue(0 in out_p['attention_mask']						)

									# p2
									# test automatic padding pair
									self.assertEqual(out_pa['input_ids'].shape[-1]      ,   52						)
									# long slice pair doesn't have padding
									self.assertFalse(pad_token_id in out_pa['input_ids'][0]						)
									self.assertFalse(0 in out_pa['attention_mask'][0]						)
									# short slice pair does have padding
									self.assertTrue(pad_token_id in out_pa['input_ids'][1]						)
									self.assertTrue(0 in out_pa['attention_mask'][1]						)
		def      UpperCAmelCase__    (      self						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      '$$$'
									_UpperCAmelCase					=      GPTaTokenizer.from_pretrained(self.tmpdirname      ,   bos_token=_SCREAMING_SNAKE_CASE      ,   add_bos_token=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'This is a simple input'
									_UpperCAmelCase					=      ['This is a simple input 1', 'This is a simple input 2']

									_UpperCAmelCase					=      tokenizer.bos_token_id

									_UpperCAmelCase					=      tokenizer(_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      tokenizer(_SCREAMING_SNAKE_CASE						)

									self.assertEqual(out_s.input_ids[0]      ,   _SCREAMING_SNAKE_CASE						)
									self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids						)						)

									_UpperCAmelCase					=      tokenizer.decode(out_s.input_ids						)
									_UpperCAmelCase					=      tokenizer.batch_decode(out_sa.input_ids						)

									self.assertEqual(decode_s.split()[0]      ,   _SCREAMING_SNAKE_CASE						)
									self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa						)						)
		def      UpperCAmelCase__    (      self						)			->	str:




									"""simple docstring"""





									pass






		def      UpperCAmelCase__    (      self						)			->	int:




									"""simple docstring"""





									_UpperCAmelCase					=      [self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE      ,   add_bos_token=_SCREAMING_SNAKE_CASE						)]
									for tokenizer in tokenizers:
																with self.subTest(f'''{tokenizer.__class__.__name__}'''						):
																							_UpperCAmelCase					=      'Encode this.'
																							_UpperCAmelCase					=      'This one too please.'
																							_UpperCAmelCase					=      tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
																							encoded_sequence += tokenizer.encode(_SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE						)
																							_UpperCAmelCase					=      tokenizer.encode_plus(
																							    _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE      ,   add_special_tokens=_SCREAMING_SNAKE_CASE      ,   return_special_tokens_mask=_SCREAMING_SNAKE_CASE      ,   )
																							_UpperCAmelCase					=      encoded_sequence_dict['input_ids']
																							_UpperCAmelCase					=      encoded_sequence_dict['special_tokens_mask']
																							self.assertEqual(len(_SCREAMING_SNAKE_CASE						)      ,   len(_SCREAMING_SNAKE_CASE						)						)

																							_UpperCAmelCase					=      [
																							    (x if not special_tokens_mask[i] else None) for i, x in enumerate(_SCREAMING_SNAKE_CASE						)
																							]
																							_UpperCAmelCase					=      [x for x in filtered_sequence if x is not None]
																							self.assertEqual(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)



@require_tokenizers
class 		__a				(    unittest.TestCase      ):
		def      UpperCAmelCase__    (      self						)			->	Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=      AutoTokenizer.from_pretrained('facebook/opt-350m'      ,   from_slow=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      'A photo of a cat'

									_UpperCAmelCase					=      tokenizer.encode(
									    _SCREAMING_SNAKE_CASE      ,   )
									self.assertEqual(_SCREAMING_SNAKE_CASE      ,   [2, 250, 1345, 9, 10, 4758]						)
									tokenizer.save_pretrained('test_opt'						)

									_UpperCAmelCase					=      AutoTokenizer.from_pretrained('./test_opt'						)
									_UpperCAmelCase					=      tokenizer.encode(
									    _SCREAMING_SNAKE_CASE      ,   )
									self.assertEqual(_SCREAMING_SNAKE_CASE      ,   [2, 250, 1345, 9, 10, 4758]						)
		def      UpperCAmelCase__    (      self						)			->	Union[str, Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      AutoTokenizer.from_pretrained('facebook/opt-350m'      ,   use_slow=_SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      'A photo of a cat'

									_UpperCAmelCase					=      tokenizer.encode(
									    _SCREAMING_SNAKE_CASE      ,   )
									# Same as above
									self.assertEqual(_SCREAMING_SNAKE_CASE      ,   [2, 250, 1345, 9, 10, 4758]						)






		@unittest.skip('This test is failing because of a bug in the fast tokenizer'						)
		def      UpperCAmelCase__    (      self						)			->	List[Any]:




									"""simple docstring"""





									_UpperCAmelCase					=      AutoTokenizer.from_pretrained('facebook/opt-350m'      ,   from_slow=_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      'bos'
									_UpperCAmelCase					=      tokenizer.get_vocab()['bos']

									_UpperCAmelCase					=      'A photo of a cat'
									_UpperCAmelCase					=      tokenizer.encode(
									    _SCREAMING_SNAKE_CASE      ,   )
									# We changed the bos token
									self.assertEqual(_SCREAMING_SNAKE_CASE      ,   [31957, 250, 1345, 9, 10, 4758]						)
									tokenizer.save_pretrained('./tok'						)
									_UpperCAmelCase					=      AutoTokenizer.from_pretrained('./tok'						)
									self.assertTrue(tokenizer.is_fast						)
									_UpperCAmelCase					=      tokenizer.encode(
									    _SCREAMING_SNAKE_CASE      ,   )
									self.assertEqual(_SCREAMING_SNAKE_CASE      ,   [31957, 250, 1345, 9, 10, 4758]						)




import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin

class 		__a				(    UpperCAmelCase      ):
		_a     :  Optional[int]           =     'MCTCTFeatureExtractor'
		_a     :  int           =     'AutoTokenizer'
		def __init__(      self      ,   _SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)			->	List[str]:




									"""simple docstring"""





									super().__init__(_SCREAMING_SNAKE_CASE      ,   _SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      self.feature_extractor
									_UpperCAmelCase					=      False
		def __call__(      self      ,   *_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)			->	Tuple:




									"""simple docstring"""





									if self._in_target_context_manager:
																return self.current_processor(*_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)

									if "raw_speech" in kwargs:
																warnings.warn('Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'						)
																_UpperCAmelCase					=      kwargs.pop('raw_speech'						)
									else:
																_UpperCAmelCase					=      kwargs.pop('audio'      ,   _SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      kwargs.pop('sampling_rate'      ,   _SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      kwargs.pop('text'      ,   _SCREAMING_SNAKE_CASE						)
									if len(_SCREAMING_SNAKE_CASE						) > 0:
																_UpperCAmelCase					=      args[0]
																_UpperCAmelCase					=      args[1:]

									if audio is None and text is None:
																raise ValueError('You need to specify either an `audio` or `text` input to process.'						)

									if audio is not None:
																_UpperCAmelCase					=      self.feature_extractor(_SCREAMING_SNAKE_CASE      ,   *_SCREAMING_SNAKE_CASE      ,   sampling_rate=_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)
									if text is not None:
																_UpperCAmelCase					=      self.tokenizer(_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)

									if text is None:
																return inputs
									elif audio is None:
																return encodings
									else:
																_UpperCAmelCase					=      encodings['input_ids']
																return inputs
		def      UpperCAmelCase__    (      self      ,   *_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)			->	Optional[int]:




									"""simple docstring"""





									return self.tokenizer.batch_decode(*_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)
		def      UpperCAmelCase__    (      self      ,   *_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)			->	str:




									"""simple docstring"""





									if self._in_target_context_manager:
																return self.current_processor.pad(*_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)

									_UpperCAmelCase					=      kwargs.pop('input_features'      ,   _SCREAMING_SNAKE_CASE						)
									_UpperCAmelCase					=      kwargs.pop('labels'      ,   _SCREAMING_SNAKE_CASE						)
									if len(_SCREAMING_SNAKE_CASE						) > 0:
																_UpperCAmelCase					=      args[0]
																_UpperCAmelCase					=      args[1:]

									if input_features is not None:
																_UpperCAmelCase					=      self.feature_extractor.pad(_SCREAMING_SNAKE_CASE      ,   *_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)
									if labels is not None:
																_UpperCAmelCase					=      self.tokenizer.pad(_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)

									if labels is None:
																return input_features
									elif input_features is None:
																return labels
									else:
																_UpperCAmelCase					=      labels['input_ids']
																return input_features
		def      UpperCAmelCase__    (      self      ,   *_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)			->	int:




									"""simple docstring"""





									return self.tokenizer.decode(*_SCREAMING_SNAKE_CASE      ,   **_SCREAMING_SNAKE_CASE						)






		@contextmanager
		def      UpperCAmelCase__    (      self						)			->	Optional[Any]:




									"""simple docstring"""





									warnings.warn(
									    '`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '
									    'labels by using the argument `text` of the regular `__call__` method (either in the same call as '
									    'your audio inputs, or in a separate call.'						)
									_UpperCAmelCase					=      True
									_UpperCAmelCase					=      self.tokenizer
									yield
									_UpperCAmelCase					=      self.feature_extractor
									_UpperCAmelCase					=      False
