
"""simple docstring"""







import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 			A__	(     _lowerCamelCase	,					unittest.TestCase):
		A_						:	Optional[int]           =     GPTaTokenizer
		A_						:	Any           =     GPTaTokenizerFast
		A_						:	str           =     True
		A_						:	Any           =     {'add_prefix_space': True}
		A_						:	Optional[Any]           =     False

		def 							__lowerCamelCase							(				self			):
			super().setUp()

			# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
			__lowerCAmelCase      :						Union[str, Any]						      =				[
			    'l',
			    'o',
			    'w',
			    'e',
			    'r',
			    's',
			    't',
			    'i',
			    'd',
			    'n',
			    '\u0120',
			    '\u0120l',
			    '\u0120n',
			    '\u0120lo',
			    '\u0120low',
			    'er',
			    '\u0120lowest',
			    '\u0120newer',
			    '\u0120wider',
			    '<unk>',
			    '<|endoftext|>',
			]
			__lowerCAmelCase      :						Optional[Any]						      =				dict(zip(_SCREAMING_SNAKE_CASE	,		range(len(_SCREAMING_SNAKE_CASE			)			)			)			)
			__lowerCAmelCase      :						Optional[Any]						      =				['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
			__lowerCAmelCase      :						Optional[int]						      =				{'unk_token': '<unk>'}

			__lowerCAmelCase      :						str						      =				os.path.join(self.tmpdirname	,		VOCAB_FILES_NAMES['vocab_file']			)
			__lowerCAmelCase      :						Optional[Any]						      =				os.path.join(self.tmpdirname	,		VOCAB_FILES_NAMES['merges_file']			)
			with open(self.vocab_file	,		'w'	,		encoding='utf-8'			) as fp:
				fp.write(json.dumps(_SCREAMING_SNAKE_CASE			) + '\n'			)
			with open(self.merges_file	,		'w'	,		encoding='utf-8'			) as fp:
				fp.write('\n'.join(_SCREAMING_SNAKE_CASE			)			)

		def 							__lowerCamelCase							(				self	,		**_SCREAMING_SNAKE_CASE			):
			kwargs.update(self.special_tokens_map			)
			return GPTaTokenizer.from_pretrained(self.tmpdirname	,		**_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self	,		**_SCREAMING_SNAKE_CASE			):
			kwargs.update(self.special_tokens_map			)
			return GPTaTokenizerFast.from_pretrained(self.tmpdirname	,		**_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE			):
			__lowerCAmelCase      :						Union[str, Any]						      =				'lower newer'
			__lowerCAmelCase      :						str						      =				'lower newer'
			return input_text, output_text

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Dict						      =				GPTaTokenizer(self.vocab_file	,		self.merges_file	,		**self.special_tokens_map			)
			__lowerCAmelCase      :						List[Any]						      =				'lower newer'
			__lowerCAmelCase      :						List[str]						      =				['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
			__lowerCAmelCase      :						Optional[int]						      =				tokenizer.tokenize(_SCREAMING_SNAKE_CASE	,		add_prefix_space=_SCREAMING_SNAKE_CASE			)
			self.assertListEqual(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						Union[str, Any]						      =				tokens + [tokenizer.unk_token]
			__lowerCAmelCase      :						Any						      =				[14, 15, 10, 9, 3, 2, 15, 19]
			self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE			)	,		_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self			):
			if not self.test_rust_tokenizer:
				return

			__lowerCAmelCase      :						Union[str, Any]						      =				self.get_tokenizer()
			__lowerCAmelCase      :						int						      =				self.get_rust_tokenizer(add_prefix_space=_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						Optional[int]						      =				'lower newer'

			# Testing tokenization
			__lowerCAmelCase      :						Dict						      =				tokenizer.tokenize(_SCREAMING_SNAKE_CASE	,		add_prefix_space=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						List[Any]						      =				rust_tokenizer.tokenize(_SCREAMING_SNAKE_CASE			)
			self.assertListEqual(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

			# Testing conversion to ids without special tokens
			__lowerCAmelCase      :						Any						      =				tokenizer.encode(_SCREAMING_SNAKE_CASE	,		add_special_tokens=_SCREAMING_SNAKE_CASE	,		add_prefix_space=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						Optional[Any]						      =				rust_tokenizer.encode(_SCREAMING_SNAKE_CASE	,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
			self.assertListEqual(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

			# Testing conversion to ids with special tokens
			__lowerCAmelCase      :						List[Any]						      =				self.get_rust_tokenizer(add_prefix_space=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						Any						      =				tokenizer.encode(_SCREAMING_SNAKE_CASE	,		add_prefix_space=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						Optional[int]						      =				rust_tokenizer.encode(_SCREAMING_SNAKE_CASE			)
			self.assertListEqual(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)

			# Testing the unknown token
			__lowerCAmelCase      :						Optional[Any]						      =				tokens + [rust_tokenizer.unk_token]
			__lowerCAmelCase      :						Optional[int]						      =				[14, 15, 10, 9, 3, 2, 15, 19]
			self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE			)	,		_SCREAMING_SNAKE_CASE			)

		def 							__lowerCamelCase							(				self	,		*_SCREAMING_SNAKE_CASE	,		**_SCREAMING_SNAKE_CASE			):
			# It's very difficult to mix/test pretokenization with byte-level
			# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
			pass

		def 							__lowerCamelCase							(				self	,		_SCREAMING_SNAKE_CASE=15			):
			for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
				with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"			):
					__lowerCAmelCase      :						Union[str, Any]						      =				self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE	,		**_SCREAMING_SNAKE_CASE			)

					# Simple input
					__lowerCAmelCase      :						Union[str, Any]						      =				'This is a simple input'
					__lowerCAmelCase      :						Union[str, Any]						      =				['This is a simple input 1', 'This is a simple input 2']
					__lowerCAmelCase      :						Any						      =				('This is a simple input', 'This is a pair')
					__lowerCAmelCase      :						List[Any]						      =				[
					    ('This is a simple input 1', 'This is a simple input 2'),
					    ('This is a simple pair 1', 'This is a simple pair 2'),
					]

					# Simple input tests
					self.assertRaises(_SCREAMING_SNAKE_CASE	,		tokenizer_r.encode	,		_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	,		padding='max_length'			)

					# Simple input
					self.assertRaises(_SCREAMING_SNAKE_CASE	,		tokenizer_r.encode_plus	,		_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	,		padding='max_length'			)

					# Simple input
					self.assertRaises(
					    _SCREAMING_SNAKE_CASE	,		tokenizer_r.batch_encode_plus	,		_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	,		padding='max_length'	,		)

					# Pair input
					self.assertRaises(_SCREAMING_SNAKE_CASE	,		tokenizer_r.encode	,		_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	,		padding='max_length'			)

					# Pair input
					self.assertRaises(_SCREAMING_SNAKE_CASE	,		tokenizer_r.encode_plus	,		_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	,		padding='max_length'			)

					# Pair input
					self.assertRaises(
					    _SCREAMING_SNAKE_CASE	,		tokenizer_r.batch_encode_plus	,		_SCREAMING_SNAKE_CASE	,		max_length=_SCREAMING_SNAKE_CASE	,		padding='max_length'	,		)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Any						      =				GPTaTokenizer.from_pretrained(self.tmpdirname	,		pad_token='<pad>'			)

			# Simple input
			__lowerCAmelCase      :						Union[str, Any]						      =				'This is a simple input'
			__lowerCAmelCase      :						List[Any]						      =				['This is a simple input looooooooong', 'This is a simple input']
			__lowerCAmelCase      :						int						      =				('This is a simple input', 'This is a pair')
			__lowerCAmelCase      :						List[Any]						      =				[
			    ('This is a simple input loooooong', 'This is a simple input'),
			    ('This is a simple pair loooooong', 'This is a simple pair'),
			]

			__lowerCAmelCase      :						str						      =				tokenizer.pad_token_id

			__lowerCAmelCase      :						int						      =				tokenizer(_SCREAMING_SNAKE_CASE	,		padding='max_length'	,		max_length=30	,		return_tensors='np'			)
			__lowerCAmelCase      :						str						      =				tokenizer(_SCREAMING_SNAKE_CASE	,		padding=_SCREAMING_SNAKE_CASE	,		truncate=_SCREAMING_SNAKE_CASE	,		return_tensors='np'			)
			__lowerCAmelCase      :						Dict						      =				tokenizer(*_SCREAMING_SNAKE_CASE	,		padding='max_length'	,		max_length=60	,		return_tensors='np'			)
			__lowerCAmelCase      :						List[Any]						      =				tokenizer(_SCREAMING_SNAKE_CASE	,		padding=_SCREAMING_SNAKE_CASE	,		truncate=_SCREAMING_SNAKE_CASE	,		return_tensors='np'			)

			# s
			# test single string max_length padding
			self.assertEqual(out_s['input_ids'].shape[-1]	,		30			)
			self.assertTrue(pad_token_id in out_s['input_ids']			)
			self.assertTrue(0 in out_s['attention_mask']			)

			# s2
			# test automatic padding
			self.assertEqual(out_sa['input_ids'].shape[-1]	,		33			)
			# long slice doesn't have padding
			self.assertFalse(pad_token_id in out_sa['input_ids'][0]			)
			self.assertFalse(0 in out_sa['attention_mask'][0]			)
			# short slice does have padding
			self.assertTrue(pad_token_id in out_sa['input_ids'][1]			)
			self.assertTrue(0 in out_sa['attention_mask'][1]			)

			# p
			# test single pair max_length padding
			self.assertEqual(out_p['input_ids'].shape[-1]	,		60			)
			self.assertTrue(pad_token_id in out_p['input_ids']			)
			self.assertTrue(0 in out_p['attention_mask']			)

			# p2
			# test automatic padding pair
			self.assertEqual(out_pa['input_ids'].shape[-1]	,		52			)
			# long slice pair doesn't have padding
			self.assertFalse(pad_token_id in out_pa['input_ids'][0]			)
			self.assertFalse(0 in out_pa['attention_mask'][0]			)
			# short slice pair does have padding
			self.assertTrue(pad_token_id in out_pa['input_ids'][1]			)
			self.assertTrue(0 in out_pa['attention_mask'][1]			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						int						      =				'$$$'
			__lowerCAmelCase      :						Tuple						      =				GPTaTokenizer.from_pretrained(self.tmpdirname	,		bos_token=_SCREAMING_SNAKE_CASE	,		add_bos_token=_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						List[str]						      =				'This is a simple input'
			__lowerCAmelCase      :						Union[str, Any]						      =				['This is a simple input 1', 'This is a simple input 2']

			__lowerCAmelCase      :						Any						      =				tokenizer.bos_token_id

			__lowerCAmelCase      :						List[Any]						      =				tokenizer(_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						List[Any]						      =				tokenizer(_SCREAMING_SNAKE_CASE			)

			self.assertEqual(out_s.input_ids[0]	,		_SCREAMING_SNAKE_CASE			)
			self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids			)			)

			__lowerCAmelCase      :						int						      =				tokenizer.decode(out_s.input_ids			)
			__lowerCAmelCase      :						Dict						      =				tokenizer.batch_decode(out_sa.input_ids			)

			self.assertEqual(decode_s.split()[0]	,		_SCREAMING_SNAKE_CASE			)
			self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa			)			)

		def 							__lowerCamelCase							(				self			):
			pass







		def 							__lowerCamelCase							(				self			):
			# TODO: change to self.get_tokenizers() when the fast version is implemented
			__lowerCAmelCase      :						Dict						      =				[self.get_tokenizer(do_lower_case=_SCREAMING_SNAKE_CASE	,		add_bos_token=_SCREAMING_SNAKE_CASE			)]
			for tokenizer in tokenizers:
				with self.subTest(f"{tokenizer.__class__.__name__}"			):
					__lowerCAmelCase      :						List[Any]						      =				'Encode this.'
					__lowerCAmelCase      :						Optional[int]						      =				'This one too please.'
					__lowerCAmelCase      :						Optional[int]						      =				tokenizer.encode(_SCREAMING_SNAKE_CASE	,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
					encoded_sequence += tokenizer.encode(_SCREAMING_SNAKE_CASE	,		add_special_tokens=_SCREAMING_SNAKE_CASE			)
					__lowerCAmelCase      :						List[str]						      =				tokenizer.encode_plus(
					    _SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE	,		add_special_tokens=_SCREAMING_SNAKE_CASE	,		return_special_tokens_mask=_SCREAMING_SNAKE_CASE	,		)
					__lowerCAmelCase      :						Tuple						      =				encoded_sequence_dict['input_ids']
					__lowerCAmelCase      :						Optional[int]						      =				encoded_sequence_dict['special_tokens_mask']
					self.assertEqual(len(_SCREAMING_SNAKE_CASE			)	,		len(_SCREAMING_SNAKE_CASE			)			)

					__lowerCAmelCase      :						List[Any]						      =				[
					    (x if not special_tokens_mask[i] else None) for i, x in enumerate(_SCREAMING_SNAKE_CASE			)
					]
					__lowerCAmelCase      :						Any						      =				[x for x in filtered_sequence if x is not None]
					self.assertEqual(_SCREAMING_SNAKE_CASE	,		_SCREAMING_SNAKE_CASE			)




@require_tokenizers
class 			A__	(     unittest.TestCase):

		def 							__lowerCamelCase							(				self			):
			# More context:
			# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
			# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
			# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

			__lowerCAmelCase      :						Optional[int]						      =				AutoTokenizer.from_pretrained('facebook/opt-350m'	,		from_slow=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						Any						      =				'A photo of a cat'

			__lowerCAmelCase      :						Tuple						      =				tokenizer.encode(
			    _SCREAMING_SNAKE_CASE	,		)
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		[2, 2_50, 13_45, 9, 10, 47_58]			)
			tokenizer.save_pretrained('test_opt'			)

			__lowerCAmelCase      :						Tuple						      =				AutoTokenizer.from_pretrained('./test_opt'			)
			__lowerCAmelCase      :						str						      =				tokenizer.encode(
			    _SCREAMING_SNAKE_CASE	,		)
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		[2, 2_50, 13_45, 9, 10, 47_58]			)

		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						List[Any]						      =				AutoTokenizer.from_pretrained('facebook/opt-350m'	,		use_slow=_SCREAMING_SNAKE_CASE			)
			__lowerCAmelCase      :						Tuple						      =				'A photo of a cat'

			__lowerCAmelCase      :						str						      =				tokenizer.encode(
			    _SCREAMING_SNAKE_CASE	,		)
			# Same as above
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		[2, 2_50, 13_45, 9, 10, 47_58]			)







		@unittest.skip('This test is failing because of a bug in the fast tokenizer'			)
		def 							__lowerCamelCase							(				self			):
			__lowerCAmelCase      :						Dict						      =				AutoTokenizer.from_pretrained('facebook/opt-350m'	,		from_slow=_SCREAMING_SNAKE_CASE			)

			__lowerCAmelCase      :						Optional[Any]						      =				'bos'
			__lowerCAmelCase      :						Dict						      =				tokenizer.get_vocab()['bos']

			__lowerCAmelCase      :						Dict						      =				'A photo of a cat'
			__lowerCAmelCase      :						Tuple						      =				tokenizer.encode(
			    _SCREAMING_SNAKE_CASE	,		)
			# We changed the bos token
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		[3_19_57, 2_50, 13_45, 9, 10, 47_58]			)
			tokenizer.save_pretrained('./tok'			)
			__lowerCAmelCase      :						Optional[int]						      =				AutoTokenizer.from_pretrained('./tok'			)
			self.assertTrue(tokenizer.is_fast			)
			__lowerCAmelCase      :						Union[str, Any]						      =				tokenizer.encode(
			    _SCREAMING_SNAKE_CASE	,		)
			self.assertEqual(_SCREAMING_SNAKE_CASE	,		[3_19_57, 2_50, 13_45, 9, 10, 47_58]			)

"""simple docstring"""







lowerCamelCase__        =		"""0.21.0"""

from .accelerator import Accelerator
from .big_modeling import (
    cpu_offload,
    cpu_offload_with_hook,
    disk_offload,
    dispatch_model,
    init_empty_weights,
    init_on_device,
    load_checkpoint_and_dispatch,
)
from .data_loader import skip_first_batches
from .launchers import debug_launcher, notebook_launcher
from .state import PartialState
from .utils import (
    DeepSpeedPlugin,
    DistributedDataParallelKwargs,
    DistributedType,
    FullyShardedDataParallelPlugin,
    GradScalerKwargs,
    InitProcessGroupKwargs,
    find_executable_batch_size,
    infer_auto_device_map,
    is_rich_available,
    load_checkpoint_in_model,
    synchronize_rng_states,
)


if is_rich_available():
							from .utils import rich