



import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class      UpperCamelCase    (						_UpperCAmelCase       ,    unittest.TestCase     ):
	lowerCAmelCase	:   Tuple				=	CTRLTokenizer
	lowerCAmelCase	:   Any				=	False
	lowerCAmelCase	:   Any				=	False



	def 			__A							(       self		):
							super().setUp()

							# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
							A__							=       ["adapt", "re@@", "a@@", "apt", "c@@", "t", "<unk>"]
							A__							=       dict(zip(UpperCAmelCase__			,	range(len(UpperCAmelCase__		)		)		)		)
							A__							=       ["#version: 0.2", "a p", "ap t</w>", "r e", "a d", "ad apt</w>", ""]
							A__							=       {"unk_token": "<unk>"}

							A__							=       os.path.join(self.tmpdirname			,	VOCAB_FILES_NAMES["vocab_file"]		)
							A__							=       os.path.join(self.tmpdirname			,	VOCAB_FILES_NAMES["merges_file"]		)
							with open(self.vocab_file			,	"w"			,	encoding="utf-8"		) as fp:
													fp.write(json.dumps(UpperCAmelCase__		) + "\n"		)
							with open(self.merges_file			,	"w"			,	encoding="utf-8"		) as fp:
													fp.write("\n".join(UpperCAmelCase__		)		)



	def 			__A							(       self			,	**UpperCAmelCase__		):
							kwargs.update(self.special_tokens_map		)
							return CTRLTokenizer.from_pretrained(self.tmpdirname			,	**UpperCAmelCase__		)



	def 			__A							(       self			,	UpperCAmelCase__		):
							A__							=       "adapt react readapt apt"
							A__							=       "adapt react readapt apt"
							return input_text, output_text



	def 			__A							(       self		):
							A__							=       CTRLTokenizer(self.vocab_file			,	self.merges_file			,	**self.special_tokens_map		)
							A__							=       "adapt react readapt apt"
							A__							=       "adapt re@@ a@@ c@@ t re@@ adapt apt".split()
							A__							=       tokenizer.tokenize(UpperCAmelCase__		)
							self.assertListEqual(UpperCAmelCase__			,	UpperCAmelCase__		)

							A__							=       tokens + [tokenizer.unk_token]

							A__							=       [0, 1, 2, 4, 5, 1, 0, 3, 6]
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)			,	UpperCAmelCase__		)










from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

UpperCAmelCase_							:      List[Any]      =  [
    "python",
    "tqdm",
    "regex",
    "requests",
    "packaging",
    "filelock",
    "numpy",
    "tokenizers",
    "huggingface-hub",
    "safetensors",
    "accelerate",
    "pyyaml",
]

for pkg in pkgs_to_check_at_runtime:
						if pkg in deps:
												if pkg == "tokenizers":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_tokenizers_available

																		if not is_tokenizers_available():
																								continue  # not required, check version only if installed
												elif pkg == "accelerate":
																		# must be loaded here, or else tqdm check may fail
																		from .utils import is_accelerate_available

																		# Maybe switch to is_torch_available in the future here so that Accelerate is hard dep of
																		# Transformers with PyTorch
																		if not is_accelerate_available():
																								continue  # not required, check version only if installed

												require_version_core(deps[pkg])
						else:
												raise ValueError(F'''can\'t find {pkg} in {deps.keys()}, check dependency_versions_table.py''')
def  UpperCamelCase						(       _A			:				List[Any]			,							_A			:				int=None    )->       Optional[int]:





						"""simple docstring"""





						require_version(deps[pkg]			,							_A    )





