





'''simple docstring'''



import argparse
import os
import re

import packaging.version


_SCREAMING_SNAKE_CASE   =      '''examples/'''
_SCREAMING_SNAKE_CASE   =      {
    '''examples''': (re.compile(R'''^check_min_version\("[^"]+"\)\s*$''', re.MULTILINE), '''check_min_version("VERSION")\n'''),
    '''init''': (re.compile(R'''^__version__\s+=\s+"([^"]+)"\s*$''', re.MULTILINE), '''__version__ = "VERSION"\n'''),
    '''setup''': (re.compile(R'''^(\s*)version\s*=\s*"[^"]+",''', re.MULTILINE), R'''\1version="VERSION",'''),
    '''doc''': (re.compile(R'''^(\s*)release\s*=\s*"[^"]+"$''', re.MULTILINE), '''release = "VERSION"\n'''),
}
_SCREAMING_SNAKE_CASE   =      {
    '''init''': '''src/diffusers/__init__.py''',
    '''setup''': '''setup.py''',
}
_SCREAMING_SNAKE_CASE   =      '''README.md'''




def 				_lowerCAmelCase  (			lowerCamelCase_     :					int						,    lowerCamelCase_     :					Tuple						,    lowerCamelCase_     :					Union[str, Any]			):
			with open(lowerCamelCase_						,    '''r'''						,    encoding='''utf-8'''						,    newline='''\n'''			) as f:
						__lowercase			  =							f.read()
			__lowercase  ,						__lowercase			  =							REPLACE_PATTERNS[pattern]
			__lowercase			  =							replace.replace('''VERSION'''						,    lowerCamelCase_			)
			__lowercase			  =							re_pattern.sub(lowerCamelCase_						,    lowerCamelCase_			)
			with open(lowerCamelCase_						,    '''w'''						,    encoding='''utf-8'''						,    newline='''\n'''			) as f:
						f.write(lowerCamelCase_			)




def 				_lowerCAmelCase  (			lowerCamelCase_     :					int			):
			for folder, directories, fnames in os.walk(lowerCamelCase_			):
						# Removing some of the folders with non-actively maintained examples from the walk
						if "research_projects" in directories:
									directories.remove('''research_projects'''			)
						if "legacy" in directories:
									directories.remove('''legacy'''			)
						for fname in fnames:
									if fname.endswith('''.py'''			):
												update_version_in_file(os.path.join(lowerCamelCase_						,    lowerCamelCase_			)						,    lowerCamelCase_						,    pattern='''examples'''			)




def 				_lowerCAmelCase  (			lowerCamelCase_     :					List[str]						,    lowerCamelCase_     :					Optional[int]=False			):
			for pattern, fname in REPLACE_FILES.items():
						update_version_in_file(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			)
			if not patch:
						update_version_in_examples(lowerCamelCase_			)




def 				_lowerCAmelCase  (			):
			__lowercase			  =							'''🤗 Transformers currently provides the following architectures'''
			__lowercase			  =							'''1. Want to contribute a new model?'''
			with open(lowerCamelCase_						,    '''r'''						,    encoding='''utf-8'''						,    newline='''\n'''			) as f:
						__lowercase			  =							f.readlines()

			# Find the start of the list.
			__lowercase			  =							0
			while not lines[start_index].startswith(_start_prompt			):
						start_index += 1
			start_index += 1

			__lowercase			  =							start_index
			# Update the lines in the model list.
			while not lines[index].startswith(_end_prompt			):
						if lines[index].startswith('''1.'''			):
									__lowercase			  =							lines[index].replace(
									    '''https://huggingface.co/docs/diffusers/main/model_doc'''						,    '''https://huggingface.co/docs/diffusers/model_doc'''						,    )
						index += 1

			with open(lowerCamelCase_						,    '''w'''						,    encoding='''utf-8'''						,    newline='''\n'''			) as f:
						f.writelines(lowerCamelCase_			)




def 				_lowerCAmelCase  (			):
			with open(REPLACE_FILES['''init''']						,    '''r'''			) as f:
						__lowercase			  =							f.read()
			__lowercase			  =							REPLACE_PATTERNS['''init'''][0].search(lowerCamelCase_			).groups()[0]
			return packaging.version.parse(lowerCamelCase_			)




def 				_lowerCAmelCase  (			lowerCamelCase_     :					List[str]=False			):
			__lowercase			  =							get_version()
			if patch and default_version.is_devrelease:
						raise ValueError('''Can\'t create a patch version from the dev branch, checkout a released version!'''			)
			if default_version.is_devrelease:
						__lowercase			  =							default_version.base_version
			elif patch:
						__lowercase			  =							f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}"
			else:
						__lowercase			  =							f"{default_version.major}.{default_version.minor + 1}.0"

			# Now let's ask nicely if that's the right one.
			__lowercase			  =							input(f"Which version are you releasing? [{default_version}]"			)
			if len(lowerCamelCase_			) == 0:
						__lowercase			  =							default_version

			print(f"Updating version to {version}."			)
			global_version_update(lowerCamelCase_						,    patch=lowerCamelCase_			)




def 				_lowerCAmelCase  (			):
			__lowercase			  =							get_version()
			__lowercase			  =							f"{current_version.major}.{current_version.minor + 1}.0.dev0"
			__lowercase			  =							current_version.base_version

			# Check with the user we got that right.
			__lowercase			  =							input(f"Which version are we developing now? [{dev_version}]"			)
			if len(lowerCamelCase_			) == 0:
						__lowercase			  =							dev_version

			print(f"Updating version to {version}."			)
			global_version_update(lowerCamelCase_			)


#    print("Cleaning main README, don't forget to run `make fix-copies`.")
#    clean_main_ref_in_model_list()


if __name__ == "__main__":
	_SCREAMING_SNAKE_CASE   =      argparse.ArgumentParser()
	parser.add_argument('''--post_release''', action='''store_true''', help='''Whether this is pre or post release.''')
	parser.add_argument('''--patch''', action='''store_true''', help='''Whether or not this is a patch release.''')
	_SCREAMING_SNAKE_CASE   =      parser.parse_args()
	if not args.post_release:
		pre_release_work(patch=args.patch)
	elif args.patch:
		print('''Nothing to do after a patch :-)''')
	else:
		post_release_work()












'''simple docstring'''



import json
import os
import shutil
import tempfile
import unittest

from transformers import BatchEncoding, CanineTokenizer
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.tokenization_utils import AddedToken
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin






class 							__lowercase    ( lowerCAmelCase__	, unittest.TestCase  ):







	'''simple docstring'''

	a      :			List[Any]     =    CanineTokenizer
	a      :			Union[str, Any]     =    False


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				super().setUp()
				__lowercase			  =							CanineTokenizer()
				tokenizer.save_pretrained(self.tmpdirname	)


	@cached_property
	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				return CanineTokenizer.from_pretrained('''google/canine-s'''	)


	def   _UpperCAmelCase       (self       ,**_lowerCamelCase	)							->							CanineTokenizer:







				'''simple docstring'''





				__lowercase			  =							self.tokenizer_class.from_pretrained(self.tmpdirname       ,**_lowerCamelCase	)
				__lowercase			  =							1024
				return tokenizer


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Optional[int]:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Life is like a box of chocolates.''', '''You never know what you\'re gonna get.''']
				# fmt: off
				__lowercase			  =							[57344, 76, 105, 102, 101, 32, 105, 115, 32, 108, 105, 107, 101, 32, 97, 32, 98, 111, 120, 32, 111, 102, 32, 99, 104, 111, 99, 111, 108, 97, 116, 101, 115, 46, 57345, 0, 0, 0, 0]
				# fmt: on
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertIsInstance(_lowerCamelCase       ,_lowerCamelCase	)

				__lowercase			  =							list(batch.input_ids.numpy()[0]	)

				self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

				self.assertEqual((2, 39)       ,batch.input_ids.shape	)
				self.assertEqual((2, 39)       ,batch.attention_mask.shape	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							['''Once there was a man.''', '''He wrote a test in HuggingFace Tranformers.''']
				__lowercase			  =							tokenizer(_lowerCamelCase       ,padding=_lowerCamelCase       ,return_tensors='''pt'''	)
				# check if input_ids, attention_mask and token_type_ids are returned
				self.assertIn('''input_ids'''       ,_lowerCamelCase	)
				self.assertIn('''attention_mask'''       ,_lowerCamelCase	)
				self.assertIn('''token_type_ids'''       ,_lowerCamelCase	)


	@require_torch
	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.canine_tokenizer
				__lowercase			  =							[
				    '''What\'s the weater?''',
				    '''It\'s about 25 degrees.''',
				]
				__lowercase			  =							tokenizer(
				    text_target=_lowerCamelCase       ,max_length=32       ,padding='''max_length'''       ,truncation=_lowerCamelCase       ,return_tensors='''pt'''	)
				self.assertEqual(32       ,targets['''input_ids'''].shape[1]	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										self.assertNotEqual(tokenizer.model_max_length       ,42	)

        # Now let's start the test
				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)

										shutil.rmtree(_lowerCamelCase	)

				__lowercase			  =							self.get_tokenizers(model_max_length=42	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# Isolate this from the other tests because we save additional tokens/etc
										__lowercase			  =							tempfile.mkdtemp()

										__lowercase			  =							''' He is very happy, UNwant\u00E9d,running'''

										__lowercase			  =							tokenizer.additional_special_tokens

										# We can add a new special token for Canine as follows:
										__lowercase			  =							chr(0xe_0_0_7	)
										additional_special_tokens.append(_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										tokenizer.save_pretrained(_lowerCamelCase	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase	)
										__lowercase			  =							after_tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertListEqual(_lowerCamelCase       ,_lowerCamelCase	)
										self.assertIn(_lowerCamelCase       ,after_tokenizer.additional_special_tokens	)
										self.assertEqual(after_tokenizer.model_max_length       ,42	)

										__lowercase			  =							tokenizer.__class__.from_pretrained(_lowerCamelCase       ,model_max_length=43	)
										self.assertEqual(tokenizer.model_max_length       ,43	)

										shutil.rmtree(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase  ,						__lowercase			  =							self.get_clean_sequence(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_5
										__lowercase			  =							chr(_lowerCamelCase	)

										tokenizer.add_special_tokens({'''cls_token''': special_token}	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)

										__lowercase			  =							tokenizer.decode(ids + encoded_special_token       ,clean_up_tokenization_spaces=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)

										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										self.assertEqual(_lowerCamelCase       ,input_encoded + special_token_id	)

										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,skip_special_tokens=_lowerCamelCase	)
										self.assertTrue(special_token not in decoded	)


	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							chr(0xe_0_0_5	)
										__lowercase			  =							chr(0xe_0_0_6	)

										# `add_tokens` method stores special tokens only in `tokenizer.unique_no_split_tokens`. (in tokenization_utils.py)
										tokenizer.add_tokens([SPECIAL_TOKEN_1]       ,special_tokens=_lowerCamelCase	)
										# `add_special_tokens` method stores special tokens in `tokenizer.additional_special_tokens`,
										# which also occur in `tokenizer.all_special_tokens`. (in tokenization_utils_base.py)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [SPECIAL_TOKEN_2]}	)

										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)
										__lowercase			  =							tokenizer.tokenize(_lowerCamelCase	)

										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(len(_lowerCamelCase	)       ,1	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)
										self.assertEqual(token_a[0]       ,_lowerCamelCase	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)
										tokenizer.add_special_tokens({'''additional_special_tokens''': [new_token]}	)

										with tempfile.TemporaryDirectory() as tmp_dir_name:
													tokenizer.save_pretrained(_lowerCamelCase	)
													tokenizer.from_pretrained(_lowerCamelCase	)


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				__lowercase			  =							[]
				if self.test_slow_tokenizer:
							tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())	)

				if self.test_rust_tokenizer:
							tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())	)

				for tokenizer_class, tokenizer_utils in tokenizer_list:
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer_utils.save_pretrained(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,encoding='''utf-8'''	) as json_file:
													__lowercase			  =							json.load(_lowerCamelCase	)

										# a special token for Canine can be defined as follows:
										__lowercase			  =							0xe_0_0_6
										__lowercase			  =							chr(_lowerCamelCase	)

										__lowercase			  =							[new_token_a]
										__lowercase			  =							[new_token_a]

										with open(os.path.join(_lowerCamelCase       ,'''special_tokens_map.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)
										with open(os.path.join(_lowerCamelCase       ,'''tokenizer_config.json'''	)       ,'''w'''       ,encoding='''utf-8'''	) as outfile:
													json.dump(_lowerCamelCase       ,_lowerCamelCase	)

										# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
										# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
										# "special_tokens_map.json" files
										__lowercase			  =							tokenizer_class.from_pretrained(_lowerCamelCase       ,extra_ids=0	)
										self.assertIn(_lowerCamelCase       ,tokenizer_without_change_in_init.additional_special_tokens	)
										# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer_without_change_in_init.convert_ids_to_tokens(
										        tokenizer_without_change_in_init.convert_tokens_to_ids([new_token_a]	)	)       ,)

										__lowercase			  =							0xe_0_0_7
										__lowercase			  =							chr(_lowerCamelCase	)
										# Now we test that we can change the value of additional_special_tokens in the from_pretrained
										__lowercase			  =							[AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase	)]
										__lowercase			  =							tokenizer_class.from_pretrained(
										    _lowerCamelCase       ,additional_special_tokens=_lowerCamelCase       ,extra_ids=0	)

										self.assertIn(_lowerCamelCase       ,tokenizer.additional_special_tokens	)
										# self.assertIn(new_token_2,tokenizer.get_vocab()) # ByT5Tokenization no vocab
										self.assertEqual(
										    [new_token_a]       ,tokenizer.convert_ids_to_tokens(tokenizer.convert_tokens_to_ids([new_token_a]	)	)	)


	@require_tokenizers
	def   _UpperCAmelCase       (self	)							->							Union[str, Any]:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers(do_lower_case=_lowerCamelCase	)
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							'''hello world'''
										if self.space_between_special_tokens:
													__lowercase			  =							'''[CLS] hello world [SEP]'''
										else:
													__lowercase			  =							input
										__lowercase			  =							tokenizer.encode(_lowerCamelCase       ,add_special_tokens=_lowerCamelCase	)
										__lowercase			  =							tokenizer.decode(_lowerCamelCase       ,spaces_between_special_tokens=self.space_between_special_tokens	)
										self.assertIn(_lowerCamelCase       ,[output, output.lower()]	)


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							self.get_tokenizers()
				for tokenizer in tokenizers:
							with self.subTest(f"{tokenizer.__class__.__name__}"	):
										__lowercase			  =							[
										    '''bos_token''',
										    '''eos_token''',
										    '''unk_token''',
										    '''sep_token''',
										    '''pad_token''',
										    '''cls_token''',
										    '''mask_token''',
										]

				__lowercase			  =							'''a'''
				__lowercase			  =							ord(_lowerCamelCase	)

				for attr in attributes_list:
							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

							setattr(_lowerCamelCase       ,attr + '''_id'''       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,_lowerCamelCase	)       ,_lowerCamelCase	)
							self.assertEqual(getattr(_lowerCamelCase       ,attr + '''_id'''	)       ,_lowerCamelCase	)

				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[]	)

				__lowercase			  =							0xe_0_0_6
				__lowercase			  =							chr(_lowerCamelCase	)
				setattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''       ,[additional_special_token_id]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens'''	)       ,[additional_special_token]	)
				self.assertListEqual(getattr(_lowerCamelCase       ,'''additional_special_tokens_ids'''	)       ,[additional_special_token_id]	)


	def   _UpperCAmelCase       (self	)							->							List[Any]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Tuple:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							List[str]:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Any:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							int:







				'''simple docstring'''





				pass


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				pass





