





import gc
import random
import tempfile
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMInverseScheduler,
    DDIMScheduler,
    DPMSolverMultistepInverseScheduler,
    DPMSolverMultistepScheduler,
    StableDiffusionDiffEditPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import load_image, slow
from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, require_torch_gpu, torch_device

from ..pipeline_params import TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, TEXT_GUIDED_IMAGE_INPAINTING_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()

class 		UpperCamelCase__			(				lowerCAmelCase_							,				lowerCAmelCase_							,				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     Tuple		     =      StableDiffusionDiffEditPipeline
     __snake_case   :     Union[str, Any]		     =      TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"height", "width", "image"} | {"image_latents"}
     __snake_case   :     Optional[Any]		     =      TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS - {"image"} | {"image_latents"}
     __snake_case   :     Tuple		     =      frozenset(
         []       )  # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess
     __snake_case   :     Tuple		     =      frozenset([]       )






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		List[Any]:




           '''simple docstring'''


           torch.manual_seed(0	)
           SCREAMING_SNAKE_CASE		=						UNetaDConditionModel(
               block_out_channels=(32, 64)						,layers_per_block=2						,sample_size=32						,in_channels=4						,out_channels=4						,down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")						,up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")						,cross_attention_dim=32						,attention_head_dim=(2, 4)						,use_linear_projection=lowerCamelCase__						,)
           SCREAMING_SNAKE_CASE		=						DDIMScheduler(
               beta_start=0.00085						,beta_end=0.012						,beta_schedule="""scaled_linear"""						,clip_sample=lowerCamelCase__						,set_alpha_to_one=lowerCamelCase__						,)
           SCREAMING_SNAKE_CASE		=						DDIMInverseScheduler(
               beta_start=0.00085						,beta_end=0.012						,beta_schedule="""scaled_linear"""						,clip_sample=lowerCamelCase__						,set_alpha_to_zero=lowerCamelCase__						,)
           torch.manual_seed(0	)
           SCREAMING_SNAKE_CASE		=						AutoencoderKL(
               block_out_channels=[32, 64]						,in_channels=3						,out_channels=3						,down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]						,up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]						,latent_channels=4						,sample_size=128						,)
           torch.manual_seed(0	)
           SCREAMING_SNAKE_CASE		=						CLIPTextConfig(
               bos_token_id=0						,eos_token_id=2						,hidden_size=32						,intermediate_size=37						,layer_norm_eps=1e-0_5						,num_attention_heads=4						,num_hidden_layers=5						,pad_token_id=1						,vocab_size=1000						,hidden_act="""gelu"""						,projection_dim=512						,)
           SCREAMING_SNAKE_CASE		=						CLIPTextModel(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""	)

           SCREAMING_SNAKE_CASE		=						{
               """unet""": unet,
               """scheduler""": scheduler,
               """inverse_scheduler""": inverse_scheduler,
               """vae""": vae,
               """text_encoder""": text_encoder,
               """tokenizer""": tokenizer,
               """safety_checker""": None,
               """feature_extractor""": None,
           }

           return components






     def     SCREAMING_SNAKE_CASE__  (							self						:   int						,lowerCamelCase__						:   Optional[Any]						,lowerCamelCase__						:   Any=0	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						floats_tensor((1, 16, 16)						,rng=random.Random(lowerCamelCase__	)	).to(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						floats_tensor((1, 2, 4, 16, 16)						,rng=random.Random(lowerCamelCase__	)	).to(lowerCamelCase__	)
           if str(lowerCamelCase__	).startswith("""mps"""	):
                 SCREAMING_SNAKE_CASE		=						torch.manual_seed(lowerCamelCase__	)
           else:
                 SCREAMING_SNAKE_CASE		=						torch.Generator(device=lowerCamelCase__	).manual_seed(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						{
               """prompt""": """a dog and a newt""",
               """mask_image""": mask,
               """image_latents""": latents,
               """generator""": generator,
               """num_inference_steps""": 2,
               """inpaint_strength""": 1.0,
               """guidance_scale""": 6.0,
               """output_type""": """numpy""",
           }

           return inputs






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]						,lowerCamelCase__						:   Any						,lowerCamelCase__						:   Dict=0	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						floats_tensor((1, 3, 32, 32)						,rng=random.Random(lowerCamelCase__	)	).to(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						image.cpu().permute(0						,2						,3						,1	)[0]
           SCREAMING_SNAKE_CASE		=						Image.fromarray(np.uinta(lowerCamelCase__	)	).convert("""RGB"""	)
           if str(lowerCamelCase__	).startswith("""mps"""	):
                 SCREAMING_SNAKE_CASE		=						torch.manual_seed(lowerCamelCase__	)
           else:
                 SCREAMING_SNAKE_CASE		=						torch.Generator(device=lowerCamelCase__	).manual_seed(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						{
               """image""": image,
               """source_prompt""": """a cat and a frog""",
               """target_prompt""": """a dog and a newt""",
               """generator""": generator,
               """num_inference_steps""": 2,
               """num_maps_per_mask""": 2,
               """mask_encode_strength""": 1.0,
               """guidance_scale""": 6.0,
               """output_type""": """numpy""",
           }

           return inputs






     def     SCREAMING_SNAKE_CASE__  (							self						:   int						,lowerCamelCase__						:   Optional[Any]						,lowerCamelCase__						:   List[Any]=0	)					->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						floats_tensor((1, 3, 32, 32)						,rng=random.Random(lowerCamelCase__	)	).to(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						image.cpu().permute(0						,2						,3						,1	)[0]
           SCREAMING_SNAKE_CASE		=						Image.fromarray(np.uinta(lowerCamelCase__	)	).convert("""RGB"""	)
           if str(lowerCamelCase__	).startswith("""mps"""	):
                 SCREAMING_SNAKE_CASE		=						torch.manual_seed(lowerCamelCase__	)
           else:
                 SCREAMING_SNAKE_CASE		=						torch.Generator(device=lowerCamelCase__	).manual_seed(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						{
               """image""": image,
               """prompt""": """a cat and a frog""",
               """generator""": generator,
               """num_inference_steps""": 2,
               """inpaint_strength""": 1.0,
               """guidance_scale""": 6.0,
               """decode_latents""": True,
               """output_type""": """numpy""",
           }
           return inputs






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           if not hasattr(self.pipeline_class						,"""_optional_components"""	):
                 return

           SCREAMING_SNAKE_CASE		=						self.get_dummy_components()
           SCREAMING_SNAKE_CASE		=						self.pipeline_class(**lowerCamelCase__	)
           pipe.to(lowerCamelCase__	)
           pipe.set_progress_bar_config(disable=lowerCamelCase__	)

           # set all optional components to None and update pipeline config accordingly
           for optional_component in pipe._optional_components:
                 setattr(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__	)
           pipe.register_modules(**{optional_component: None for optional_component in pipe._optional_components}	)

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inputs(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						pipe(**lowerCamelCase__	)[0]

           with tempfile.TemporaryDirectory() as tmpdir:
                 pipe.save_pretrained(lowerCamelCase__	)
                 SCREAMING_SNAKE_CASE		=						self.pipeline_class.from_pretrained(lowerCamelCase__	)
                 pipe_loaded.to(lowerCamelCase__	)
                 pipe_loaded.set_progress_bar_config(disable=lowerCamelCase__	)

           for optional_component in pipe._optional_components:
                 self.assertTrue(
                     getattr(lowerCamelCase__						,lowerCamelCase__	) is None						,F"""`{optional_component}` did not stay set to None after loading."""						,)

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inputs(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						pipe_loaded(**lowerCamelCase__	)[0]

           SCREAMING_SNAKE_CASE		=						np.abs(output - output_loaded	).max()
           self.assertLess(lowerCamelCase__						,1e-4	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""cpu"""

           SCREAMING_SNAKE_CASE		=						self.get_dummy_components()
           SCREAMING_SNAKE_CASE		=						self.pipeline_class(**lowerCamelCase__	)
           pipe.to(lowerCamelCase__	)
           pipe.set_progress_bar_config(disable=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_dummy_mask_inputs(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						pipe.generate_mask(**lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						mask[0, -3:, -3:]

           self.assertEqual(mask.shape						,(1, 16, 16)	)
           SCREAMING_SNAKE_CASE		=						np.array([0] * 9	)
           SCREAMING_SNAKE_CASE		=						np.abs(mask_slice.flatten() - expected_slice	).max()
           self.assertLessEqual(lowerCamelCase__						,1e-3	)
           self.assertEqual(mask[0, -3, -4]						,0	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""cpu"""

           SCREAMING_SNAKE_CASE		=						self.get_dummy_components()
           SCREAMING_SNAKE_CASE		=						self.pipeline_class(**lowerCamelCase__	)
           pipe.to(lowerCamelCase__	)
           pipe.set_progress_bar_config(disable=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inversion_inputs(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						pipe.invert(**lowerCamelCase__	).images
           SCREAMING_SNAKE_CASE		=						image[0, -1, -3:, -3:]

           self.assertEqual(image.shape						,(2, 32, 32, 3)	)
           SCREAMING_SNAKE_CASE		=						np.array(
               [0.5150, 0.5134, 0.5043, 0.5376, 0.4694, 0.51050, 0.5015, 0.4407, 0.4799]						,)
           SCREAMING_SNAKE_CASE		=						np.abs(image_slice.flatten() - expected_slice	).max()
           self.assertLessEqual(lowerCamelCase__						,1e-3	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		Any:




           '''simple docstring'''


           super().test_inference_batch_single_identical(expected_max_diff=5e-3	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""cpu"""

           SCREAMING_SNAKE_CASE		=						self.get_dummy_components()

           SCREAMING_SNAKE_CASE		=						{"""beta_start""": 0.00085, """beta_end""": 0.012, """beta_schedule""": """scaled_linear"""}
           SCREAMING_SNAKE_CASE		=						DPMSolverMultistepScheduler(**lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						DPMSolverMultistepInverseScheduler(**lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.pipeline_class(**lowerCamelCase__	)
           pipe.to(lowerCamelCase__	)
           pipe.set_progress_bar_config(disable=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_dummy_inversion_inputs(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						pipe.invert(**lowerCamelCase__	).images
           SCREAMING_SNAKE_CASE		=						image[0, -1, -3:, -3:]

           self.assertEqual(image.shape						,(2, 32, 32, 3)	)
           SCREAMING_SNAKE_CASE		=						np.array(
               [0.5150, 0.5134, 0.5043, 0.5376, 0.4694, 0.51050, 0.5015, 0.4407, 0.4799]						,)
           SCREAMING_SNAKE_CASE		=						np.abs(image_slice.flatten() - expected_slice	).max()
           self.assertLessEqual(lowerCamelCase__						,1e-3	)

@require_torch_gpu
@slow
class 		UpperCamelCase__			(				unittest.TestCase       ):



     '''simple docstring'''

     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		str:




           '''simple docstring'''


           super().tearDown()
           gc.collect()
           torch.cuda.empty_cache()






     @classmethod
     def     SCREAMING_SNAKE_CASE__  (							cls						:   List[Any]	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						load_image(
               """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/diffedit/fruit.png"""	)

           SCREAMING_SNAKE_CASE		=						raw_image.convert("""RGB"""	).resize((768, 768)	)

           SCREAMING_SNAKE_CASE		=						raw_image






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						torch.manual_seed(0	)

           SCREAMING_SNAKE_CASE		=						StableDiffusionDiffEditPipeline.from_pretrained(
               """stabilityai/stable-diffusion-2-1"""						,safety_checker=lowerCamelCase__						,torch_dtype=torch.floataa	)
           SCREAMING_SNAKE_CASE		=						DDIMScheduler.from_config(pipe.scheduler.config	)
           SCREAMING_SNAKE_CASE		=						DDIMInverseScheduler.from_config(pipe.scheduler.config	)
           pipe.enable_model_cpu_offload()
           pipe.set_progress_bar_config(disable=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						"""a bowl of fruit"""
           SCREAMING_SNAKE_CASE		=						"""a bowl of pears"""

           SCREAMING_SNAKE_CASE		=						pipe.generate_mask(
               image=self.raw_image						,source_prompt=lowerCamelCase__						,target_prompt=lowerCamelCase__						,generator=lowerCamelCase__						,)

           SCREAMING_SNAKE_CASE		=						pipe.invert(
               prompt=lowerCamelCase__						,image=self.raw_image						,inpaint_strength=0.7						,generator=lowerCamelCase__	).latents

           SCREAMING_SNAKE_CASE		=						pipe(
               prompt=lowerCamelCase__						,mask_image=lowerCamelCase__						,image_latents=lowerCamelCase__						,generator=lowerCamelCase__						,negative_prompt=lowerCamelCase__						,inpaint_strength=0.7						,output_type="""numpy"""						,).images[0]

           SCREAMING_SNAKE_CASE		=						(
               np.array(
                   load_image(
                       """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
                       """/diffedit/pears.png"""	).resize((768, 768)	)	)
               / 255
           )
           assert np.abs((expected_image - image).max()	) < 5e-1






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[Any]	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						torch.manual_seed(0	)

           SCREAMING_SNAKE_CASE		=						StableDiffusionDiffEditPipeline.from_pretrained(
               """stabilityai/stable-diffusion-2-1"""						,safety_checker=lowerCamelCase__						,torch_dtype=torch.floataa	)
           SCREAMING_SNAKE_CASE		=						DPMSolverMultistepScheduler.from_config(pipe.scheduler.config	)
           SCREAMING_SNAKE_CASE		=						DPMSolverMultistepInverseScheduler.from_config(pipe.scheduler.config	)
           pipe.enable_model_cpu_offload()
           pipe.set_progress_bar_config(disable=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						"""a bowl of fruit"""
           SCREAMING_SNAKE_CASE		=						"""a bowl of pears"""

           SCREAMING_SNAKE_CASE		=						pipe.generate_mask(
               image=self.raw_image						,source_prompt=lowerCamelCase__						,target_prompt=lowerCamelCase__						,generator=lowerCamelCase__						,)

           SCREAMING_SNAKE_CASE		=						pipe.invert(
               prompt=lowerCamelCase__						,image=self.raw_image						,inpaint_strength=0.7						,generator=lowerCamelCase__						,num_inference_steps=25						,).latents

           SCREAMING_SNAKE_CASE		=						pipe(
               prompt=lowerCamelCase__						,mask_image=lowerCamelCase__						,image_latents=lowerCamelCase__						,generator=lowerCamelCase__						,negative_prompt=lowerCamelCase__						,inpaint_strength=0.7						,num_inference_steps=25						,output_type="""numpy"""						,).images[0]

           SCREAMING_SNAKE_CASE		=						(
               np.array(
                   load_image(
                       """https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main"""
                       """/diffedit/pears.png"""	).resize((768, 768)	)	)
               / 255
           )
           assert np.abs((expected_image - image).max()	) < 5e-1









import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 		UpperCamelCase__			(				lowerCAmelCase_							,				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     Any		     =      BertTokenizer
     __snake_case   :     Dict		     =      BertTokenizerFast
     __snake_case   :     Tuple		     =      True
     __snake_case   :     List[Any]		     =      True
     __snake_case   :     Optional[Any]		     =      filter_non_english






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		Any:




           '''simple docstring'''


           super().setUp()

           SCREAMING_SNAKE_CASE		=						[
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """[PAD]""",
               """[MASK]""",
               """want""",
               """##want""",
               """##ed""",
               """wa""",
               """un""",
               """runn""",
               """##ing""",
               """,""",
               """low""",
               """lowest""",
           ]
           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""vocab_file"""]	)
           with open(self.vocab_file						,"""w"""						,encoding="""utf-8"""	) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str						,lowerCamelCase__						:   List[Any]	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""
           SCREAMING_SNAKE_CASE		=						"""unwanted, running"""
           return input_text, output_text






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer_class(self.vocab_file	)

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize("""UNwant\u00E9d,running"""	)
           self.assertListEqual(lowerCamelCase__						,["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__	)						,[9, 6, 7, 12, 10, 11]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		List[Any]:




           '''simple docstring'''


           if not self.test_rust_tokenizer:
                 return

           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()

           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           # With lower casing
           SCREAMING_SNAKE_CASE		=						self.get_tokenizer(do_lower_case=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer(do_lower_case=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""	)						,["""ah""", """\u535A""", """\u63A8""", """zz"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)						,["""hello""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hällo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""h\u00E9llo"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)						,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,never_split=["""[UNK]"""]	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""	)						,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer()
           SCREAMING_SNAKE_CASE		=						"""a\n'll !!to?'d of, can't."""
           SCREAMING_SNAKE_CASE		=						["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
           self.assertListEqual(tokenizer.tokenize(lowerCamelCase__	)						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

           SCREAMING_SNAKE_CASE		=						{}
           for i, token in enumerate(lowerCamelCase__	):
                 SCREAMING_SNAKE_CASE		=						i
           SCREAMING_SNAKE_CASE		=						WordpieceTokenizer(vocab=lowerCamelCase__						,unk_token="""[UNK]"""	)

           self.assertListEqual(tokenizer.tokenize(""""""	)						,[]	)

           self.assertListEqual(tokenizer.tokenize("""unwanted running"""	)						,["""un""", """##want""", """##ed""", """runn""", """##ing"""]	)

           self.assertListEqual(tokenizer.tokenize("""unwantedX running"""	)						,["""[UNK]""", """runn""", """##ing"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Dict:




           '''simple docstring'''


           self.assertTrue(_is_whitespace(""" """	)	)
           self.assertTrue(_is_whitespace("""\t"""	)	)
           self.assertTrue(_is_whitespace("""\r"""	)	)
           self.assertTrue(_is_whitespace("""\n"""	)	)
           self.assertTrue(_is_whitespace("""\u00A0"""	)	)

           self.assertFalse(_is_whitespace("""A"""	)	)
           self.assertFalse(_is_whitespace("""-"""	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[int]:




           '''simple docstring'''


           self.assertTrue(_is_control("""\u0005"""	)	)

           self.assertFalse(_is_control("""A"""	)	)
           self.assertFalse(_is_control(""" """	)	)
           self.assertFalse(_is_control("""\t"""	)	)
           self.assertFalse(_is_control("""\r"""	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple	)					->		List[str]:




           '''simple docstring'''


           self.assertTrue(_is_punctuation("""-"""	)	)
           self.assertTrue(_is_punctuation("""$"""	)	)
           self.assertTrue(_is_punctuation("""`"""	)	)
           self.assertTrue(_is_punctuation("""."""	)	)

           self.assertFalse(_is_punctuation("""A"""	)	)
           self.assertFalse(_is_punctuation(""" """	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowerCamelCase__	) for t in ["""Test""", """\xad""", """test"""]]						,[["""[UNK]"""], [], ["""[UNK]"""]]	)

           self.assertListEqual(
               [rust_tokenizer.tokenize(lowerCamelCase__	) for t in ["""Test""", """\xad""", """test"""]]						,[["""[UNK]"""], [], ["""[UNK]"""]]	)






     @slow
     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained("""bert-base-uncased"""	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode("""sequence builders"""						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tokenizer.encode("""multi-sequence build"""						,add_special_tokens=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.build_inputs_with_special_tokens(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tokenizer.build_inputs_with_special_tokens(lowerCamelCase__						,lowerCamelCase__	)

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		str:




           '''simple docstring'''


           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode_plus(
                           lowerCamelCase__						,return_attention_mask=lowerCamelCase__						,return_token_type_ids=lowerCamelCase__						,return_offsets_mapping=lowerCamelCase__						,add_special_tokens=lowerCamelCase__						,)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.do_lower_case if hasattr(lowerCamelCase__						,"""do_lower_case"""	) else False
                       SCREAMING_SNAKE_CASE		=						(
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """A"""),
                               ((1, 2), ""","""),
                               ((3, 5), """na"""),
                               ((5, 6), """##ï"""),
                               ((6, 8), """##ve"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """Allen"""),
                               ((21, 23), """##NL"""),
                               ((23, 24), """##P"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """a"""),
                               ((1, 2), ""","""),
                               ((3, 8), """naive"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """allen"""),
                               ((21, 23), """##nl"""),
                               ((23, 24), """##p"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]						,tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]	)	)
                       self.assertEqual([e[0] for e in expected_results]						,tokens["""offset_mapping"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						["""的""", """人""", """有"""]
           SCREAMING_SNAKE_CASE		=						"""""".join(lowerCamelCase__	)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						True
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_p.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.convert_ids_to_tokens(lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.convert_ids_to_tokens(lowerCamelCase__	)

                       # it is expected that each Chinese character is not preceded by "##"
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						False
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.convert_ids_to_tokens(lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.convert_ids_to_tokens(lowerCamelCase__	)

                       # it is expected that only the first Chinese character is not preceded by "##".
                       SCREAMING_SNAKE_CASE		=						[
                           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__	)
                       ]
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)


