





import math
from collections import defaultdict
from typing import List, Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput



def 			__lowercase       (				_SCREAMING_SNAKE_CASE			,						_SCREAMING_SNAKE_CASE=0.999			,						_SCREAMING_SNAKE_CASE="cosine"			,						)			->       int:


      '''simple docstring'''


      if alpha_transform_type == "cosine":

            def alpha_bar_fn(_SCREAMING_SNAKE_CASE     ):
                  return math.cos((t + 0.008) / 1.008 * math.pi / 2     ) ** 2

      elif alpha_transform_type == "exp":

            def alpha_bar_fn(_SCREAMING_SNAKE_CASE     ):
                  return math.exp(t * -12.0     )

      else:
            raise ValueError(F"""Unsupported alpha_tranform_type: {alpha_transform_type}"""     )

      SCREAMING_SNAKE_CASE		=						[]
      for i in range(_SCREAMING_SNAKE_CASE     ):
            SCREAMING_SNAKE_CASE		=						i / num_diffusion_timesteps
            SCREAMING_SNAKE_CASE		=						(i + 1) / num_diffusion_timesteps
            betas.append(min(1 - alpha_bar_fn(_SCREAMING_SNAKE_CASE     ) / alpha_bar_fn(_SCREAMING_SNAKE_CASE     )			,						_SCREAMING_SNAKE_CASE     )     )
      return torch.tensor(_SCREAMING_SNAKE_CASE			,						dtype=torch.floataa     )



class 		UpperCamelCase__			(				lowerCAmelCase_							,				lowerCAmelCase_       ):



     '''simple docstring'''

     __snake_case   :     Optional[Any]		     =      [e.name for e in KarrasDiffusionSchedulers]
     __snake_case   :     Any		     =      2






     @register_to_config
     def __init__(							self						:   Any						,lowerCamelCase__						:   int = 1000						,lowerCamelCase__						:   float = 0.00085						,lowerCamelCase__						:   float = 0.012						,lowerCamelCase__						:   str = "linear"						,lowerCamelCase__						:   Optional[Union[np.ndarray, List[float]]] = None						,lowerCamelCase__						:   str = "epsilon"						,lowerCamelCase__						:   Optional[bool] = False						,lowerCamelCase__						:   Optional[bool] = False						,lowerCamelCase__						:   float = 1.0						,lowerCamelCase__						:   str = "linspace"						,lowerCamelCase__						:   int = 0						,)					->		Tuple:




           '''simple docstring'''


           if trained_betas is not None:
                 SCREAMING_SNAKE_CASE		=						torch.tensor(lowerCamelCase__						,dtype=torch.floataa	)
           elif beta_schedule == "linear":
                 SCREAMING_SNAKE_CASE		=						torch.linspace(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__						,dtype=torch.floataa	)
           elif beta_schedule == "scaled_linear":
                 # this schedule is very specific to the latent diffusion model.
                 SCREAMING_SNAKE_CASE		=						(
                     torch.linspace(beta_start**0.5						,beta_end**0.5						,lowerCamelCase__						,dtype=torch.floataa	) ** 2
                 )
           elif beta_schedule == "squaredcos_cap_v2":
                 # Glide cosine schedule
                 SCREAMING_SNAKE_CASE		=						betas_for_alpha_bar(lowerCamelCase__						,alpha_transform_type="""cosine"""	)
           elif beta_schedule == "exp":
                 SCREAMING_SNAKE_CASE		=						betas_for_alpha_bar(lowerCamelCase__						,alpha_transform_type="""exp"""	)
           else:
                 raise NotImplementedError(F"""{beta_schedule} does is not implemented for {self.__class__}"""	)

           SCREAMING_SNAKE_CASE		=						1.0 - self.betas
           SCREAMING_SNAKE_CASE		=						torch.cumprod(self.alphas						,dim=0	)

           #  set all values
           self.set_timesteps(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						use_karras_sigmas






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]						,lowerCamelCase__						:   Union[str, Any]						,lowerCamelCase__						:   Tuple=None	)					->		str:




           '''simple docstring'''


           if schedule_timesteps is None:
                 SCREAMING_SNAKE_CASE		=						self.timesteps

           SCREAMING_SNAKE_CASE		=						(schedule_timesteps == timestep).nonzero()

           # The sigma index that is taken for the **very** first `step`
           # is always the second index (or the last index if there is only 1)
           # This way we can ensure we don't accidentally skip a sigma in
           # case we start in the middle of the denoising schedule (e.g. for image-to-image)
           if len(self._index_counter	) == 0:
                 SCREAMING_SNAKE_CASE		=						1 if len(lowerCamelCase__	) > 1 else 0
           else:
                 SCREAMING_SNAKE_CASE		=						timestep.cpu().item() if torch.is_tensor(lowerCamelCase__	) else timestep
                 SCREAMING_SNAKE_CASE		=						self._index_counter[timestep_int]

           return indices[pos].item()






     @property
     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		List[str]:




           '''simple docstring'''


           if self.config.timestep_spacing in ["linspace", "trailing"]:
                 return self.sigmas.max()

           return (self.sigmas.max() ** 2 + 1) ** 0.5






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict						,lowerCamelCase__						:   torch.FloatTensor						,lowerCamelCase__						:   Union[float, torch.FloatTensor]						,)					->		torch.FloatTensor:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.index_for_timestep(lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.sigmas[step_index]
           SCREAMING_SNAKE_CASE		=						sample / ((sigma**2 + 1) ** 0.5)
           return sample






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any						,lowerCamelCase__						:   int						,lowerCamelCase__						:   Union[str, torch.device] = None						,lowerCamelCase__						:   Optional[int] = None						,)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						num_inference_steps

           SCREAMING_SNAKE_CASE		=						num_train_timesteps or self.config.num_train_timesteps

           # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
           if self.config.timestep_spacing == "linspace":
                 SCREAMING_SNAKE_CASE		=						np.linspace(0						,num_train_timesteps - 1						,lowerCamelCase__						,dtype=lowerCamelCase__	)[::-1].copy()
           elif self.config.timestep_spacing == "leading":
                 SCREAMING_SNAKE_CASE		=						num_train_timesteps // self.num_inference_steps
                 # creates integer timesteps by multiplying by ratio
                 # casting to int to avoid issues when num_inference_step is power of 3
                 SCREAMING_SNAKE_CASE		=						(np.arange(0						,lowerCamelCase__	) * step_ratio).round()[::-1].copy().astype(lowerCamelCase__	)
                 timesteps += self.config.steps_offset
           elif self.config.timestep_spacing == "trailing":
                 SCREAMING_SNAKE_CASE		=						num_train_timesteps / self.num_inference_steps
                 # creates integer timesteps by multiplying by ratio
                 # casting to int to avoid issues when num_inference_step is power of 3
                 SCREAMING_SNAKE_CASE		=						(np.arange(lowerCamelCase__						,0						,-step_ratio	)).round().copy().astype(lowerCamelCase__	)
                 timesteps -= 1
           else:
                 raise ValueError(
                     F"""{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'."""	)

           SCREAMING_SNAKE_CASE		=						np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5	)
           SCREAMING_SNAKE_CASE		=						np.log(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						np.interp(lowerCamelCase__						,np.arange(0						,len(lowerCamelCase__	)	)						,lowerCamelCase__	)

           if self.config.use_karras_sigmas:
                 SCREAMING_SNAKE_CASE		=						self._convert_to_karras(in_sigmas=lowerCamelCase__						,num_inference_steps=self.num_inference_steps	)
                 SCREAMING_SNAKE_CASE		=						np.array([self._sigma_to_t(lowerCamelCase__						,lowerCamelCase__	) for sigma in sigmas]	)

           SCREAMING_SNAKE_CASE		=						np.concatenate([sigmas, [0.0]]	).astype(np.floataa	)
           SCREAMING_SNAKE_CASE		=						torch.from_numpy(lowerCamelCase__	).to(device=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						torch.cat([sigmas[:1], sigmas[1:-1].repeat_interleave(2	), sigmas[-1:]]	)

           SCREAMING_SNAKE_CASE		=						torch.from_numpy(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						torch.cat([timesteps[:1], timesteps[1:].repeat_interleave(2	)]	)

           if str(lowerCamelCase__	).startswith("""mps"""	):
                 # mps does not support float64
                 SCREAMING_SNAKE_CASE		=						timesteps.to(lowerCamelCase__						,dtype=torch.floataa	)
           else:
                 SCREAMING_SNAKE_CASE		=						timesteps.to(device=lowerCamelCase__	)

           # empty dt and derivative
           SCREAMING_SNAKE_CASE		=						None
           SCREAMING_SNAKE_CASE		=						None

           # for exp beta schedules, such as the one for `pipeline_shap_e.py`
           # we need an index counter
           SCREAMING_SNAKE_CASE		=						defaultdict(lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   int						,lowerCamelCase__						:   Dict	)					->		List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						np.log(lowerCamelCase__	)

           # get distribution
           SCREAMING_SNAKE_CASE		=						log_sigma - log_sigmas[:, np.newaxis]

           # get sigmas range
           SCREAMING_SNAKE_CASE		=						np.cumsum((dists >= 0)						,axis=0	).argmax(axis=0	).clip(max=log_sigmas.shape[0] - 2	)
           SCREAMING_SNAKE_CASE		=						low_idx + 1

           SCREAMING_SNAKE_CASE		=						log_sigmas[low_idx]
           SCREAMING_SNAKE_CASE		=						log_sigmas[high_idx]

           # interpolate sigmas
           SCREAMING_SNAKE_CASE		=						(low - log_sigma) / (low - high)
           SCREAMING_SNAKE_CASE		=						np.clip(lowerCamelCase__						,0						,1	)

           # transform interpolation to time range
           SCREAMING_SNAKE_CASE		=						(1 - w) * low_idx + w * high_idx
           SCREAMING_SNAKE_CASE		=						t.reshape(sigma.shape	)
           return t






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   torch.FloatTensor						,lowerCamelCase__						:   int	)					->		torch.FloatTensor:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						in_sigmas[-1].item()
           SCREAMING_SNAKE_CASE		=						in_sigmas[0].item()

           SCREAMING_SNAKE_CASE		=						7.0  # 7.0 is the value used in the paper
           SCREAMING_SNAKE_CASE		=						np.linspace(0						,1						,lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						sigma_min ** (1 / rho)
           SCREAMING_SNAKE_CASE		=						sigma_max ** (1 / rho)
           SCREAMING_SNAKE_CASE		=						(max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
           return sigmas






     @property
     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Optional[Any]:




           '''simple docstring'''


           return self.dt is None






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple						,lowerCamelCase__						:   Union[torch.FloatTensor, np.ndarray]						,lowerCamelCase__						:   Union[float, torch.FloatTensor]						,lowerCamelCase__						:   Union[torch.FloatTensor, np.ndarray]						,lowerCamelCase__						:   bool = True						,)					->		Union[SchedulerOutput, Tuple]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.index_for_timestep(lowerCamelCase__	)

           # advance index counter by 1
           SCREAMING_SNAKE_CASE		=						timestep.cpu().item() if torch.is_tensor(lowerCamelCase__	) else timestep
           self._index_counter[timestep_int] += 1

           if self.state_in_first_order:
                 SCREAMING_SNAKE_CASE		=						self.sigmas[step_index]
                 SCREAMING_SNAKE_CASE		=						self.sigmas[step_index + 1]
           else:
                 # 2nd order / Heun's method
                 SCREAMING_SNAKE_CASE		=						self.sigmas[step_index - 1]
                 SCREAMING_SNAKE_CASE		=						self.sigmas[step_index]

           # currently only gamma=0 is supported. This usually works best anyways.
           # We can support gamma in the future but then need to scale the timestep before
           # passing it to the model which requires a change in API
           SCREAMING_SNAKE_CASE		=						0
           SCREAMING_SNAKE_CASE		=						sigma * (gamma + 1)  # Note: sigma_hat == sigma for now

           # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
           if self.config.prediction_type == "epsilon":
                 SCREAMING_SNAKE_CASE		=						sigma_hat if self.state_in_first_order else sigma_next
                 SCREAMING_SNAKE_CASE		=						sample - sigma_input * model_output
           elif self.config.prediction_type == "v_prediction":
                 SCREAMING_SNAKE_CASE		=						sigma_hat if self.state_in_first_order else sigma_next
                 SCREAMING_SNAKE_CASE		=						model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + (
                     sample / (sigma_input**2 + 1)
                 )
           elif self.config.prediction_type == "sample":
                 SCREAMING_SNAKE_CASE		=						model_output
           else:
                 raise ValueError(
                     F"""prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`"""	)

           if self.config.clip_sample:
                 SCREAMING_SNAKE_CASE		=						pred_original_sample.clamp(
                     -self.config.clip_sample_range						,self.config.clip_sample_range	)

           if self.state_in_first_order:
                 # 2. Convert to an ODE derivative for 1st order
                 SCREAMING_SNAKE_CASE		=						(sample - pred_original_sample) / sigma_hat
                 # 3. delta timestep
                 SCREAMING_SNAKE_CASE		=						sigma_next - sigma_hat

                 # store for 2nd order step
                 SCREAMING_SNAKE_CASE		=						derivative
                 SCREAMING_SNAKE_CASE		=						dt
                 SCREAMING_SNAKE_CASE		=						sample
           else:
                 # 2. 2nd order / Heun's method
                 SCREAMING_SNAKE_CASE		=						(sample - pred_original_sample) / sigma_next
                 SCREAMING_SNAKE_CASE		=						(self.prev_derivative + derivative) / 2

                 # 3. take prev timestep & sample
                 SCREAMING_SNAKE_CASE		=						self.dt
                 SCREAMING_SNAKE_CASE		=						self.sample

                 # free dt and derivative
                 # Note, this puts the scheduler in "first order mode"
                 SCREAMING_SNAKE_CASE		=						None
                 SCREAMING_SNAKE_CASE		=						None
                 SCREAMING_SNAKE_CASE		=						None

           SCREAMING_SNAKE_CASE		=						sample + derivative * dt

           if not return_dict:
                 return (prev_sample,)

           return SchedulerOutput(prev_sample=lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]						,lowerCamelCase__						:   torch.FloatTensor						,lowerCamelCase__						:   torch.FloatTensor						,lowerCamelCase__						:   torch.FloatTensor						,)					->		torch.FloatTensor:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.sigmas.to(device=original_samples.device						,dtype=original_samples.dtype	)
           if original_samples.device.type == "mps" and torch.is_floating_point(lowerCamelCase__	):
                 # mps does not support float64
                 SCREAMING_SNAKE_CASE		=						self.timesteps.to(original_samples.device						,dtype=torch.floataa	)
                 SCREAMING_SNAKE_CASE		=						timesteps.to(original_samples.device						,dtype=torch.floataa	)
           else:
                 SCREAMING_SNAKE_CASE		=						self.timesteps.to(original_samples.device	)
                 SCREAMING_SNAKE_CASE		=						timesteps.to(original_samples.device	)

           SCREAMING_SNAKE_CASE		=						[self.index_for_timestep(lowerCamelCase__						,lowerCamelCase__	) for t in timesteps]

           SCREAMING_SNAKE_CASE		=						sigmas[step_indices].flatten()
           while len(sigma.shape	) < len(original_samples.shape	):
                 SCREAMING_SNAKE_CASE		=						sigma.unsqueeze(-1	)

           SCREAMING_SNAKE_CASE		=						original_samples + noise * sigma
           return noisy_samples






     def __len__(							self						:   List[Any]	)					->		int:




           '''simple docstring'''


           return self.config.num_train_timesteps









import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 		UpperCamelCase__			(				lowerCAmelCase_							,				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     Any		     =      BertTokenizer
     __snake_case   :     Dict		     =      BertTokenizerFast
     __snake_case   :     Tuple		     =      True
     __snake_case   :     List[Any]		     =      True
     __snake_case   :     Optional[Any]		     =      filter_non_english






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		Any:




           '''simple docstring'''


           super().setUp()

           SCREAMING_SNAKE_CASE		=						[
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """[PAD]""",
               """[MASK]""",
               """want""",
               """##want""",
               """##ed""",
               """wa""",
               """un""",
               """runn""",
               """##ing""",
               """,""",
               """low""",
               """lowest""",
           ]
           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""vocab_file"""]	)
           with open(self.vocab_file						,"""w"""						,encoding="""utf-8"""	) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str						,lowerCamelCase__						:   List[Any]	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""
           SCREAMING_SNAKE_CASE		=						"""unwanted, running"""
           return input_text, output_text






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer_class(self.vocab_file	)

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize("""UNwant\u00E9d,running"""	)
           self.assertListEqual(lowerCamelCase__						,["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__	)						,[9, 6, 7, 12, 10, 11]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		List[Any]:




           '''simple docstring'''


           if not self.test_rust_tokenizer:
                 return

           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()

           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           # With lower casing
           SCREAMING_SNAKE_CASE		=						self.get_tokenizer(do_lower_case=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer(do_lower_case=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""	)						,["""ah""", """\u535A""", """\u63A8""", """zz"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)						,["""hello""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hällo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""h\u00E9llo"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)						,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,never_split=["""[UNK]"""]	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""	)						,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer()
           SCREAMING_SNAKE_CASE		=						"""a\n'll !!to?'d of, can't."""
           SCREAMING_SNAKE_CASE		=						["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
           self.assertListEqual(tokenizer.tokenize(lowerCamelCase__	)						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

           SCREAMING_SNAKE_CASE		=						{}
           for i, token in enumerate(lowerCamelCase__	):
                 SCREAMING_SNAKE_CASE		=						i
           SCREAMING_SNAKE_CASE		=						WordpieceTokenizer(vocab=lowerCamelCase__						,unk_token="""[UNK]"""	)

           self.assertListEqual(tokenizer.tokenize(""""""	)						,[]	)

           self.assertListEqual(tokenizer.tokenize("""unwanted running"""	)						,["""un""", """##want""", """##ed""", """runn""", """##ing"""]	)

           self.assertListEqual(tokenizer.tokenize("""unwantedX running"""	)						,["""[UNK]""", """runn""", """##ing"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Dict:




           '''simple docstring'''


           self.assertTrue(_is_whitespace(""" """	)	)
           self.assertTrue(_is_whitespace("""\t"""	)	)
           self.assertTrue(_is_whitespace("""\r"""	)	)
           self.assertTrue(_is_whitespace("""\n"""	)	)
           self.assertTrue(_is_whitespace("""\u00A0"""	)	)

           self.assertFalse(_is_whitespace("""A"""	)	)
           self.assertFalse(_is_whitespace("""-"""	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[int]:




           '''simple docstring'''


           self.assertTrue(_is_control("""\u0005"""	)	)

           self.assertFalse(_is_control("""A"""	)	)
           self.assertFalse(_is_control(""" """	)	)
           self.assertFalse(_is_control("""\t"""	)	)
           self.assertFalse(_is_control("""\r"""	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple	)					->		List[str]:




           '''simple docstring'''


           self.assertTrue(_is_punctuation("""-"""	)	)
           self.assertTrue(_is_punctuation("""$"""	)	)
           self.assertTrue(_is_punctuation("""`"""	)	)
           self.assertTrue(_is_punctuation("""."""	)	)

           self.assertFalse(_is_punctuation("""A"""	)	)
           self.assertFalse(_is_punctuation(""" """	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowerCamelCase__	) for t in ["""Test""", """\xad""", """test"""]]						,[["""[UNK]"""], [], ["""[UNK]"""]]	)

           self.assertListEqual(
               [rust_tokenizer.tokenize(lowerCamelCase__	) for t in ["""Test""", """\xad""", """test"""]]						,[["""[UNK]"""], [], ["""[UNK]"""]]	)






     @slow
     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained("""bert-base-uncased"""	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode("""sequence builders"""						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tokenizer.encode("""multi-sequence build"""						,add_special_tokens=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.build_inputs_with_special_tokens(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tokenizer.build_inputs_with_special_tokens(lowerCamelCase__						,lowerCamelCase__	)

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		str:




           '''simple docstring'''


           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode_plus(
                           lowerCamelCase__						,return_attention_mask=lowerCamelCase__						,return_token_type_ids=lowerCamelCase__						,return_offsets_mapping=lowerCamelCase__						,add_special_tokens=lowerCamelCase__						,)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.do_lower_case if hasattr(lowerCamelCase__						,"""do_lower_case"""	) else False
                       SCREAMING_SNAKE_CASE		=						(
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """A"""),
                               ((1, 2), ""","""),
                               ((3, 5), """na"""),
                               ((5, 6), """##ï"""),
                               ((6, 8), """##ve"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """Allen"""),
                               ((21, 23), """##NL"""),
                               ((23, 24), """##P"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """a"""),
                               ((1, 2), ""","""),
                               ((3, 8), """naive"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """allen"""),
                               ((21, 23), """##nl"""),
                               ((23, 24), """##p"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]						,tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]	)	)
                       self.assertEqual([e[0] for e in expected_results]						,tokens["""offset_mapping"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						["""的""", """人""", """有"""]
           SCREAMING_SNAKE_CASE		=						"""""".join(lowerCamelCase__	)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						True
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_p.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.convert_ids_to_tokens(lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.convert_ids_to_tokens(lowerCamelCase__	)

                       # it is expected that each Chinese character is not preceded by "##"
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						False
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.convert_ids_to_tokens(lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.convert_ids_to_tokens(lowerCamelCase__	)

                       # it is expected that only the first Chinese character is not preceded by "##".
                       SCREAMING_SNAKE_CASE		=						[
                           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__	)
                       ]
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)


