





from typing import Optional

import torch
import torch.utils.checkpoint
from torch import Tensor, nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...file_utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward
from ...modeling_outputs import (
    BaseModelOutputWithNoAttention,
    BaseModelOutputWithPoolingAndNoAttention,
    ImageClassifierOutputWithNoAttention,
)
from ...modeling_utils import PreTrainedModel
from ...utils import logging
from .configuration_regnet import RegNetConfig


SCREAMING_SNAKE_CASE_						       =	logging.get_logger(__name__)

# General docstring
SCREAMING_SNAKE_CASE_						       =	"""RegNetConfig"""

# Base docstring
SCREAMING_SNAKE_CASE_						       =	"""facebook/regnet-y-040"""
SCREAMING_SNAKE_CASE_						       =	[1, 1_0_8_8, 7, 7]

# Image classification docstring
SCREAMING_SNAKE_CASE_						       =	"""facebook/regnet-y-040"""
SCREAMING_SNAKE_CASE_						       =	"""tabby, tabby cat"""

SCREAMING_SNAKE_CASE_						       =	[
    """facebook/regnet-y-040""",
    # See all regnet models at https://huggingface.co/models?filter=regnet
]

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   str						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int = 3						,lowerCamelCase__						:   int = 1						,lowerCamelCase__						:   int = 1						,lowerCamelCase__						:   Optional[str] = "relu"						,)					->		Union[str, Any]:




           '''simple docstring'''


           super().__init__()
           SCREAMING_SNAKE_CASE		=						nn.Convad(
               lowerCamelCase__						,lowerCamelCase__						,kernel_size=lowerCamelCase__						,stride=lowerCamelCase__						,padding=kernel_size // 2						,groups=lowerCamelCase__						,bias=lowerCamelCase__						,)
           SCREAMING_SNAKE_CASE		=						nn.BatchNormad(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						ACTaFN[activation] if activation is not None else nn.Identity()






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]						,lowerCamelCase__						:   Tuple	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.convolution(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.normalization(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.activation(lowerCamelCase__	)
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   Optional[Any]						,lowerCamelCase__						:   RegNetConfig	)					->		List[str]:




           '''simple docstring'''


           super().__init__()
           SCREAMING_SNAKE_CASE		=						RegNetConvLayer(
               config.num_channels						,config.embedding_size						,kernel_size=3						,stride=2						,activation=config.hidden_act	)
           SCREAMING_SNAKE_CASE		=						config.num_channels






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]						,lowerCamelCase__						:   Optional[int]	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						pixel_values.shape[1]
           if num_channels != self.num_channels:
                 raise ValueError(
                     """Make sure that the channel dimension of the pixel values match with the one set in the configuration."""	)
           SCREAMING_SNAKE_CASE		=						self.embedder(lowerCamelCase__	)
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   List[Any]						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int = 2	)					->		List[str]:




           '''simple docstring'''


           super().__init__()
           SCREAMING_SNAKE_CASE		=						nn.Convad(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1						,stride=lowerCamelCase__						,bias=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						nn.BatchNormad(lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str						,lowerCamelCase__						:   Tensor	)					->		Tensor:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.convolution(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.normalization(lowerCamelCase__	)
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   List[Any]						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int	)					->		int:




           '''simple docstring'''


           super().__init__()

           SCREAMING_SNAKE_CASE		=						nn.AdaptiveAvgPoolad((1, 1)	)
           SCREAMING_SNAKE_CASE		=						nn.Sequential(
               nn.Convad(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1	)						,nn.ReLU()						,nn.Convad(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1	)						,nn.Sigmoid()						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]						,lowerCamelCase__						:   Any	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.pooler(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.attention(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						hidden_state * attention
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   Optional[int]						,lowerCamelCase__						:   RegNetConfig						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int = 1	)					->		str:




           '''simple docstring'''


           super().__init__()
           SCREAMING_SNAKE_CASE		=						in_channels != out_channels or stride != 1
           SCREAMING_SNAKE_CASE		=						max(1						,out_channels // config.groups_width	)
           SCREAMING_SNAKE_CASE		=						(
               RegNetShortCut(lowerCamelCase__						,lowerCamelCase__						,stride=lowerCamelCase__	) if should_apply_shortcut else nn.Identity()
           )
           SCREAMING_SNAKE_CASE		=						nn.Sequential(
               RegNetConvLayer(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1						,activation=config.hidden_act	)						,RegNetConvLayer(lowerCamelCase__						,lowerCamelCase__						,stride=lowerCamelCase__						,groups=lowerCamelCase__						,activation=config.hidden_act	)						,RegNetConvLayer(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1						,activation=lowerCamelCase__	)						,)
           SCREAMING_SNAKE_CASE		=						ACTaFN[config.hidden_act]






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict						,lowerCamelCase__						:   Optional[Any]	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						hidden_state
           SCREAMING_SNAKE_CASE		=						self.layer(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.shortcut(lowerCamelCase__	)
           hidden_state += residual
           SCREAMING_SNAKE_CASE		=						self.activation(lowerCamelCase__	)
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   Dict						,lowerCamelCase__						:   RegNetConfig						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int = 1	)					->		Optional[int]:




           '''simple docstring'''


           super().__init__()
           SCREAMING_SNAKE_CASE		=						in_channels != out_channels or stride != 1
           SCREAMING_SNAKE_CASE		=						max(1						,out_channels // config.groups_width	)
           SCREAMING_SNAKE_CASE		=						(
               RegNetShortCut(lowerCamelCase__						,lowerCamelCase__						,stride=lowerCamelCase__	) if should_apply_shortcut else nn.Identity()
           )
           SCREAMING_SNAKE_CASE		=						nn.Sequential(
               RegNetConvLayer(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1						,activation=config.hidden_act	)						,RegNetConvLayer(lowerCamelCase__						,lowerCamelCase__						,stride=lowerCamelCase__						,groups=lowerCamelCase__						,activation=config.hidden_act	)						,RegNetSELayer(lowerCamelCase__						,reduced_channels=int(round(in_channels / 4	)	)	)						,RegNetConvLayer(lowerCamelCase__						,lowerCamelCase__						,kernel_size=1						,activation=lowerCamelCase__	)						,)
           SCREAMING_SNAKE_CASE		=						ACTaFN[config.hidden_act]






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict						,lowerCamelCase__						:   Tuple	)					->		Tuple:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						hidden_state
           SCREAMING_SNAKE_CASE		=						self.layer(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.shortcut(lowerCamelCase__	)
           hidden_state += residual
           SCREAMING_SNAKE_CASE		=						self.activation(lowerCamelCase__	)
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   int						,lowerCamelCase__						:   RegNetConfig						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int						,lowerCamelCase__						:   int = 2						,lowerCamelCase__						:   int = 2						,)					->		Tuple:




           '''simple docstring'''


           super().__init__()

           SCREAMING_SNAKE_CASE		=						RegNetXLayer if config.layer_type == """x""" else RegNetYLayer

           SCREAMING_SNAKE_CASE		=						nn.Sequential(
               # downsampling is done in the first layer with stride of 2
               layer(
                   lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__						,stride=lowerCamelCase__						,)						,*[layer(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__	) for _ in range(depth - 1	)]						,)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict						,lowerCamelCase__						:   Union[str, Any]	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.layers(lowerCamelCase__	)
           return hidden_state

class 		UpperCamelCase__			(				nn.Module       ):



     '''simple docstring'''

     def __init__(							self						:   List[Any]						,lowerCamelCase__						:   RegNetConfig	)					->		str:




           '''simple docstring'''


           super().__init__()
           SCREAMING_SNAKE_CASE		=						nn.ModuleList([]	)
           # based on `downsample_in_first_stage`, the first layer of the first stage may or may not downsample the input
           self.stages.append(
               RegNetStage(
                   lowerCamelCase__						,config.embedding_size						,config.hidden_sizes[0]						,stride=2 if config.downsample_in_first_stage else 1						,depth=config.depths[0]						,)	)
           SCREAMING_SNAKE_CASE		=						zip(config.hidden_sizes						,config.hidden_sizes[1:]	)
           for (in_channels, out_channels), depth in zip(lowerCamelCase__						,config.depths[1:]	):
                 self.stages.append(RegNetStage(lowerCamelCase__						,lowerCamelCase__						,lowerCamelCase__						,depth=lowerCamelCase__	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int						,lowerCamelCase__						:   Tensor						,lowerCamelCase__						:   bool = False						,lowerCamelCase__						:   bool = True	)					->		BaseModelOutputWithNoAttention:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						() if output_hidden_states else None

           for stage_module in self.stages:
                 if output_hidden_states:
                       SCREAMING_SNAKE_CASE		=						hidden_states + (hidden_state,)

                 SCREAMING_SNAKE_CASE		=						stage_module(lowerCamelCase__	)

           if output_hidden_states:
                 SCREAMING_SNAKE_CASE		=						hidden_states + (hidden_state,)

           if not return_dict:
                 return tuple(v for v in [hidden_state, hidden_states] if v is not None	)

           return BaseModelOutputWithNoAttention(last_hidden_state=lowerCamelCase__						,hidden_states=lowerCamelCase__	)

class 		UpperCamelCase__			(				lowerCAmelCase_       ):



     '''simple docstring'''

     __snake_case   :     List[Any]		     =      RegNetConfig
     __snake_case   :     Union[str, Any]		     =      "regnet"
     __snake_case   :     Optional[Any]		     =      "pixel_values"
     __snake_case   :     List[Any]		     =      True






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]						,lowerCamelCase__						:   int	)					->		Any:




           '''simple docstring'''


           if isinstance(lowerCamelCase__						,nn.Convad	):
                 nn.init.kaiming_normal_(module.weight						,mode="""fan_out"""						,nonlinearity="""relu"""	)
           elif isinstance(lowerCamelCase__						,(nn.BatchNormad, nn.GroupNorm)	):
                 nn.init.constant_(module.weight						,1	)
                 nn.init.constant_(module.bias						,0	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   Tuple						,lowerCamelCase__						:   str=False	)					->		str:




           '''simple docstring'''


           if isinstance(lowerCamelCase__						,lowerCamelCase__	):
                 SCREAMING_SNAKE_CASE		=						value
SCREAMING_SNAKE_CASE_						       =	r"""
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`RegNetConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""

SCREAMING_SNAKE_CASE_						       =	r"""
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`ConvNextImageProcessor.__call__`] for details.

        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
"""

@add_start_docstrings(
    "The bare RegNet model outputting raw features without any specific head on top."							,				lowerCAmelCase_							,				)
# Copied from transformers.models.resnet.modeling_resnet.ResNetModel with RESNET->REGNET,ResNet->RegNet
class 		UpperCamelCase__			(				lowerCAmelCase_       ):



     '''simple docstring'''

     def __init__(							self						:   str						,lowerCamelCase__						:   str	)					->		Any:




           '''simple docstring'''


           super().__init__(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						config
           SCREAMING_SNAKE_CASE		=						RegNetEmbeddings(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						RegNetEncoder(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						nn.AdaptiveAvgPoolad((1, 1)	)
           # Initialize weights and apply final processing
           self.post_init()






     @add_start_docstrings_to_model_forward(lowerCamelCase__	)
     @add_code_sample_docstrings(
         checkpoint=_CHECKPOINT_FOR_DOC						,output_type=lowerCamelCase__						,config_class=_CONFIG_FOR_DOC						,modality="""vision"""						,expected_output=_EXPECTED_OUTPUT_SHAPE						,)
     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]						,lowerCamelCase__						:   Tensor						,lowerCamelCase__						:   Optional[bool] = None						,lowerCamelCase__						:   Optional[bool] = None	)					->		BaseModelOutputWithPoolingAndNoAttention:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						(
               output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
           )
           SCREAMING_SNAKE_CASE		=						return_dict if return_dict is not None else self.config.use_return_dict

           SCREAMING_SNAKE_CASE		=						self.embedder(lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.encoder(
               lowerCamelCase__						,output_hidden_states=lowerCamelCase__						,return_dict=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						encoder_outputs[0]

           SCREAMING_SNAKE_CASE		=						self.pooler(lowerCamelCase__	)

           if not return_dict:
                 return (last_hidden_state, pooled_output) + encoder_outputs[1:]

           return BaseModelOutputWithPoolingAndNoAttention(
               last_hidden_state=lowerCamelCase__						,pooler_output=lowerCamelCase__						,hidden_states=encoder_outputs.hidden_states						,)

@add_start_docstrings(
    "\n    RegNet Model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    "							,				lowerCAmelCase_							,				)
# Copied from transformers.models.resnet.modeling_resnet.ResNetForImageClassification with RESNET->REGNET,ResNet->RegNet,resnet->regnet
class 		UpperCamelCase__			(				lowerCAmelCase_       ):



     '''simple docstring'''

     def __init__(							self						:   Any						,lowerCamelCase__						:   Optional[int]	)					->		List[Any]:




           '''simple docstring'''


           super().__init__(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						config.num_labels
           SCREAMING_SNAKE_CASE		=						RegNetModel(lowerCamelCase__	)
           # classification head
           SCREAMING_SNAKE_CASE		=						nn.Sequential(
               nn.Flatten()						,nn.Linear(config.hidden_sizes[-1]						,config.num_labels	) if config.num_labels > 0 else nn.Identity()						,)
           # initialize weights and apply final processing
           self.post_init()






     @add_start_docstrings_to_model_forward(lowerCamelCase__	)
     @add_code_sample_docstrings(
         checkpoint=_IMAGE_CLASS_CHECKPOINT						,output_type=lowerCamelCase__						,config_class=_CONFIG_FOR_DOC						,expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT						,)
     def     SCREAMING_SNAKE_CASE__  (							self						:   str						,lowerCamelCase__						:   Optional[torch.FloatTensor] = None						,lowerCamelCase__						:   Optional[torch.LongTensor] = None						,lowerCamelCase__						:   Optional[bool] = None						,lowerCamelCase__						:   Optional[bool] = None						,)					->		ImageClassifierOutputWithNoAttention:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						return_dict if return_dict is not None else self.config.use_return_dict

           SCREAMING_SNAKE_CASE		=						self.regnet(lowerCamelCase__						,output_hidden_states=lowerCamelCase__						,return_dict=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						outputs.pooler_output if return_dict else outputs[1]

           SCREAMING_SNAKE_CASE		=						self.classifier(lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						None

           if labels is not None:
                 if self.config.problem_type is None:
                       if self.num_labels == 1:
                             SCREAMING_SNAKE_CASE		=						"""regression"""
                       elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                             SCREAMING_SNAKE_CASE		=						"""single_label_classification"""
                       else:
                             SCREAMING_SNAKE_CASE		=						"""multi_label_classification"""
                 if self.config.problem_type == "regression":
                       SCREAMING_SNAKE_CASE		=						MSELoss()
                       if self.num_labels == 1:
                             SCREAMING_SNAKE_CASE		=						loss_fct(logits.squeeze()						,labels.squeeze()	)
                       else:
                             SCREAMING_SNAKE_CASE		=						loss_fct(lowerCamelCase__						,lowerCamelCase__	)
                 elif self.config.problem_type == "single_label_classification":
                       SCREAMING_SNAKE_CASE		=						CrossEntropyLoss()
                       SCREAMING_SNAKE_CASE		=						loss_fct(logits.view(-1						,self.num_labels	)						,labels.view(-1	)	)
                 elif self.config.problem_type == "multi_label_classification":
                       SCREAMING_SNAKE_CASE		=						BCEWithLogitsLoss()
                       SCREAMING_SNAKE_CASE		=						loss_fct(lowerCamelCase__						,lowerCamelCase__	)

           if not return_dict:
                 SCREAMING_SNAKE_CASE		=						(logits,) + outputs[2:]
                 return (loss,) + output if loss is not None else output

           return ImageClassifierOutputWithNoAttention(loss=lowerCamelCase__						,logits=lowerCamelCase__						,hidden_states=outputs.hidden_states	)









import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 		UpperCamelCase__			(				lowerCAmelCase_							,				unittest.TestCase       ):



     '''simple docstring'''

     __snake_case   :     Any		     =      BertTokenizer
     __snake_case   :     Dict		     =      BertTokenizerFast
     __snake_case   :     Tuple		     =      True
     __snake_case   :     List[Any]		     =      True
     __snake_case   :     Optional[Any]		     =      filter_non_english






     def     SCREAMING_SNAKE_CASE__  (							self						:   Union[str, Any]	)					->		Any:




           '''simple docstring'''


           super().setUp()

           SCREAMING_SNAKE_CASE		=						[
               """[UNK]""",
               """[CLS]""",
               """[SEP]""",
               """[PAD]""",
               """[MASK]""",
               """want""",
               """##want""",
               """##ed""",
               """wa""",
               """un""",
               """runn""",
               """##ing""",
               """,""",
               """low""",
               """lowest""",
           ]
           SCREAMING_SNAKE_CASE		=						os.path.join(self.tmpdirname						,VOCAB_FILES_NAMES["""vocab_file"""]	)
           with open(self.vocab_file						,"""w"""						,encoding="""utf-8"""	) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str						,lowerCamelCase__						:   List[Any]	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""
           SCREAMING_SNAKE_CASE		=						"""unwanted, running"""
           return input_text, output_text






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer_class(self.vocab_file	)

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize("""UNwant\u00E9d,running"""	)
           self.assertListEqual(lowerCamelCase__						,["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]	)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__	)						,[9, 6, 7, 12, 10, 11]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		List[Any]:




           '''simple docstring'''


           if not self.test_rust_tokenizer:
                 return

           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()

           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           # With lower casing
           SCREAMING_SNAKE_CASE		=						self.get_tokenizer(do_lower_case=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer(do_lower_case=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						"""UNwant\u00E9d,running"""

           SCREAMING_SNAKE_CASE		=						tokenizer.tokenize(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.tokenize(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()
           SCREAMING_SNAKE_CASE		=						tokenizer.encode(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						rust_tokenizer.encode(lowerCamelCase__	)
           self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""	)						,["""ah""", """\u535A""", """\u63A8""", """zz"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)						,["""hello""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Dict:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hällo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""h\u00E9llo"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   str	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   List[str]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""hallo""", """!""", """how""", """are""", """you""", """?"""]	)
           self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""	)						,["""hello"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Union[str, Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """	)						,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   int	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,strip_accents=lowerCamelCase__	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """	)						,["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer(do_lower_case=lowerCamelCase__						,never_split=["""[UNK]"""]	)

           self.assertListEqual(
               tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""	)						,["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						BasicTokenizer()
           SCREAMING_SNAKE_CASE		=						"""a\n'll !!to?'d of, can't."""
           SCREAMING_SNAKE_CASE		=						["""a""", """'""", """ll""", """!""", """!""", """to""", """?""", """'""", """d""", """of""", """,""", """can""", """'""", """t""", """."""]
           self.assertListEqual(tokenizer.tokenize(lowerCamelCase__	)						,lowerCamelCase__	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Dict	)					->		int:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

           SCREAMING_SNAKE_CASE		=						{}
           for i, token in enumerate(lowerCamelCase__	):
                 SCREAMING_SNAKE_CASE		=						i
           SCREAMING_SNAKE_CASE		=						WordpieceTokenizer(vocab=lowerCamelCase__						,unk_token="""[UNK]"""	)

           self.assertListEqual(tokenizer.tokenize(""""""	)						,[]	)

           self.assertListEqual(tokenizer.tokenize("""unwanted running"""	)						,["""un""", """##want""", """##ed""", """runn""", """##ing"""]	)

           self.assertListEqual(tokenizer.tokenize("""unwantedX running"""	)						,["""[UNK]""", """runn""", """##ing"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Dict:




           '''simple docstring'''


           self.assertTrue(_is_whitespace(""" """	)	)
           self.assertTrue(_is_whitespace("""\t"""	)	)
           self.assertTrue(_is_whitespace("""\r"""	)	)
           self.assertTrue(_is_whitespace("""\n"""	)	)
           self.assertTrue(_is_whitespace("""\u00A0"""	)	)

           self.assertFalse(_is_whitespace("""A"""	)	)
           self.assertFalse(_is_whitespace("""-"""	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		Optional[int]:




           '''simple docstring'''


           self.assertTrue(_is_control("""\u0005"""	)	)

           self.assertFalse(_is_control("""A"""	)	)
           self.assertFalse(_is_control(""" """	)	)
           self.assertFalse(_is_control("""\t"""	)	)
           self.assertFalse(_is_control("""\r"""	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Tuple	)					->		List[str]:




           '''simple docstring'''


           self.assertTrue(_is_punctuation("""-"""	)	)
           self.assertTrue(_is_punctuation("""$"""	)	)
           self.assertTrue(_is_punctuation("""`"""	)	)
           self.assertTrue(_is_punctuation("""."""	)	)

           self.assertFalse(_is_punctuation("""A"""	)	)
           self.assertFalse(_is_punctuation(""" """	)	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[Any]	)					->		Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.get_tokenizer()
           SCREAMING_SNAKE_CASE		=						self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(lowerCamelCase__	) for t in ["""Test""", """\xad""", """test"""]]						,[["""[UNK]"""], [], ["""[UNK]"""]]	)

           self.assertListEqual(
               [rust_tokenizer.tokenize(lowerCamelCase__	) for t in ["""Test""", """\xad""", """test"""]]						,[["""[UNK]"""], [], ["""[UNK]"""]]	)






     @slow
     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		Optional[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained("""bert-base-uncased"""	)

           SCREAMING_SNAKE_CASE		=						tokenizer.encode("""sequence builders"""						,add_special_tokens=lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tokenizer.encode("""multi-sequence build"""						,add_special_tokens=lowerCamelCase__	)

           SCREAMING_SNAKE_CASE		=						tokenizer.build_inputs_with_special_tokens(lowerCamelCase__	)
           SCREAMING_SNAKE_CASE		=						tokenizer.build_inputs_with_special_tokens(lowerCamelCase__						,lowerCamelCase__	)

           assert encoded_sentence == [101] + text + [102]
           assert encoded_pair == [101] + text + [102] + text_a + [102]






     def     SCREAMING_SNAKE_CASE__  (							self						:   Optional[int]	)					->		str:




           '''simple docstring'''


           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode_plus(
                           lowerCamelCase__						,return_attention_mask=lowerCamelCase__						,return_token_type_ids=lowerCamelCase__						,return_offsets_mapping=lowerCamelCase__						,add_special_tokens=lowerCamelCase__						,)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.do_lower_case if hasattr(lowerCamelCase__						,"""do_lower_case"""	) else False
                       SCREAMING_SNAKE_CASE		=						(
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """A"""),
                               ((1, 2), ""","""),
                               ((3, 5), """na"""),
                               ((5, 6), """##ï"""),
                               ((6, 8), """##ve"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """Allen"""),
                               ((21, 23), """##NL"""),
                               ((23, 24), """##P"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), """a"""),
                               ((1, 2), ""","""),
                               ((3, 8), """naive"""),
                               ((9, 15), tokenizer_r.mask_token),
                               ((16, 21), """allen"""),
                               ((21, 23), """##nl"""),
                               ((23, 24), """##p"""),
                               ((25, 33), """sentence"""),
                               ((33, 34), """."""),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]						,tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]	)	)
                       self.assertEqual([e[0] for e in expected_results]						,tokens["""offset_mapping"""]	)






     def     SCREAMING_SNAKE_CASE__  (							self						:   Any	)					->		List[Any]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE		=						["""的""", """人""", """有"""]
           SCREAMING_SNAKE_CASE		=						"""""".join(lowerCamelCase__	)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                       SCREAMING_SNAKE_CASE		=						True
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_p.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.convert_ids_to_tokens(lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.convert_ids_to_tokens(lowerCamelCase__	)

                       # it is expected that each Chinese character is not preceded by "##"
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						False
                       SCREAMING_SNAKE_CASE		=						self.rust_tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						self.tokenizer_class.from_pretrained(lowerCamelCase__						,**lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.encode(lowerCamelCase__						,add_special_tokens=lowerCamelCase__	)

                       SCREAMING_SNAKE_CASE		=						tokenizer_r.convert_ids_to_tokens(lowerCamelCase__	)
                       SCREAMING_SNAKE_CASE		=						tokenizer_p.convert_ids_to_tokens(lowerCamelCase__	)

                       # it is expected that only the first Chinese character is not preceded by "##".
                       SCREAMING_SNAKE_CASE		=						[
                           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__	)
                       ]
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)
                       self.assertListEqual(lowerCamelCase__						,lowerCamelCase__	)


