

'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


_snake_case         =  logging.get_logger(__name__)

_snake_case         =  {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

_snake_case         =  {
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

_snake_case         =  {
    'junnyu/roformer_chinese_small': 1_536,
    'junnyu/roformer_chinese_base': 1_536,
    'junnyu/roformer_chinese_char_small': 512,
    'junnyu/roformer_chinese_char_base': 512,
    'junnyu/roformer_small_discriminator': 128,
    'junnyu/roformer_small_generator': 128,
}


_snake_case         =  {
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					int     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Dict     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							PRETRAINED_INIT_CONFIGURATION
   _SCREAMING_SNAKE_CASE   :					Optional[int]     	=							RoFormerTokenizer


   def __init__(			self      ,    _UpperCamelCase=None      ,    _UpperCamelCase=None      ,    _UpperCamelCase=True      ,    _UpperCamelCase="[UNK]"      ,    _UpperCamelCase="[SEP]"      ,    _UpperCamelCase="[PAD]"      ,    _UpperCamelCase="[CLS]"      ,    _UpperCamelCase="[MASK]"      ,    _UpperCamelCase=True      ,    _UpperCamelCase=None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      super().__init__(
          _UpperCamelCase      ,    tokenizer_file=_UpperCamelCase      ,    do_lower_case=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    sep_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    cls_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    tokenize_chinese_chars=_UpperCamelCase      ,    strip_accents=_UpperCamelCase      ,    **_UpperCamelCase      ,    )

      _lowercase      :				Tuple      					=       json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
      if (
          pre_tok_state.get("lowercase"      ,    _UpperCamelCase  ) != do_lower_case
          or pre_tok_state.get("strip_accents"      ,    _UpperCamelCase  ) != strip_accents
      ):
         _lowercase      :				str      					=       getattr(_UpperCamelCase      ,    pre_tok_state.pop("type"  )  )
         _lowercase      :				Union[str, Any]      					=       do_lower_case
         _lowercase      :				List[Any]      					=       strip_accents
         _lowercase      :				Union[str, Any]      					=       pre_tok_class(**_UpperCamelCase  )

      _lowercase      :				int      					=       do_lower_case


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       self.__dict__.copy()
      _lowercase      :				Tuple      					=       BertPreTokenizer()
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       d
      _lowercase      :				Optional[int]      					=       self.__dict__["_tokenizer"].get_vocab()
      _lowercase      :				List[str]      					=       PreTokenizer.custom(JiebaPreTokenizer(_UpperCamelCase  )  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      _lowercase      :				str      					=       [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
         output += token_ids_a + [self.sep_token_id]

      return output


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      _lowercase      :				int      					=       [self.sep_token_id]
      _lowercase      :				Any      					=       [self.cls_token_id]
      if token_ids_a is None:
         return len(cls + token_ids_a + sep  ) * [0]
      return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       self._tokenizer.model.save(_UpperCamelCase      ,    name=_UpperCamelCase  )
      return tuple(_UpperCamelCase  )






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None      ,    _UpperCamelCase=None      ,    _UpperCamelCase=False      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       BertPreTokenizer()
      return super().save_pretrained(_UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    _UpperCamelCase      ,    **_UpperCamelCase  )









'''simple docstring'''
import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


_snake_case         =  logging.getLogger()


@unittest.skip('Temporarily disable the doc tests.'  )
@require_torch
@require_tf
@slow
class        a__  (							unittest.TestCase  ):


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = None      ,    _UpperCamelCase = None      ,    _UpperCamelCase = True      ,    ):





      """simple docstring"""





      _lowercase      :				str      					=       [file for file in os.listdir(_UpperCamelCase  ) if os.path.isfile(os.path.join(_UpperCamelCase      ,    _UpperCamelCase  )  )]

      if identifier is not None:
         _lowercase      :				str      					=       [file for file in files if identifier in file]

      if n_identifier is not None:
         if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            for n_ in n_identifier:
               _lowercase      :				Dict      					=       [file for file in files if n_ not in file]
         else:
            _lowercase      :				Optional[Any]      					=       [file for file in files if n_identifier not in file]

      _lowercase      :				Dict      					=       ignore_files or []
      ignore_files.append("__init__.py"  )
      _lowercase      :				List[str]      					=       [file for file in files if file not in ignore_files]

      for file in files:
         # Open all files
         print("Testing"      ,    _UpperCamelCase  )

         if only_modules:
            _lowercase      :				Optional[Any]      					=       file.split("."  )[0]
            try:
               _lowercase      :				Union[str, Any]      					=       getattr(_UpperCamelCase      ,    _UpperCamelCase  )
               _lowercase      :				Optional[int]      					=       doctest.DocTestSuite(_UpperCamelCase  )
               _lowercase      :				Tuple      					=       unittest.TextTestRunner().run(_UpperCamelCase  )
               self.assertIs(len(result.failures  )      ,    0  )
            except AttributeError:
               logger.info(f'''{module_identifier} is not a module.'''  )
         else:
            _lowercase      :				Optional[Any]      					=       doctest.testfile(str(".." / directory / file  )      ,    optionflags=doctest.ELLIPSIS  )
            self.assertIs(result.failed      ,    0  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       Path("src/transformers"  )
      _lowercase      :				str      					=       "modeling"
      _lowercase      :				Tuple      					=       [
          "modeling_ctrl.py",
          "modeling_tf_ctrl.py",
      ]
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase      ,    ignore_files=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       Path("src/transformers"  )
      _lowercase      :				Any      					=       "tokenization"
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       Path("src/transformers"  )
      _lowercase      :				Optional[Any]      					=       "configuration"
      self.analyze_directory(_UpperCamelCase      ,    identifier=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Union[str, Any]      					=       Path("src/transformers"  )
      _lowercase      :				List[Any]      					=       ["configuration", "modeling", "tokenization"]
      self.analyze_directory(_UpperCamelCase      ,    n_identifier=_UpperCamelCase  )






   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Dict      					=       Path("docs/source"  )
      _lowercase      :				int      					=       ["favicon.ico"]
      self.analyze_directory(_UpperCamelCase      ,    ignore_files=_UpperCamelCase      ,    only_modules=_UpperCamelCase  )






