




'''simple docstring'''


from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case				:				Optional[Any]       							=		logging.get_logger(__name__)


_snake_case				:				List[Any]       							=		{
    'nielsr/canine-s': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
_snake_case				:				Dict       							=		1114112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_snake_case				:				str       							=		0

_snake_case				:				List[Any]       							=		0xE_000
_snake_case				:				Optional[int]       							=		0xE_001
_snake_case				:				List[str]       							=		0xE_002
_snake_case				:				Optional[int]       							=		0xE_003
_snake_case				:				int       							=		0xE_004

# Maps special codepoints to human-readable names.
_snake_case				:				Dict[int, str]       							=		{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_snake_case				:				Dict[str, int]       							=		{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}

class       A   (		_a   ):
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







							def __init__(  self	:  List[Any]							,      lowerCAmelCase_	:  List[Any]=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Any=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  int=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Optional[int]=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  int=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Union[str, Any]=chr(lowerCAmelCase_ )							,      lowerCAmelCase_	:  Union[str, Any]=False							,      lowerCAmelCase_	:  List[Any]=20_48							,      **lowerCAmelCase_	:  Optional[int]							,      )      ->							Any:


														"""simple docstring"""


														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else bos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else eos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else sep_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else cls_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else pad_token

														# Mask token behave like a normal word, i.e. include the space before it
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else mask_token

														super().__init__(
														    bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      add_prefix_space=lowerCAmelCase_							,      model_max_length=lowerCAmelCase_							,      **lowerCAmelCase_							,      )

														# Creates a mapping for looking up the IDs of special symbols.
														_a     					=    {}
														for codepoint, name in SPECIAL_CODEPOINTS.items():
																					_a     					=    codepoint

														# Creates a mapping for looking up the string forms of special symbol IDs.
														_a     					=    {
														    codepoint: name for name, codepoint in self._special_codepoints.items()
														}

														_a     					=    UNICODE_VOCAB_SIZE
														_a     					=    len(self._special_codepoints )







							@property
							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							int:


														"""simple docstring"""


														return self._unicode_vocab_size







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														return list(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Dict							,      lowerCAmelCase_	:  str )      ->							int:


														"""simple docstring"""


														try:
																					return ord(lowerCAmelCase_ )
														except TypeError:
																					raise ValueError(F'invalid token: \'{token}\'' )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  int )      ->							str:


														"""simple docstring"""


														try:
																					if index in SPECIAL_CODEPOINTS:
																												return SPECIAL_CODEPOINTS[index]
																					return chr(lowerCAmelCase_ )
														except TypeError:
																					raise ValueError(F'invalid id: {index}' )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Optional[int]:


														"""simple docstring"""


														return "".join(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]

														_a     					=    cls + token_ids_a + sep
														if token_ids_a is not None:
																					result += token_ids_a + sep
														return result







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None							,      lowerCAmelCase_	:  bool = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														_a     					=    [1] + ([0] * len(lowerCAmelCase_ )) + [1]
														if token_ids_a is not None:
																					result += ([0] * len(lowerCAmelCase_ )) + [1]
														return result







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]

														_a     					=    len(cls + token_ids_a + sep ) * [0]
														if token_ids_a is not None:
																					result += len(token_ids_a + sep ) * [1]
														return result







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Any:


														"""simple docstring"""


														return ()







'''simple docstring'''


from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand
def 		snake_case_   (UpperCamelCase	:  Namespace				):





							'''simple docstring'''


							return ConvertCommand(
							    args.model_type			,						args.tf_checkpoint			,						args.pytorch_dump_output			,						args.config			,						args.finetuning_task_name				)


_snake_case				:				List[Any]       							=		'\ntransformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires\nTensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.\n'

class       A   (		_a   ):







							@staticmethod
							def        __lowerCAmelCase      (  lowerCAmelCase_	:  ArgumentParser )      ->							Any:


														"""simple docstring"""


														_a     					=    parser.add_parser(
														    '''convert'''							,      help='''CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.'''							,      )
														train_parser.add_argument('''--model_type'''							,      type=lowerCAmelCase_							,      required=lowerCAmelCase_							,      help='''Model\'s type.''' )
														train_parser.add_argument(
														    '''--tf_checkpoint'''							,      type=lowerCAmelCase_							,      required=lowerCAmelCase_							,      help='''TensorFlow checkpoint path or folder.''' )
														train_parser.add_argument(
														    '''--pytorch_dump_output'''							,      type=lowerCAmelCase_							,      required=lowerCAmelCase_							,      help='''Path to the PyTorch saved model output.''' )
														train_parser.add_argument('''--config'''							,      type=lowerCAmelCase_							,      default=''''''							,      help='''Configuration file path or folder.''' )
														train_parser.add_argument(
														    '''--finetuning_task_name'''							,      type=lowerCAmelCase_							,      default=lowerCAmelCase_							,      help='''Optional fine-tuning task name if the TF model was a finetuned model.'''							,      )
														train_parser.set_defaults(func=lowerCAmelCase_ )







							def __init__(  self	:  int							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  str							,      *lowerCAmelCase_	:  Optional[Any]							,      )      ->							List[str]:


														"""simple docstring"""


														_a     					=    logging.get_logger('''transformers-cli/converting''' )

														self._logger.info(F'Loading model {model_type}' )
														_a     					=    model_type
														_a     					=    tf_checkpoint
														_a     					=    pytorch_dump_output
														_a     					=    config
														_a     					=    finetuning_task_name







							def        __lowerCAmelCase      (  self	:  Dict )      ->							Dict:


														"""simple docstring"""


														if self._model_type == "albert":
																					try:
																												from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
																												    convert_tf_checkpoint_to_pytorch,
																												)
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					convert_tf_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														elif self._model_type == "bert":
																					try:
																												from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
																												    convert_tf_checkpoint_to_pytorch,
																												)
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					convert_tf_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														elif self._model_type == "funnel":
																					try:
																												from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
																												    convert_tf_checkpoint_to_pytorch,
																												)
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					convert_tf_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														elif self._model_type == "t5":
																					try:
																												from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					convert_tf_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														elif self._model_type == "gpt":
																					from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
																					    convert_openai_checkpoint_to_pytorch,
																					)

																					convert_openai_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														elif self._model_type == "transfo_xl":
																					try:
																												from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
																												    convert_transfo_xl_checkpoint_to_pytorch,
																												)
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					if "ckpt" in self._tf_checkpoint.lower():
																												_a     					=    self._tf_checkpoint
																												_a     					=    ''''''
																					else:
																												_a     					=    self._tf_checkpoint
																												_a     					=    ''''''
																					convert_transfo_xl_checkpoint_to_pytorch(
																					    lowerCAmelCase_							,      self._config							,      self._pytorch_dump_output							,      lowerCAmelCase_ )
														elif self._model_type == "gpt2":
																					try:
																												from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
																												    convert_gpta_checkpoint_to_pytorch,
																												)
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														elif self._model_type == "xlnet":
																					try:
																												from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
																												    convert_xlnet_checkpoint_to_pytorch,
																												)
																					except ImportError:
																												raise ImportError(lowerCAmelCase_ )

																					convert_xlnet_checkpoint_to_pytorch(
																					    self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output							,      self._finetuning_task_name )
														elif self._model_type == "xlm":
																					from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
																					    convert_xlm_checkpoint_to_pytorch,
																					)

																					convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint							,      self._pytorch_dump_output )
														elif self._model_type == "lxmert":
																					from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
																					    convert_lxmert_checkpoint_to_pytorch,
																					)

																					convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint							,      self._pytorch_dump_output )
														elif self._model_type == "rembert":
																					from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
																					    convert_rembert_tf_checkpoint_to_pytorch,
																					)

																					convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint							,      self._config							,      self._pytorch_dump_output )
														else:
																					raise ValueError(
																					    '''--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]''' )

