



from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


lowerCAmelCase__      =    logging.get_logger(__name__)

lowerCAmelCase__      =    {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase__      =    {
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

lowerCAmelCase__      =    {'''allegro/herbert-base-cased''': 514}
lowerCAmelCase__      =    {}






class 	__snake_case	(     _lowercase):
  snake_case__    :       List[str]        =						VOCAB_FILES_NAMES
  snake_case__    :       Dict        =						PRETRAINED_VOCAB_FILES_MAP
  snake_case__    :       Any        =						PRETRAINED_INIT_CONFIGURATION
  snake_case__    :       List[Any]        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  snake_case__    :       List[str]        =						HerbertTokenizer




  def __init__(			self					:						str						, __lowerCAmelCase					:						str=None						, __lowerCAmelCase					:						Optional[Any]=None						, __lowerCAmelCase					:						Dict=None						, __lowerCAmelCase					:						Optional[int]="<s>"						, __lowerCAmelCase					:						Tuple="<unk>"						, __lowerCAmelCase					:						Optional[int]="<pad>"						, __lowerCAmelCase					:						Optional[Any]="<mask>"						, __lowerCAmelCase					:						int="</s>"						, **__lowerCAmelCase					:						Dict						, ):






         """simple docstring"""






         super().__init__(
             __lowerCAmelCase						, __lowerCAmelCase						, tokenizer_file=__lowerCAmelCase						, cls_token=__lowerCAmelCase						, unk_token=__lowerCAmelCase						, pad_token=__lowerCAmelCase						, mask_token=__lowerCAmelCase						, sep_token=__lowerCAmelCase						, **__lowerCAmelCase						, )




  def 						SCREAMING_SNAKE_CASE    (			self					:						Any						, __lowerCAmelCase					:						List[int]						, __lowerCAmelCase					:						Optional[List[int]] = None		):






         """simple docstring"""






         _lowerCamelCase						:    List[Any]         =			[self.cls_token_id]
         _lowerCamelCase						:    Optional[int]         =			[self.sep_token_id]
         if token_ids_a is None:
                return cls + token_ids_a + sep

         return cls + token_ids_a + sep + token_ids_a + sep




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple						, __lowerCAmelCase					:						List[int]						, __lowerCAmelCase					:						Optional[List[int]] = None						, __lowerCAmelCase					:						bool = False		):






         """simple docstring"""






         if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=__lowerCAmelCase						, token_ids_a=__lowerCAmelCase						, already_has_special_tokens=__lowerCAmelCase		)

         if token_ids_a is None:
                return [1] + ([0] * len(__lowerCAmelCase		)) + [1]
         return [1] + ([0] * len(__lowerCAmelCase		)) + [1] + ([0] * len(__lowerCAmelCase		)) + [1]




  def 						SCREAMING_SNAKE_CASE    (			self					:						Any						, __lowerCAmelCase					:						List[int]						, __lowerCAmelCase					:						Optional[List[int]] = None		):






         """simple docstring"""






         _lowerCamelCase						:    Tuple         =			[self.sep_token_id]
         _lowerCamelCase						:    List[str]         =			[self.cls_token_id]

         if token_ids_a is None:
                return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Optional[str] = None		):






         """simple docstring"""






         _lowerCamelCase						:    Any         =			self._tokenizer.model.save(__lowerCAmelCase						, name=__lowerCAmelCase		)
         return tuple(__lowerCAmelCase		)







"""simple docstring"""






from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__      =    logging.get_logger(__name__)

lowerCAmelCase__      =    {
    '''facebook/xglm-564M''': '''https://huggingface.co/facebook/xglm-564M/resolve/main/config.json''',
    # See all XGLM models at https://huggingface.co/models?filter=xglm
}






class 	__snake_case	(     _lowercase):
  snake_case__    :       List[Any]        =						"xglm"
  snake_case__    :       Dict        =						["past_key_values"]

  snake_case__    :       str        =						{
      "num_attention_heads": "attention_heads",
      "hidden_size": "d_model",
      "num_hidden_layers": "num_layers",
  }




  def __init__(			self					:						List[str]						, __lowerCAmelCase					:						List[Any]=2_5_6_0_0_8						, __lowerCAmelCase					:						int=2_0_4_8						, __lowerCAmelCase					:						Dict=1_0_2_4						, __lowerCAmelCase					:						List[str]=4_0_9_6						, __lowerCAmelCase					:						Tuple=2_4						, __lowerCAmelCase					:						Dict=1_6						, __lowerCAmelCase					:						Tuple="gelu"						, __lowerCAmelCase					:						Tuple=0.1						, __lowerCAmelCase					:						Tuple=0.1						, __lowerCAmelCase					:						Optional[Any]=0.0						, __lowerCAmelCase					:						List[Any]=0.0						, __lowerCAmelCase					:						int=0.02						, __lowerCAmelCase					:						Any=True						, __lowerCAmelCase					:						Tuple=True						, __lowerCAmelCase					:						str=2						, __lowerCAmelCase					:						Dict=1						, __lowerCAmelCase					:						Dict=0						, __lowerCAmelCase					:						List[Any]=2						, **__lowerCAmelCase					:						Optional[Any]						, ):






         """simple docstring"""






         _lowerCamelCase						:    List[Any]         =			vocab_size
         _lowerCamelCase						:    List[Any]         =			max_position_embeddings
         _lowerCamelCase						:    int         =			d_model
         _lowerCamelCase						:    Optional[Any]         =			ffn_dim
         _lowerCamelCase						:    Any         =			num_layers
         _lowerCamelCase						:    Union[str, Any]         =			attention_heads
         _lowerCamelCase						:    List[str]         =			activation_function
         _lowerCamelCase						:    Union[str, Any]         =			dropout
         _lowerCamelCase						:    int         =			attention_dropout
         _lowerCamelCase						:    Optional[int]         =			activation_dropout
         _lowerCamelCase						:    Any         =			layerdrop
         _lowerCamelCase						:    List[str]         =			init_std
         _lowerCamelCase						:    Union[str, Any]         =			scale_embedding  # scale factor will be sqrt(d_model) if True
         _lowerCamelCase						:    str         =			use_cache

         super().__init__(
             pad_token_id=__lowerCAmelCase						, bos_token_id=__lowerCAmelCase						, eos_token_id=__lowerCAmelCase						, decoder_start_token_id=__lowerCAmelCase						, **__lowerCAmelCase						, )


