import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
	from transformers import LlamaTokenizerFast
except ImportError as e:
	warnings.warn(e)
	warnings.warn(
	    """The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion"""
	)
	_UpperCAmelCase            =   None
_UpperCAmelCase            =   {
    """7B""": 11008,
    """13B""": 13824,
    """30B""": 17920,
    """65B""": 22016,
    """70B""": 28672,
}
_UpperCAmelCase            =   {
    """7B""": 1,
    """7Bf""": 1,
    """13B""": 2,
    """13Bf""": 2,
    """30B""": 4,
    """65B""": 8,
    """70B""": 8,
    """70Bf""": 8,
}




def     UpperCamelCase						(	__lowercase						:			str						,__lowercase						:			Dict=1						,__lowercase						:			List[str]=2_56     ):
		'''simple docstring'''




		return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3     )     ) + multiple_of - 1) // multiple_of)




def     UpperCamelCase						(	__lowercase						:			Optional[Any]     ):
		'''simple docstring'''




		with open(__lowercase						,'r'     ) as f:
				return json.load(__lowercase     )




def     UpperCamelCase						(	__lowercase						:			str						,__lowercase						:			List[str]     ):
		'''simple docstring'''




		with open(__lowercase						,'w'     ) as f:
				json.dump(__lowercase						,__lowercase     )




def     UpperCamelCase						(	__lowercase						:			Dict						,__lowercase						:			str						,__lowercase						:			Dict						,__lowercase						:			Optional[Any]=True     ):
		'''simple docstring'''




		os.makedirs(__lowercase						,exist_ok=__lowercase     )
		A_		:    str         =				os.path.join(__lowercase						,'tmp'     )
		os.makedirs(__lowercase						,exist_ok=__lowercase     )

		A_		:    Union[str, Any]         =				read_json(os.path.join(__lowercase						,'params.json'     )     )
		A_		:    str         =				NUM_SHARDS[model_size]
		A_		:    Optional[Any]         =				params['n_layers']
		A_		:    int         =				params['n_heads']
		A_		:    List[Any]         =				n_heads // num_shards
		A_		:    Any         =				params['dim']
		A_		:    Tuple         =				dim // n_heads
		A_		:    Optional[int]         =				1_00_00.0
		A_		:    str         =				1.0 / (base ** (torch.arange(0						,__lowercase						,2     ).float() / dims_per_head))

		if "n_kv_heads" in params:
				A_		:    List[Any]         =				params['n_kv_heads']  # for GQA / MQA
				A_		:    List[Any]         =				n_heads_per_shard // num_key_value_heads
				A_		:    Optional[int]         =				dim // num_key_value_heads
		else:  # compatibility with other checkpoints
				A_		:    Any         =				n_heads
				A_		:    Union[str, Any]         =				n_heads_per_shard
				A_		:    str         =				dim

		# permute for sliced rotary
		def permute(__lowercase						:			str						,__lowercase						:			int=n_heads						,__lowercase						:			Optional[Any]=dim						,__lowercase						:			Tuple=dim     ):
				return w.view(__lowercase						,dima // n_heads // 2						,2						,__lowercase     ).transpose(1						,2     ).reshape(__lowercase						,__lowercase     )

		print(f'''Fetching all parameters from the checkpoint at {input_base_path}.'''     )
		# Load weights
		if model_size == "7B":
				# Not sharded
				# (The sharded implementation would also work, but this is simpler.)
				A_		:    List[Any]         =				torch.load(os.path.join(__lowercase						,'consolidated.00.pth'     )						,map_location='cpu'     )
		else:
				# Sharded
				A_		:    int         =				[
				    torch.load(os.path.join(__lowercase						,f'''consolidated.{i:02d}.pth'''     )						,map_location='cpu'     )
				    for i in range(__lowercase     )
				]
		A_		:    Tuple         =				0
		A_		:    Optional[int]         =				{'weight_map': {}}
		for layer_i in range(__lowercase     ):
				A_		:    str         =				f'''pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin'''
				if model_size == "7B":
						# Unsharded
						A_		:    int         =				{
						    f'''model.layers.{layer_i}.self_attn.q_proj.weight''': permute(
						        loaded[f'''layers.{layer_i}.attention.wq.weight''']     ),
						    f'''model.layers.{layer_i}.self_attn.k_proj.weight''': permute(
						        loaded[f'''layers.{layer_i}.attention.wk.weight''']     ),
						    f'''model.layers.{layer_i}.self_attn.v_proj.weight''': loaded[f'''layers.{layer_i}.attention.wv.weight'''],
						    f'''model.layers.{layer_i}.self_attn.o_proj.weight''': loaded[f'''layers.{layer_i}.attention.wo.weight'''],
						    f'''model.layers.{layer_i}.mlp.gate_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w1.weight'''],
						    f'''model.layers.{layer_i}.mlp.down_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w2.weight'''],
						    f'''model.layers.{layer_i}.mlp.up_proj.weight''': loaded[f'''layers.{layer_i}.feed_forward.w3.weight'''],
						    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[f'''layers.{layer_i}.attention_norm.weight'''],
						    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[f'''layers.{layer_i}.ffn_norm.weight'''],
						}
				else:
						# Sharded
						# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
						# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
						# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

						A_		:    List[str]         =				{
						    f'''model.layers.{layer_i}.input_layernorm.weight''': loaded[0][
						        f'''layers.{layer_i}.attention_norm.weight'''
						    ].clone(),
						    f'''model.layers.{layer_i}.post_attention_layernorm.weight''': loaded[0][
						        f'''layers.{layer_i}.ffn_norm.weight'''
						    ].clone(),
						}
						A_		:    List[str]         =				permute(
						    torch.cat(
						        [
						            loaded[i][f'''layers.{layer_i}.attention.wq.weight'''].view(__lowercase						,__lowercase						,__lowercase     )
						            for i in range(__lowercase     )
						        ]						,dim=0						,).reshape(__lowercase						,__lowercase     )     )
						A_		:    List[str]         =				permute(
						    torch.cat(
						        [
						            loaded[i][f'''layers.{layer_i}.attention.wk.weight'''].view(
						                __lowercase						,__lowercase						,__lowercase     )
						            for i in range(__lowercase     )
						        ]						,dim=0						,).reshape(__lowercase						,__lowercase     )						,__lowercase						,__lowercase						,__lowercase						,)
						A_		:    List[Any]         =				torch.cat(
						    [
						        loaded[i][f'''layers.{layer_i}.attention.wv.weight'''].view(
						            __lowercase						,__lowercase						,__lowercase     )
						        for i in range(__lowercase     )
						    ]						,dim=0						,).reshape(__lowercase						,__lowercase     )

						A_		:    str         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.attention.wo.weight'''] for i in range(__lowercase     )]						,dim=1     )
						A_		:    str         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w1.weight'''] for i in range(__lowercase     )]						,dim=0     )
						A_		:    Tuple         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w2.weight'''] for i in range(__lowercase     )]						,dim=1     )
						A_		:    Union[str, Any]         =				torch.cat(
						    [loaded[i][f'''layers.{layer_i}.feed_forward.w3.weight'''] for i in range(__lowercase     )]						,dim=0     )

				A_		:    List[Any]         =				inv_freq
				for k, v in state_dict.items():
						A_		:    Dict         =				filename
						param_count += v.numel()
				torch.save(__lowercase						,os.path.join(__lowercase						,__lowercase     )     )

		A_		:    int         =				f'''pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin'''
		if model_size == "7B":
				# Unsharded
				A_		:    Tuple         =				{
				    'model.embed_tokens.weight': loaded['tok_embeddings.weight'],
				    'model.norm.weight': loaded['norm.weight'],
				    'lm_head.weight': loaded['output.weight'],
				}
		else:
				A_		:    List[str]         =				{
				    'model.norm.weight': loaded[0]['norm.weight'],
				    'model.embed_tokens.weight': torch.cat(
				        [loaded[i]['tok_embeddings.weight'] for i in range(__lowercase     )]						,dim=1     ),
				    'lm_head.weight': torch.cat([loaded[i]['output.weight'] for i in range(__lowercase     )]						,dim=0     ),
				}

		for k, v in state_dict.items():
				A_		:    Any         =				filename
				param_count += v.numel()
		torch.save(__lowercase						,os.path.join(__lowercase						,__lowercase     )     )

		# Write configs
		A_		:    List[str]         =				{'total_size': param_count * 2}
		write_json(__lowercase						,os.path.join(__lowercase						,'pytorch_model.bin.index.json'     )     )
		A_		:    Union[str, Any]         =				params['ffn_dim_multiplier'] if 'ffn_dim_multiplier' in params else 1
		A_		:    str         =				params['multiple_of'] if 'multiple_of' in params else 2_56
		A_		:    List[str]         =				LlamaConfig(
		    hidden_size=__lowercase						,intermediate_size=compute_intermediate_size(__lowercase						,__lowercase						,__lowercase     )						,num_attention_heads=params['n_heads']						,num_hidden_layers=params['n_layers']						,rms_norm_eps=params['norm_eps']						,num_key_value_heads=__lowercase						,)
		config.save_pretrained(__lowercase     )

		# Make space so we can load the model properly now.
		del state_dict
		del loaded
		gc.collect()

		print('Loading the checkpoint in a Llama model.'     )
		A_		:    List[Any]         =				LlamaForCausalLM.from_pretrained(__lowercase						,torch_dtype=torch.floataa						,low_cpu_mem_usage=__lowercase     )
		# Avoid saving this as part of the config.
		del model.config._name_or_path

		print('Saving in the Transformers format.'     )
		model.save_pretrained(__lowercase						,safe_serialization=__lowercase     )
		shutil.rmtree(__lowercase     )




def     UpperCamelCase						(	__lowercase						:			Tuple						,__lowercase						:			Optional[Any]     ):
		'''simple docstring'''




		A_		:    List[str]         =				LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
		print(f'''Saving a {tokenizer_class.__name__} to {tokenizer_path}.'''     )
		A_		:    Dict         =				tokenizer_class(__lowercase     )
		tokenizer.save_pretrained(__lowercase     )




def     UpperCamelCase						(	):
		'''simple docstring'''




		A_		:    Optional[int]         =				argparse.ArgumentParser()
		parser.add_argument(
		    '--input_dir'						,help='Location of LLaMA weights, which contains tokenizer.model and model folders'						,)
		parser.add_argument(
		    '--model_size'						,choices=['7B', '7Bf', '13B', '13Bf', '30B', '65B', '70B', '70Bf', 'tokenizer_only']						,)
		parser.add_argument(
		    '--output_dir'						,help='Location to write HF model and tokenizer'						,)
		parser.add_argument('--safe_serialization'						,type=__lowercase						,help='Whether or not to save using `safetensors`.'     )
		A_		:    List[Any]         =				parser.parse_args()
		if args.model_size != "tokenizer_only":
				write_model(
				    model_path=args.output_dir						,input_base_path=os.path.join(args.input_dir						,args.model_size     )						,model_size=args.model_size						,safe_serialization=args.safe_serialization						,)
		A_		:    Dict         =				os.path.join(args.input_dir						,'tokenizer.model'     )
		write_tokenizer(args.output_dir						,__lowercase     )


if __name__ == "__main__":
	main()







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase            =		logging.get_logger(__name__)


_UpperCAmelCase            =		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

_UpperCAmelCase            =		{
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

_UpperCAmelCase            =		{
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def     UpperCamelCase						(	):
		'''simple docstring'''




		A_		:    Optional[int]         =				(
		    list(range(ord('!'     )						,ord('~'     ) + 1     )     ) + list(range(ord('¡'     )						,ord('¬'     ) + 1     )     ) + list(range(ord('®'     )						,ord('ÿ'     ) + 1     )     )
		)
		A_		:    Any         =				bs[:]
		A_		:    Optional[Any]         =				0
		for b in range(2**8     ):
				if b not in bs:
						bs.append(__lowercase     )
						cs.append(2**8 + n     )
						n += 1
		A_		:    str         =				[chr(__lowercase     ) for n in cs]
		return dict(zip(__lowercase						,__lowercase     )     )




def     UpperCamelCase						(	__lowercase						:			Dict     ):
		'''simple docstring'''




		A_		:    List[str]         =				set()
		A_		:    List[Any]         =				word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)     )
				A_		:    Union[str, Any]         =				char
		return pairs





class UpperCAmelCase						( __A  ):


				'''simple docstring'''






				lowerCamelCase_    = VOCAB_FILES_NAMES
				lowerCamelCase_    = PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase_    = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase_    = ['''input_ids''', '''attention_mask''']
				def __init__(						self		,     lowercase		,     lowercase		,     lowercase="replace"		,     lowercase="<s>"		,     lowercase="</s>"		,     lowercase="</s>"		,     lowercase="<s>"		,     lowercase="<unk>"		,     lowercase="<pad>"		,     lowercase="<mask>"		,     lowercase=False		,     **lowercase		,     ):



						"""simple docstring"""





						A_		:    Union[str, Any]         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else bos_token
						A_		:    str         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else eos_token
						A_		:    str         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else sep_token
						A_		:    Tuple         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else cls_token
						A_		:    List[str]         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else unk_token
						A_		:    Optional[int]         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else pad_token

						# Mask token behave like a normal word, i.e. include the space before it
						A_		:    int         =				AddedToken(lowercase		,     lstrip=lowercase		,     rstrip=lowercase ) if isinstance(lowercase		,     lowercase ) else mask_token

						super().__init__(
						    errors=lowercase		,     bos_token=lowercase		,     eos_token=lowercase		,     unk_token=lowercase		,     sep_token=lowercase		,     cls_token=lowercase		,     pad_token=lowercase		,     mask_token=lowercase		,     add_prefix_space=lowercase		,     **lowercase		,     )

						with open(lowercase		,     encoding='utf-8' ) as vocab_handle:
								A_		:    int         =				json.load(lowercase )
						A_		:    Union[str, Any]         =				{v: k for k, v in self.encoder.items()}
						A_		:    Union[str, Any]         =				errors  # how to handle errors in decoding
						A_		:    Tuple         =				bytes_to_unicode()
						A_		:    str         =				{v: k for k, v in self.byte_encoder.items()}
						with open(lowercase		,     encoding='utf-8' ) as merges_handle:
								A_		:    str         =				merges_handle.read().split('\n' )[1:-1]
						A_		:    Optional[Any]         =				[tuple(merge.split() ) for merge in bpe_merges]
						A_		:    Any         =				dict(zip(lowercase		,     range(len(lowercase ) ) ) )
						A_		:    str         =				{}
						A_		:    List[Any]         =				add_prefix_space

						# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
						A_		:    Optional[Any]         =				re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+' )
				@property
				def   lowerCAmelCase_       (						self ):



						"""simple docstring"""





						return len(self.encoder )
				def   lowerCAmelCase_       (						self ):



						"""simple docstring"""





						return dict(self.encoder		,     **self.added_tokens_encoder )
				def   lowerCAmelCase_       (						self		,     lowercase ):



						"""simple docstring"""





						if token in self.cache:
								return self.cache[token]
						A_		:    List[Any]         =				tuple(lowercase )
						A_		:    List[Any]         =				get_pairs(lowercase )

						if not pairs:
								return token

						while True:
								A_		:    List[str]         =				min(lowercase		,     key=lambda lowercase : self.bpe_ranks.get(lowercase		,     float('inf' ) ) )
								if bigram not in self.bpe_ranks:
										break
								A_       ,      A_		:    Optional[Any]         =				bigram
								A_		:    List[Any]         =				[]
								A_		:    Tuple         =				0
								while i < len(lowercase ):
										try:
												A_		:    Optional[Any]         =				word.index(lowercase		,     lowercase )
										except ValueError:
												new_word.extend(word[i:] )
												break
										else:
												new_word.extend(word[i:j] )
												A_		:    int         =				j

										if word[i] == first and i < len(lowercase ) - 1 and word[i + 1] == second:
												new_word.append(first + second )
												i += 2
										else:
												new_word.append(word[i] )
												i += 1
								A_		:    Union[str, Any]         =				tuple(lowercase )
								A_		:    Tuple         =				new_word
								if len(lowercase ) == 1:
										break
								else:
										A_		:    Optional[int]         =				get_pairs(lowercase )
						A_		:    Optional[Any]         =				' '.join(lowercase )
						A_		:    str         =				word
						return word
				def   lowerCAmelCase_       (						self		,     lowercase ):



						"""simple docstring"""





						A_		:    Optional[int]         =				[]
						for token in re.findall(self.pat		,     lowercase ):
								A_		:    Optional[Any]         =				''.join(
								    self.byte_encoder[b] for b in token.encode('utf-8' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
								bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase ).split(' ' ) )
						return bpe_tokens
				def   lowerCAmelCase_       (						self		,     lowercase ):



						"""simple docstring"""





						return self.encoder.get(lowercase		,     self.encoder.get(self.unk_token ) )
				def   lowerCAmelCase_       (						self		,     lowercase ):



						"""simple docstring"""





						return self.decoder.get(lowercase )
				def   lowerCAmelCase_       (						self		,     lowercase ):



						"""simple docstring"""





						A_		:    Union[str, Any]         =				''.join(lowercase )
						A_		:    List[Any]         =				bytearray([self.byte_decoder[c] for c in text] ).decode('utf-8'		,     errors=self.errors )
						return text
				def   lowerCAmelCase_       (						self		,     lowercase		,     lowercase = None ):



						"""simple docstring"""





						if not os.path.isdir(lowercase ):
								logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
								return
						A_		:    Optional[int]         =				os.path.join(
						    lowercase		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
						A_		:    Dict         =				os.path.join(
						    lowercase		,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

						with open(lowercase		,     'w'		,     encoding='utf-8' ) as f:
								f.write(json.dumps(self.encoder		,     indent=2		,     sort_keys=lowercase		,     ensure_ascii=lowercase ) + '\n' )

						A_		:    Dict         =				0
						with open(lowercase		,     'w'		,     encoding='utf-8' ) as writer:
								writer.write('#version: 0.2\n' )
								for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,     key=lambda lowercase : kv[1] ):
										if index != token_index:
												logger.warning(
												    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
												    ' Please check that the tokenizer is not corrupted!' )
												A_		:    int         =				token_index
										writer.write(' '.join(lowercase ) + '\n' )
										index += 1

						return vocab_file, merge_file
				def   lowerCAmelCase_       (						self		,     lowercase		,     lowercase = None ):



						"""simple docstring"""





						if token_ids_a is None:
								return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						A_		:    Tuple         =				[self.cls_token_id]
						A_		:    Any         =				[self.sep_token_id]
						return cls + token_ids_a + sep + sep + token_ids_a + sep
				def   lowerCAmelCase_       (						self		,     lowercase		,     lowercase = None		,     lowercase = False ):



						"""simple docstring"""





						if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=lowercase		,     token_ids_a=lowercase		,     already_has_special_tokens=lowercase )

						if token_ids_a is None:
								return [1] + ([0] * len(lowercase )) + [1]
						return [1] + ([0] * len(lowercase )) + [1, 1] + ([0] * len(lowercase )) + [1]
				def   lowerCAmelCase_       (						self		,     lowercase		,     lowercase = None ):



						"""simple docstring"""





						A_		:    Optional[int]         =				[self.sep_token_id]
						A_		:    List[Any]         =				[self.cls_token_id]

						if token_ids_a is None:
								return len(cls + token_ids_a + sep ) * [0]
						return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]







				def   lowerCAmelCase_       (						self		,     lowercase		,     lowercase=False		,     **lowercase ):



						"""simple docstring"""





						A_		:    Tuple         =				kwargs.pop('add_prefix_space'		,     self.add_prefix_space )
						if (is_split_into_words or add_prefix_space) and (len(lowercase ) > 0 and not text[0].isspace()):
								A_		:    int         =				' ' + text
						return (text, kwargs)






