



"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case_    					=	logging.get_logger(__name__)


snake_case_    					=	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
snake_case_    					=	{
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

snake_case_    					=	{
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}

@lru_cache()
def 			_lowerCAmelCase				(       ):
						UpperCAmelCase					=			(
						    list(range(ord('!'						)		,     ord('~'						) + 1						)						) + list(range(ord('¡'						)		,     ord('¬'						) + 1						)						) + list(range(ord('®'						)		,     ord('ÿ'						) + 1						)						)
						)
						UpperCAmelCase					=			bs[:]
						UpperCAmelCase					=			0
						for b in range(2**8						):
												if b not in bs:
																		bs.append(__snake_case						)
																		cs.append(2**8 + n						)
																		n += 1
						UpperCAmelCase					=			[chr(__snake_case						) for n in cs]
						return dict(zip(__snake_case		,     __snake_case						)						)

def 			_lowerCAmelCase				(       lowercase_						):
						UpperCAmelCase					=			set()
						UpperCAmelCase					=			word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)						)
												UpperCAmelCase					=			char
						return pairs







class 				A_      (					lowerCamelCase__    ):



						"""simple docstring"""
						__UpperCamelCase		    =							VOCAB_FILES_NAMES
						__UpperCamelCase		    =							PRETRAINED_VOCAB_FILES_MAP
						__UpperCamelCase		    =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__UpperCamelCase		    =							['input_ids', 'attention_mask']






						def __init__( self			:List[Any]       ,  lowercase_			:List[str]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:str="replace"       ,  lowercase_			:str="<s>"       ,  lowercase_			:Tuple="</s>"       ,  lowercase_			:Dict="</s>"       ,  lowercase_			:Union[str, Any]="<s>"       ,  lowercase_			:Dict="<unk>"       ,  lowercase_			:List[Any]="<pad>"       ,  lowercase_			:str="<mask>"       ,  lowercase_			:Tuple=False       ,  **lowercase_			:Tuple       ,  )    ->      List[str]:
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else bos_token
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else eos_token
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else sep_token
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else cls_token
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else unk_token
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else pad_token

												# Mask token behave like a normal word, i.e. include the space before it
												UpperCAmelCase					=			AddedToken(lowercase_       ,  lstrip=lowercase_       ,  rstrip=lowercase_ ) if isinstance(lowercase_       ,  lowercase_ ) else mask_token

												super().__init__(
												    errors=lowercase_       ,  bos_token=lowercase_       ,  eos_token=lowercase_       ,  unk_token=lowercase_       ,  sep_token=lowercase_       ,  cls_token=lowercase_       ,  pad_token=lowercase_       ,  mask_token=lowercase_       ,  add_prefix_space=lowercase_       ,  **lowercase_       ,  )

												with open(lowercase_       ,  encoding='utf-8' ) as vocab_handle:
																		UpperCAmelCase					=			json.load(lowercase_ )
												UpperCAmelCase					=			{v: k for k, v in self.encoder.items()}
												UpperCAmelCase					=			errors  # how to handle errors in decoding
												UpperCAmelCase					=			bytes_to_unicode()
												UpperCAmelCase					=			{v: k for k, v in self.byte_encoder.items()}
												with open(lowercase_       ,  encoding='utf-8' ) as merges_handle:
																		UpperCAmelCase					=			merges_handle.read().split('\n' )[1:-1]
												UpperCAmelCase					=			[tuple(merge.split() ) for merge in bpe_merges]
												UpperCAmelCase					=			dict(zip(lowercase_       ,  range(len(lowercase_ ) ) ) )
												UpperCAmelCase					=			{}
												UpperCAmelCase					=			add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												UpperCAmelCase					=			re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+' )






						@property
						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Any:
												return len(self.encoder )






						def      UpperCAmelCase__					( self			:Union[str, Any] )    ->      Optional[int]:
												return dict(self.encoder       ,  **self.added_tokens_encoder )






						def      UpperCAmelCase__					( self			:List[Any]       ,  lowercase_			:List[Any] )    ->      Any:







												if token in self.cache:
																		return self.cache[token]
												UpperCAmelCase					=			tuple(lowercase_ )
												UpperCAmelCase					=			get_pairs(lowercase_ )

												if not pairs:
																		return token

												while True:
																		UpperCAmelCase					=			min(lowercase_       ,  key=lambda lowercase_ : self.bpe_ranks.get(lowercase_       ,  float('inf' ) ) )
																		if bigram not in self.bpe_ranks:
																								break
																		UpperCAmelCase				,   UpperCAmelCase					=			bigram
																		UpperCAmelCase					=			[]
																		UpperCAmelCase					=			0
																		while i < len(lowercase_ ):
																								try:
																														UpperCAmelCase					=			word.index(lowercase_       ,  lowercase_ )
																								except ValueError:
																														new_word.extend(word[i:] )
																														break
																								else:
																														new_word.extend(word[i:j] )
																														UpperCAmelCase					=			j

																								if word[i] == first and i < len(lowercase_ ) - 1 and word[i + 1] == second:
																														new_word.append(first + second )
																														i += 2
																								else:
																														new_word.append(word[i] )
																														i += 1
																		UpperCAmelCase					=			tuple(lowercase_ )
																		UpperCAmelCase					=			new_word
																		if len(lowercase_ ) == 1:
																								break
																		else:
																								UpperCAmelCase					=			get_pairs(lowercase_ )
												UpperCAmelCase					=			' '.join(lowercase_ )
												UpperCAmelCase					=			word
												return word






						def      UpperCAmelCase__					( self			:str       ,  lowercase_			:Dict )    ->      int:
												UpperCAmelCase					=			[]
												for token in re.findall(self.pat       ,  lowercase_ ):
																		UpperCAmelCase					=			''.join(
																		    self.byte_encoder[b] for b in token.encode('utf-8' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																		bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_ ).split(' ' ) )
												return bpe_tokens






						def      UpperCAmelCase__					( self			:Any       ,  lowercase_			:int )    ->      Dict:
												return self.encoder.get(lowercase_       ,  self.encoder.get(self.unk_token ) )






						def      UpperCAmelCase__					( self			:Optional[int]       ,  lowercase_			:Dict )    ->      Dict:
												return self.decoder.get(lowercase_ )






						def      UpperCAmelCase__					( self			:Any       ,  lowercase_			:Any )    ->      Optional[int]:
												UpperCAmelCase					=			''.join(lowercase_ )
												UpperCAmelCase					=			bytearray([self.byte_decoder[c] for c in text] ).decode('utf-8'       ,  errors=self.errors )
												return text






						def      UpperCAmelCase__					( self			:List[str]       ,  lowercase_			:Dict       ,  lowercase_			:Union[str, Any] = None )    ->      Optional[int]:







												if not os.path.isdir(lowercase_ ):
																		logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
																		return
												UpperCAmelCase					=			os.path.join(
												    lowercase_       ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
												UpperCAmelCase					=			os.path.join(
												    lowercase_       ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

												with open(lowercase_       ,  'w'       ,  encoding='utf-8' ) as f:
																		f.write(json.dumps(self.encoder       ,  indent=2       ,  sort_keys=lowercase_       ,  ensure_ascii=lowercase_ ) + '\n' )

												UpperCAmelCase					=			0
												with open(lowercase_       ,  'w'       ,  encoding='utf-8' ) as writer:
																		writer.write('#version: 0.2\n' )
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,  key=lambda lowercase_ : kv[1] ):
																								if index != token_index:
																														logger.warning(
																														    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																														    ' Please check that the tokenizer is not corrupted!' )
																														UpperCAmelCase					=			token_index
																								writer.write(' '.join(lowercase_ ) + '\n' )
																								index += 1

												return vocab_file, merge_file






						def      UpperCAmelCase__					( self			:Optional[Any]       ,  lowercase_			:str       ,  lowercase_			:Dict = None )    ->      List[Any]:







												if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
												UpperCAmelCase					=			[self.cls_token_id]
												UpperCAmelCase					=			[self.sep_token_id]
												return cls + token_ids_a + sep + sep + token_ids_a + sep






						def      UpperCAmelCase__					( self			:Any       ,  lowercase_			:str       ,  lowercase_			:Dict = None       ,  lowercase_			:Union[str, Any] = False )    ->      List[Any]:







												if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=lowercase_       ,  token_ids_a=lowercase_       ,  already_has_special_tokens=lowercase_ )

												if token_ids_a is None:
																		return [1] + ([0] * len(lowercase_ )) + [1]
												return [1] + ([0] * len(lowercase_ )) + [1, 1] + ([0] * len(lowercase_ )) + [1]






						def      UpperCAmelCase__					( self			:int       ,  lowercase_			:List[str]       ,  lowercase_			:List[str] = None )    ->      Any:
												UpperCAmelCase					=			[self.sep_token_id]
												UpperCAmelCase					=			[self.cls_token_id]

												if token_ids_a is None:
																		return len(cls + token_ids_a + sep ) * [0]
												return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






						def      UpperCAmelCase__					( self			:int       ,  lowercase_			:str       ,  lowercase_			:Dict=False       ,  **lowercase_			:Optional[int] )    ->      str:
												UpperCAmelCase					=			kwargs.pop('add_prefix_space'       ,  self.add_prefix_space )
												if (is_split_into_words or add_prefix_space) and (len(lowercase_ ) > 0 and not text[0].isspace()):
																		UpperCAmelCase					=			' ' + text
												return (text, kwargs)




"""simple docstring"""


# Copyright 2022 The HuggingFace Team and The OpenBMB Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


snake_case_    					=	{
    """configuration_cpmant""": ["""CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CpmAntConfig"""],
    """tokenization_cpmant""": ["""CpmAntTokenizer"""],
}

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				snake_case_    					=	[
				    """CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST""",
				    """CpmAntForCausalLM""",
				    """CpmAntModel""",
				    """CpmAntPreTrainedModel""",
				]


if TYPE_CHECKING:
				from .configuration_cpmant import CPMANT_PRETRAINED_CONFIG_ARCHIVE_MAP, CpmAntConfig
				from .tokenization_cpmant import CpmAntTokenizer

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_cpmant import (
								    CPMANT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    CpmAntForCausalLM,
								    CpmAntModel,
								    CpmAntPreTrainedModel,
								)


else:
				import sys

				snake_case_    					=	_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





