






"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase					      =  logging.get_logger(__name__)


_UpperCamelCase					      =  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
_UpperCamelCase					      =  {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

_UpperCamelCase					      =  {
    """facebook/bart-base""": 1024,
    """facebook/bart-large""": 1024,
    """facebook/bart-large-mnli""": 1024,
    """facebook/bart-large-cnn""": 1024,
    """facebook/bart-large-xsum""": 1024,
    """yjernite/bart_eli5""": 1024,
}

@lru_cache()
def _a			(		):







      """simple docstring"""






      UpperCAmelCase          =	(
          list(range(ord("""!"""							)				,	ord("""~"""							) + 1							)							) + list(range(ord("""¡"""							)				,	ord("""¬"""							) + 1							)							) + list(range(ord("""®"""							)				,	ord("""ÿ"""							) + 1							)							)
      )
      UpperCAmelCase          =	bs[:]
      UpperCAmelCase          =	0
      for b in range(2**8							):
            if b not in bs:
                  bs.append(_snake_case							)
                  cs.append(2**8 + n							)
                  n += 1
      UpperCAmelCase          =	[chr(_snake_case							) for n in cs]
      return dict(zip(_snake_case				,	_snake_case							)							)

def _a			(		_snake_case							):







      """simple docstring"""






      UpperCAmelCase          =	set()
      UpperCAmelCase          =	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)							)
            UpperCAmelCase          =	char
      return pairs







class  lowerCamelCase__ (	snake_case			):
    SCREAMING_SNAKE_CASE		=      VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE		=      PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE		=      ['''input_ids''', '''attention_mask''']






    def __init__(			self     ,A     ,A     ,A="replace"     ,A="<s>"     ,A="</s>"     ,A="</s>"     ,A="<s>"     ,A="<unk>"     ,A="<pad>"     ,A="<mask>"     ,A=False     ,**A     ,):
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else bos_token
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else eos_token
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else sep_token
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else cls_token
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else unk_token
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else pad_token

          # Mask token behave like a normal word, i.e. include the space before it
          UpperCAmelCase          =	AddedToken(A     ,lstrip=A     ,rstrip=A ) if isinstance(A     ,A ) else mask_token

          super().__init__(
              errors=A     ,bos_token=A     ,eos_token=A     ,unk_token=A     ,sep_token=A     ,cls_token=A     ,pad_token=A     ,mask_token=A     ,add_prefix_space=A     ,**A     ,)

          with open(A     ,encoding="""utf-8""" ) as vocab_handle:
                UpperCAmelCase          =	json.load(A )
          UpperCAmelCase          =	{v: k for k, v in self.encoder.items()}
          UpperCAmelCase          =	errors  # how to handle errors in decoding
          UpperCAmelCase          =	bytes_to_unicode()
          UpperCAmelCase          =	{v: k for k, v in self.byte_encoder.items()}
          with open(A     ,encoding="""utf-8""" ) as merges_handle:
                UpperCAmelCase          =	merges_handle.read().split("""\n""" )[1:-1]
          UpperCAmelCase          =	[tuple(merge.split() ) for merge in bpe_merges]
          UpperCAmelCase          =	dict(zip(A     ,range(len(A ) ) ) )
          UpperCAmelCase          =	{}
          UpperCAmelCase          =	add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          UpperCAmelCase          =	re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" )






    @property
    def     _UpperCamelCase							(			self ):
          return len(self.encoder )






    def     _UpperCamelCase							(			self ):
          return dict(self.encoder     ,**self.added_tokens_encoder )






    def     _UpperCamelCase							(			self     ,A ):
          if token in self.cache:
                return self.cache[token]
          UpperCAmelCase          =	tuple(A )
          UpperCAmelCase          =	get_pairs(A )

          if not pairs:
                return token

          while True:
                UpperCAmelCase          =	min(A     ,key=lambda A : self.bpe_ranks.get(A     ,float("""inf""" ) ) )
                if bigram not in self.bpe_ranks:
                      break
                UpperCAmelCase      , UpperCAmelCase          =	bigram
                UpperCAmelCase          =	[]
                UpperCAmelCase          =	0
                while i < len(A ):
                      try:
                            UpperCAmelCase          =	word.index(A     ,A )
                      except ValueError:
                            new_word.extend(word[i:] )
                            break
                      else:
                            new_word.extend(word[i:j] )
                            UpperCAmelCase          =	j

                      if word[i] == first and i < len(A ) - 1 and word[i + 1] == second:
                            new_word.append(first + second )
                            i += 2
                      else:
                            new_word.append(word[i] )
                            i += 1
                UpperCAmelCase          =	tuple(A )
                UpperCAmelCase          =	new_word
                if len(A ) == 1:
                      break
                else:
                      UpperCAmelCase          =	get_pairs(A )
          UpperCAmelCase          =	""" """.join(A )
          UpperCAmelCase          =	word
          return word






    def     _UpperCamelCase							(			self     ,A ):
          UpperCAmelCase          =	[]
          for token in re.findall(self.pat     ,A ):
                UpperCAmelCase          =	"""""".join(
                    self.byte_encoder[b] for b in token.encode("""utf-8""" ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A ).split(""" """ ) )
          return bpe_tokens






    def     _UpperCamelCase							(			self     ,A ):
          return self.encoder.get(A     ,self.encoder.get(self.unk_token ) )






    def     _UpperCamelCase							(			self     ,A ):
          return self.decoder.get(A )






    def     _UpperCamelCase							(			self     ,A ):
          UpperCAmelCase          =	"""""".join(A )
          UpperCAmelCase          =	bytearray([self.byte_decoder[c] for c in text] ).decode("""utf-8"""     ,errors=self.errors )
          return text






    def     _UpperCamelCase							(			self     ,A     ,A = None ):
          if not os.path.isdir(A ):
                logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
                return
          UpperCAmelCase          =	os.path.join(
              A     ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
          UpperCAmelCase          =	os.path.join(
              A     ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""] )

          with open(A     ,"""w"""     ,encoding="""utf-8""" ) as f:
                f.write(json.dumps(self.encoder     ,indent=2     ,sort_keys=A     ,ensure_ascii=A ) + """\n""" )

          UpperCAmelCase          =	0
          with open(A     ,"""w"""     ,encoding="""utf-8""" ) as writer:
                writer.write("""#version: 0.2\n""" )
                for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,key=lambda A : kv[1] ):
                      if index != token_index:
                            logger.warning(
                                F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                """ Please check that the tokenizer is not corrupted!""" )
                            UpperCAmelCase          =	token_index
                      writer.write(""" """.join(A ) + """\n""" )
                      index += 1

          return vocab_file, merge_file






    def     _UpperCamelCase							(			self     ,A     ,A = None ):
          if token_ids_a is None:
                return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          UpperCAmelCase          =	[self.cls_token_id]
          UpperCAmelCase          =	[self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep






    def     _UpperCamelCase							(			self     ,A     ,A = None     ,A = False ):
          if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=A     ,token_ids_a=A     ,already_has_special_tokens=A )

          if token_ids_a is None:
                return [1] + ([0] * len(A )) + [1]
          return [1] + ([0] * len(A )) + [1, 1] + ([0] * len(A )) + [1]






    def     _UpperCamelCase							(			self     ,A     ,A = None ):
          UpperCAmelCase          =	[self.sep_token_id]
          UpperCAmelCase          =	[self.cls_token_id]

          if token_ids_a is None:
                return len(cls + token_ids_a + sep ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






    def     _UpperCamelCase							(			self     ,A     ,A=False     ,**A ):
          UpperCAmelCase          =	kwargs.pop("""add_prefix_space"""     ,self.add_prefix_space )
          if (is_split_into_words or add_prefix_space) and (len(A ) > 0 and not text[0].isspace()):
                UpperCAmelCase          =	""" """ + text
          return (text, kwargs)











"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_pegasus import PegasusTokenizer
else:
  _UpperCamelCase					      =  None


_UpperCamelCase					      =  logging.get_logger(__name__)


_UpperCamelCase					      =  """▁"""

_UpperCamelCase					      =  {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase					      =  {
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""},
    """tokenizer_file""": {
        """google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"""
    },
}

_UpperCamelCase					      =  {
    """google/pegasus-xsum""": 512,
}







class  lowerCamelCase__ (	snake_case			):
    SCREAMING_SNAKE_CASE		=      VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE		=      PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE		=      PegasusTokenizer
    SCREAMING_SNAKE_CASE		=      ['''input_ids''', '''attention_mask''']






    def __init__(			self     ,A=None     ,A=None     ,A="<pad>"     ,A="</s>"     ,A="<unk>"     ,A="<mask_2>"     ,A="<mask_1>"     ,A=None     ,A=103     ,**A     ,):
          UpperCAmelCase          =	offset

          if additional_special_tokens is not None:
                if not isinstance(A     ,A ):
                      raise TypeError(
                          F'''additional_special_tokens should be of type {type(A )}, but is'''
                          F''' {type(A )}''' )

                UpperCAmelCase          =	(
                    ([mask_token_sent] + additional_special_tokens)
                    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                    else additional_special_tokens
                )
                # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                additional_special_tokens_extended += [
                    F'''<unk_{i}>''' for i in range(len(A )     ,self.offset - 1 )
                ]

                if len(set(A ) ) != len(A ):
                      raise ValueError(
                          """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
                          F''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.''' )
                UpperCAmelCase          =	additional_special_tokens_extended
          else:
                UpperCAmelCase          =	[mask_token_sent] if mask_token_sent is not None else []
                additional_special_tokens += [F'''<unk_{i}>''' for i in range(2     ,self.offset )]

          super().__init__(
              A     ,tokenizer_file=A     ,pad_token=A     ,eos_token=A     ,unk_token=A     ,mask_token=A     ,mask_token_sent=A     ,offset=A     ,additional_special_tokens=A     ,**A     ,)
          UpperCAmelCase          =	vocab_file
          UpperCAmelCase          =	False if not self.vocab_file else True






    def     _UpperCamelCase							(			self     ,A ):
          UpperCAmelCase          =	set(self.all_special_ids )  # call it once instead of inside list comp
          all_special_ids.remove(self.unk_token_id )  # <unk> is only sometimes special

          if all_special_ids != set(range(len(self.additional_special_tokens ) + 3 ) ):
                raise ValueError(
                    """There should be 3 special tokens: mask_token, pad_token, and eos_token +"""
                    F''' {len(self.additional_special_tokens )} additional_special_tokens, but got {all_special_ids}''' )

          return [1 if x in all_special_ids else 0 for x in seq]






    def     _UpperCamelCase							(			self     ,A     ,A = None     ,A = False ):
          if already_has_special_tokens:
                return self._special_token_mask(A )
          elif token_ids_a is None:
                return self._special_token_mask(A ) + [1]
          else:
                return self._special_token_mask(token_ids_a + token_ids_a ) + [1]






    def     _UpperCamelCase							(			self     ,A     ,A=None ):
          if token_ids_a is None:
                return token_ids_a + [self.eos_token_id]
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return token_ids_a + token_ids_a + [self.eos_token_id]






    def     _UpperCamelCase							(			self     ,A     ,A = None ):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer.""" )

          if not os.path.isdir(A ):
                logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
                return
          UpperCAmelCase          =	os.path.join(
              A     ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

          if os.path.abspath(self.vocab_file ) != os.path.abspath(A ):
                copyfile(self.vocab_file     ,A )

          return (out_vocab_file,)



