




import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase     :		Tuple								=       logging.get_logger(__name__)


_lowerCamelCase     :		Union[str, Any]								=       {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

# See all BART models at https://huggingface.co/models?filter=bart
_lowerCamelCase     :		Dict								=       {
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
}

_lowerCamelCase     :		Tuple								=       {
    "facebook/bart-base": 1024,
    "facebook/bart-large": 1024,
    "facebook/bart-large-mnli": 1024,
    "facebook/bart-large-cnn": 1024,
    "facebook/bart-large-xsum": 1024,
    "yjernite/bart_eli5": 1024,
}







@lru_cache()
def SCREAMING_SNAKE_CASE    (	)       ->	int:



       """simple docstring"""

       A__							=   (
           list(range(ord('''!'''						)			,    ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)			,    ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)			,    ord('''ÿ'''						) + 1						)						)
       )
       A__							=   bs[:]
       A__							=   0
       for b in range(2**8						):
              if b not in bs:
                     bs.append(_UpperCAmelCase						)
                     cs.append(2**8 + n						)
                     n += 1
       A__							=   [chr(_UpperCAmelCase						) for n in cs]
       return dict(zip(_UpperCAmelCase			,    _UpperCAmelCase						)						)







def SCREAMING_SNAKE_CASE    (	lowercase_						)       ->	Dict:



       """simple docstring"""

       A__							=   set()
       A__							=   word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)						)
              A__							=   char
       return pairs


class     UpperCamelCase_   (			snake_case__					):

    '''simple docstring'''






    UpperCAmelCase__							  =				VOCAB_FILES_NAMES
    UpperCAmelCase__							  =				PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__							  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__							  =				['''input_ids''', '''attention_mask''']


    def __init__(      self				:     Optional[Any] ,			UpperCAmelCase__				:     List[Any] ,			UpperCAmelCase__				:     Optional[int] ,			UpperCAmelCase__				:     int="replace" ,			UpperCAmelCase__				:     Union[str, Any]="<s>" ,			UpperCAmelCase__				:     List[str]="</s>" ,			UpperCAmelCase__				:     List[Any]="</s>" ,			UpperCAmelCase__				:     int="<s>" ,			UpperCAmelCase__				:     int="<unk>" ,			UpperCAmelCase__				:     Dict="<pad>" ,			UpperCAmelCase__				:     Union[str, Any]="<mask>" ,			UpperCAmelCase__				:     Union[str, Any]=False ,			**UpperCAmelCase__				:     Any ,			)						->Dict:





           '''simple docstring'''

           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else bos_token
           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else eos_token
           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else sep_token
           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else cls_token
           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else unk_token
           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           A__							=   AddedToken(UpperCAmelCase_ ,			lstrip=UpperCAmelCase_ ,			rstrip=UpperCAmelCase_) if isinstance(UpperCAmelCase_ ,			UpperCAmelCase_) else mask_token

           super().__init__(
               errors=UpperCAmelCase_ ,			bos_token=UpperCAmelCase_ ,			eos_token=UpperCAmelCase_ ,			unk_token=UpperCAmelCase_ ,			sep_token=UpperCAmelCase_ ,			cls_token=UpperCAmelCase_ ,			pad_token=UpperCAmelCase_ ,			mask_token=UpperCAmelCase_ ,			add_prefix_space=UpperCAmelCase_ ,			**UpperCAmelCase_ ,			)

           with open(UpperCAmelCase_ ,			encoding='''utf-8''') as vocab_handle:
                  A__							=   json.load(UpperCAmelCase_)
           A__							=   {v: k for k, v in self.encoder.items()}
           A__							=   errors  # how to handle errors in decoding
           A__							=   bytes_to_unicode()
           A__							=   {v: k for k, v in self.byte_encoder.items()}
           with open(UpperCAmelCase_ ,			encoding='''utf-8''') as merges_handle:
                  A__							=   merges_handle.read().split('''\n''')[1:-1]
           A__							=   [tuple(merge.split()) for merge in bpe_merges]
           A__							=   dict(zip(UpperCAmelCase_ ,			range(len(UpperCAmelCase_))))
           A__							=   {}
           A__							=   add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           A__							=   re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''')


    @property
    def 	SCREAMING_SNAKE_CASE							(      self				:     Tuple)						->Dict:





           '''simple docstring'''

           return len(self.encoder)


    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[int])						->int:





           '''simple docstring'''

           return dict(self.encoder ,			**self.added_tokens_encoder)


    def 	SCREAMING_SNAKE_CASE							(      self				:     int ,			UpperCAmelCase__				:     Optional[int])						->Optional[int]:





           '''simple docstring'''

           if token in self.cache:
                  return self.cache[token]
           A__							=   tuple(UpperCAmelCase_)
           A__							=   get_pairs(UpperCAmelCase_)

           if not pairs:
                  return token

           while True:
                  A__							=   min(UpperCAmelCase_ ,			key=lambda UpperCAmelCase__: self.bpe_ranks.get(UpperCAmelCase_ ,			float('''inf''')))
                  if bigram not in self.bpe_ranks:
                         break
                  A__							=   bigram
                  A__							=   []
                  A__							=   0
                  while i < len(UpperCAmelCase_):
                         try:
                                A__							=   word.index(UpperCAmelCase_ ,			UpperCAmelCase_)
                         except ValueError:
                                new_word.extend(word[i:])
                                break
                         else:
                                new_word.extend(word[i:j])
                                A__							=   j

                         if word[i] == first and i < len(UpperCAmelCase_) - 1 and word[i + 1] == second:
                                new_word.append(first + second)
                                i += 2
                         else:
                                new_word.append(word[i])
                                i += 1
                  A__							=   tuple(UpperCAmelCase_)
                  A__							=   new_word
                  if len(UpperCAmelCase_) == 1:
                         break
                  else:
                         A__							=   get_pairs(UpperCAmelCase_)
           A__							=   " ".join(UpperCAmelCase_)
           A__							=   word
           return word


    def 	SCREAMING_SNAKE_CASE							(      self				:     Tuple ,			UpperCAmelCase__				:     Union[str, Any])						->Any:





           '''simple docstring'''

           A__							=   []
           for token in re.findall(self.pat ,			UpperCAmelCase_):
                  A__							=   "".join(
                      self.byte_encoder[b] for b in token.encode('''utf-8'''))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                  bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_).split(''' '''))
           return bpe_tokens


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[str] ,			UpperCAmelCase__				:     Dict)						->Union[str, Any]:





           '''simple docstring'''

           return self.encoder.get(UpperCAmelCase_ ,			self.encoder.get(self.unk_token))


    def 	SCREAMING_SNAKE_CASE							(      self				:     Tuple ,			UpperCAmelCase__				:     int)						->List[Any]:





           '''simple docstring'''

           return self.decoder.get(UpperCAmelCase_)


    def 	SCREAMING_SNAKE_CASE							(      self				:     str ,			UpperCAmelCase__				:     List[str])						->Union[str, Any]:





           '''simple docstring'''

           A__							=   "".join(UpperCAmelCase_)
           A__							=   bytearray([self.byte_decoder[c] for c in text]).decode('''utf-8''' ,			errors=self.errors)
           return text


    def 	SCREAMING_SNAKE_CASE							(      self				:     Dict ,			UpperCAmelCase__				:     str ,			UpperCAmelCase__				:     Optional[str] = None)						->str:





           '''simple docstring'''

           if not os.path.isdir(UpperCAmelCase_):
                  logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
                  return
           A__							=   os.path.join(
               UpperCAmelCase_ ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
           A__							=   os.path.join(
               UpperCAmelCase_ ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

           with open(UpperCAmelCase_ ,			'''w''' ,			encoding='''utf-8''') as f:
                  f.write(json.dumps(self.encoder ,			indent=2 ,			sort_keys=UpperCAmelCase_ ,			ensure_ascii=UpperCAmelCase_) + '''\n''')

           A__							=   0
           with open(UpperCAmelCase_ ,			'''w''' ,			encoding='''utf-8''') as writer:
                  writer.write('''#version: 0.2\n''')
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,			key=lambda UpperCAmelCase__: kv[1]):
                         if index != token_index:
                                logger.warning(
                                    f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                                    ''' Please check that the tokenizer is not corrupted!''')
                                A__							=   token_index
                         writer.write(''' '''.join(UpperCAmelCase_) + '''\n''')
                         index += 1

           return vocab_file, merge_file


    def 	SCREAMING_SNAKE_CASE							(      self				:     Any ,			UpperCAmelCase__				:     List[int] ,			UpperCAmelCase__				:     Optional[List[int]] = None)						->Dict:





           '''simple docstring'''

           if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           A__							=   [self.cls_token_id]
           A__							=   [self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep


    def 	SCREAMING_SNAKE_CASE							(      self				:     Tuple ,			UpperCAmelCase__				:     List[int] ,			UpperCAmelCase__				:     Optional[List[int]] = None ,			UpperCAmelCase__				:     bool = False)						->Optional[Any]:





           '''simple docstring'''

           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=UpperCAmelCase_ ,			token_ids_a=UpperCAmelCase_ ,			already_has_special_tokens=UpperCAmelCase_)

           if token_ids_a is None:
                  return [1] + ([0] * len(UpperCAmelCase_)) + [1]
           return [1] + ([0] * len(UpperCAmelCase_)) + [1, 1] + ([0] * len(UpperCAmelCase_)) + [1]


    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any] ,			UpperCAmelCase__				:     List[int] ,			UpperCAmelCase__				:     Optional[List[int]] = None)						->Optional[int]:





           '''simple docstring'''

           A__							=   [self.sep_token_id]
           A__							=   [self.cls_token_id]

           if token_ids_a is None:
                  return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]





    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[int] ,			UpperCAmelCase__				:     Optional[Any] ,			UpperCAmelCase__				:     Tuple=False ,			**UpperCAmelCase__				:     Any)						->Tuple:





           '''simple docstring'''

           A__							=   kwargs.pop('''add_prefix_space''' ,			self.add_prefix_space)
           if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase_) > 0 and not text[0].isspace()):
                  A__							=   " " + text
           return (text, kwargs)








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


_lowerCamelCase     :		Tuple								=       {
    """configuration_speech_to_text""": ["""SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """Speech2TextConfig"""],
    """processing_speech_to_text""": ["""Speech2TextProcessor"""],
}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase     :		int								=       ["""Speech2TextTokenizer"""]

try:
  if not is_speech_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase     :		Optional[int]								=       ["""Speech2TextFeatureExtractor"""]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase     :		Optional[Any]								=       [
      """TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFSpeech2TextForConditionalGeneration""",
      """TFSpeech2TextModel""",
      """TFSpeech2TextPreTrainedModel""",
  ]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _lowerCamelCase     :		List[Any]								=       [
      """SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """Speech2TextForConditionalGeneration""",
      """Speech2TextModel""",
      """Speech2TextPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
  from .processing_speech_to_text import SpeechaTextProcessor

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_speech_to_text import SpeechaTextTokenizer

  try:
    if not is_speech_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_speech_to_text import (
        TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFSpeechaTextForConditionalGeneration,
        TFSpeechaTextModel,
        TFSpeechaTextPreTrainedModel,
    )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_speech_to_text import (
        SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
        SpeechaTextForConditionalGeneration,
        SpeechaTextModel,
        SpeechaTextPreTrainedModel,
    )

else:
  import sys

  _lowerCamelCase     :		Union[str, Any]								=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


