





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase				:	Optional[Any]      						=  logging.get_logger(__name__)


lowercase				:	Union[str, Any]      						=  {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all BART models at https://huggingface.co/models?filter=bart
lowercase				:	List[str]      						=  {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
}

lowercase				:	Dict      						=  {
    """facebook/bart-base""": 1_0_2_4,
    """facebook/bart-large""": 1_0_2_4,
    """facebook/bart-large-mnli""": 1_0_2_4,
    """facebook/bart-large-cnn""": 1_0_2_4,
    """facebook/bart-large-xsum""": 1_0_2_4,
    """yjernite/bart_eli5""": 1_0_2_4,
}



@lru_cache()
def A_     (		)  ->		Optional[Any]:
							a__						:     Union[str, Any]			=    (
							    list(range(ord('!'    )		,				ord('~'    ) + 1    )    ) + list(range(ord('¡'    )		,				ord('¬'    ) + 1    )    ) + list(range(ord('®'    )		,				ord('ÿ'    ) + 1    )    )
							)
							a__						:     List[Any]			=    bs[:]
							a__						:     str			=    0
							for b in range(2**8    ):
														if b not in bs:
																					bs.append(A__    )
																					cs.append(2**8 + n    )
																					n += 1
							a__						:     List[Any]			=    [chr(A__    ) for n in cs]
							return dict(zip(A__		,				A__    )    )



def A_     (		A__    )  ->		List[Any]:
							a__						:     Optional[Any]			=    set()
							a__						:     Union[str, Any]			=    word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)    )
														a__						:     str			=    char
							return pairs





class 			A__   (							__UpperCAmelCase     ):




			"""simple docstring"""



			__A       :    Dict	  =  VOCAB_FILES_NAMES
			__A       :    Tuple	  =  PRETRAINED_VOCAB_FILES_MAP
			__A       :    int	  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__A       :    Union[str, Any]	  =  ['''input_ids''', '''attention_mask''']

			def __init__( self ,  lowercase ,  lowercase ,  lowercase="replace" ,  lowercase="<s>" ,  lowercase="</s>" ,  lowercase="</s>" ,  lowercase="<s>" ,  lowercase="<unk>" ,  lowercase="<pad>" ,  lowercase="<mask>" ,  lowercase=False ,  **lowercase ,  )      ->						Any:







										'''simple docstring'''

										a__						:     Dict			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else bos_token
										a__						:     Union[str, Any]			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else eos_token
										a__						:     Optional[int]			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else sep_token
										a__						:     str			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else cls_token
										a__						:     int			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else unk_token
										a__						:     int			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										a__						:     Any			=    AddedToken(lowercase ,  lstrip=lowercase ,  rstrip=lowercase) if isinstance(lowercase ,  lowercase) else mask_token

										super().__init__(
										    errors=lowercase ,  bos_token=lowercase ,  eos_token=lowercase ,  unk_token=lowercase ,  sep_token=lowercase ,  cls_token=lowercase ,  pad_token=lowercase ,  mask_token=lowercase ,  add_prefix_space=lowercase ,  **lowercase ,  )

										with open(lowercase ,  encoding='utf-8') as vocab_handle:
																	a__						:     Union[str, Any]			=    json.load(lowercase)
										a__						:     Optional[int]			=    {v: k for k, v in self.encoder.items()}
										a__						:     Optional[Any]			=    errors  # how to handle errors in decoding
										a__						:     List[Any]			=    bytes_to_unicode()
										a__						:     int			=    {v: k for k, v in self.byte_encoder.items()}
										with open(lowercase ,  encoding='utf-8') as merges_handle:
																	a__						:     List[Any]			=    merges_handle.read().split('\n')[1:-1]
										a__						:     Optional[Any]			=    [tuple(merge.split()) for merge in bpe_merges]
										a__						:     int			=    dict(zip(lowercase ,  range(len(lowercase))))
										a__						:     int			=    {}
										a__						:     Union[str, Any]			=    add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										a__						:     int			=    re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+')

			@property
			def 							__lowercase		( self)      ->						List[Any]:







										'''simple docstring'''

										return len(self.encoder)

			def 							__lowercase		( self)      ->						Optional[int]:







										'''simple docstring'''

										return dict(self.encoder ,  **self.added_tokens_encoder)

			def 							__lowercase		( self ,  lowercase)      ->						List[str]:







										'''simple docstring'''

										if token in self.cache:
																	return self.cache[token]
										a__						:     Dict			=    tuple(lowercase)
										a__						:     Optional[Any]			=    get_pairs(lowercase)

										if not pairs:
																	return token

										while True:
																	a__						:     Union[str, Any]			=    min(lowercase ,  key=lambda lowercase: self.bpe_ranks.get(lowercase ,  float('inf')))
																	if bigram not in self.bpe_ranks:
																								break
																	a__						:     List[Any]			=    bigram
																	a__						:     Optional[Any]			=    []
																	a__						:     List[str]			=    0
																	while i < len(lowercase):
																								try:
																															a__						:     List[str]			=    word.index(lowercase ,  lowercase)
																								except ValueError:
																															new_word.extend(word[i:])
																															break
																								else:
																															new_word.extend(word[i:j])
																															a__						:     Any			=    j

																								if word[i] == first and i < len(lowercase) - 1 and word[i + 1] == second:
																															new_word.append(first + second)
																															i += 2
																								else:
																															new_word.append(word[i])
																															i += 1
																	a__						:     Optional[Any]			=    tuple(lowercase)
																	a__						:     List[Any]			=    new_word
																	if len(lowercase) == 1:
																								break
																	else:
																								a__						:     str			=    get_pairs(lowercase)
										a__						:     Optional[Any]			=    ' '.join(lowercase)
										a__						:     Optional[Any]			=    word
										return word

			def 							__lowercase		( self ,  lowercase)      ->						Any:







										'''simple docstring'''

										a__						:     Optional[int]			=    []
										for token in re.findall(self.pat ,  lowercase):
																	a__						:     Tuple			=    ''.join(
																	    self.byte_encoder[b] for b in token.encode('utf-8'))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																	bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase).split(' '))
										return bpe_tokens

			def 							__lowercase		( self ,  lowercase)      ->						Optional[Any]:







										'''simple docstring'''

										return self.encoder.get(lowercase ,  self.encoder.get(self.unk_token))

			def 							__lowercase		( self ,  lowercase)      ->						Union[str, Any]:







										'''simple docstring'''

										return self.decoder.get(lowercase)

			def 							__lowercase		( self ,  lowercase)      ->						int:







										'''simple docstring'''

										a__						:     Tuple			=    ''.join(lowercase)
										a__						:     Optional[int]			=    bytearray([self.byte_decoder[c] for c in text]).decode('utf-8' ,  errors=self.errors)
										return text

			def 							__lowercase		( self ,  lowercase ,  lowercase = None)      ->						Tuple[str]:







										'''simple docstring'''

										if not os.path.isdir(lowercase):
																	logger.error(F'Vocabulary path ({save_directory}) should be a directory')
																	return
										a__						:     Union[str, Any]			=    os.path.join(
										    lowercase ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
										a__						:     Optional[Any]			=    os.path.join(
										    lowercase ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

										with open(lowercase ,  'w' ,  encoding='utf-8') as f:
																	f.write(json.dumps(self.encoder ,  indent=2 ,  sort_keys=lowercase ,  ensure_ascii=lowercase) + '\n')

										a__						:     Dict			=    0
										with open(lowercase ,  'w' ,  encoding='utf-8') as writer:
																	writer.write('#version: 0.2\n')
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,  key=lambda lowercase: kv[1]):
																								if index != token_index:
																															logger.warning(
																															    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																															    ' Please check that the tokenizer is not corrupted!')
																															a__						:     int			=    token_index
																								writer.write(' '.join(lowercase) + '\n')
																								index += 1

										return vocab_file, merge_file

			def 							__lowercase		( self ,  lowercase ,  lowercase = None)      ->						List[int]:







										'''simple docstring'''

										if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										a__						:     Optional[Any]			=    [self.cls_token_id]
										a__						:     Dict			=    [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep

			def 							__lowercase		( self ,  lowercase ,  lowercase = None ,  lowercase = False)      ->						List[int]:







										'''simple docstring'''

										if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=lowercase ,  token_ids_a=lowercase ,  already_has_special_tokens=lowercase)

										if token_ids_a is None:
																	return [1] + ([0] * len(lowercase)) + [1]
										return [1] + ([0] * len(lowercase)) + [1, 1] + ([0] * len(lowercase)) + [1]

			def 							__lowercase		( self ,  lowercase ,  lowercase = None)      ->						List[int]:







										'''simple docstring'''

										a__						:     Optional[Any]			=    [self.sep_token_id]
										a__						:     List[str]			=    [self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]

			def 							__lowercase		( self ,  lowercase ,  lowercase=False ,  **lowercase)      ->						str:







										'''simple docstring'''

										a__						:     str			=    kwargs.pop('add_prefix_space' ,  self.add_prefix_space)
										if (is_split_into_words or add_prefix_space) and (len(lowercase) > 0 and not text[0].isspace()):
																	a__						:     Dict			=    ' ' + text
										return (text, kwargs)









import unittest

import numpy as np

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
    from transformers.models.bert.modeling_flax_bert import (
        FlaxBertForMaskedLM,
        FlaxBertForMultipleChoice,
        FlaxBertForNextSentencePrediction,
        FlaxBertForPreTraining,
        FlaxBertForQuestionAnswering,
        FlaxBertForSequenceClassification,
        FlaxBertForTokenClassification,
        FlaxBertModel,
    )





class 			A__   (							unittest.TestCase     ):




   """simple docstring"""



   def __init__( self ,  lowercase ,  lowercase=13 ,  lowercase=7 ,  lowercase=True ,  lowercase=True ,  lowercase=True ,  lowercase=True ,  lowercase=99 ,  lowercase=32 ,  lowercase=5 ,  lowercase=4 ,  lowercase=37 ,  lowercase="gelu" ,  lowercase=0.1 ,  lowercase=0.1 ,  lowercase=512 ,  lowercase=16 ,  lowercase=2 ,  lowercase=0.02 ,  lowercase=4 ,  )      ->						Tuple:







          '''simple docstring'''

          a__						:     str			=    parent
          a__						:     Optional[Any]			=    batch_size
          a__						:     str			=    seq_length
          a__						:     int			=    is_training
          a__						:     str			=    use_attention_mask
          a__						:     List[str]			=    use_token_type_ids
          a__						:     Optional[Any]			=    use_labels
          a__						:     List[Any]			=    vocab_size
          a__						:     Tuple			=    hidden_size
          a__						:     Dict			=    num_hidden_layers
          a__						:     List[str]			=    num_attention_heads
          a__						:     int			=    intermediate_size
          a__						:     Any			=    hidden_act
          a__						:     Optional[int]			=    hidden_dropout_prob
          a__						:     Tuple			=    attention_probs_dropout_prob
          a__						:     Tuple			=    max_position_embeddings
          a__						:     Optional[int]			=    type_vocab_size
          a__						:     List[Any]			=    type_sequence_label_size
          a__						:     Union[str, Any]			=    initializer_range
          a__						:     str			=    num_choices

   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     str			=    ids_tensor([self.batch_size, self.seq_length] ,  self.vocab_size)

          a__						:     Dict			=    None
          if self.use_attention_mask:
                 a__						:     Optional[int]			=    random_attention_mask([self.batch_size, self.seq_length])

          a__						:     Dict			=    None
          if self.use_token_type_ids:
                 a__						:     int			=    ids_tensor([self.batch_size, self.seq_length] ,  self.type_vocab_size)

          a__						:     Dict			=    BertConfig(
              vocab_size=self.vocab_size ,  hidden_size=self.hidden_size ,  num_hidden_layers=self.num_hidden_layers ,  num_attention_heads=self.num_attention_heads ,  intermediate_size=self.intermediate_size ,  hidden_act=self.hidden_act ,  hidden_dropout_prob=self.hidden_dropout_prob ,  attention_probs_dropout_prob=self.attention_probs_dropout_prob ,  max_position_embeddings=self.max_position_embeddings ,  type_vocab_size=self.type_vocab_size ,  is_decoder=lowercase ,  initializer_range=self.initializer_range ,  )

          return config, input_ids, token_type_ids, attention_mask

   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     Tuple			=    self.prepare_config_and_inputs()
          a__			,							a__			,							a__			,							a__						:     Optional[int]			=    config_and_inputs
          a__						:     str			=    {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
          return config, inputs_dict

   def 							__lowercase		( self)      ->						Optional[int]:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    self.prepare_config_and_inputs()
          a__			,							a__			,							a__			,							a__						:     int			=    config_and_inputs

          a__						:     str			=    True
          a__						:     Any			=    floats_tensor([self.batch_size, self.seq_length, self.hidden_size])
          a__						:     List[str]			=    ids_tensor([self.batch_size, self.seq_length] ,  vocab_size=2)

          return (
              config,
              input_ids,
              attention_mask,
              encoder_hidden_states,
              encoder_attention_mask,
          )





@require_flax
class 			A__   (							__UpperCAmelCase			,  unittest.TestCase     ):




   """simple docstring"""



   __A       :    Optional[Any]	  =  True

   __A       :    Tuple	  =  (
       (
           FlaxBertModel,
           FlaxBertForPreTraining,
           FlaxBertForMaskedLM,
           FlaxBertForMultipleChoice,
           FlaxBertForQuestionAnswering,
           FlaxBertForNextSentencePrediction,
           FlaxBertForSequenceClassification,
           FlaxBertForTokenClassification,
           FlaxBertForQuestionAnswering,
       )
       if is_flax_available()
       else ()
   )

   def 							__lowercase		( self)      ->						Tuple:







          '''simple docstring'''

          a__						:     Optional[Any]			=    FlaxBertModelTester(self)

   @slow
   def 							__lowercase		( self)      ->						Optional[Any]:







          '''simple docstring'''

          a__						:     Union[str, Any]			=    FlaxBertModel.from_pretrained('bert-base-cased')
          a__						:     Optional[Any]			=    model(np.ones((1, 1)))
          self.assertIsNotNone(lowercase)


