




"""simple docstring"""


import numpy as np

from transformers import BatchFeature
from transformers.testing_utils import require_tf, require_torch

from .test_feature_extraction_common import FeatureExtractionSavingTestMixin


class _lowerCamelCase    (					a_						):
   # to overwrite at feature extractactor specific tests
   _lowerCamelCase    :Optional[int]     				=     None
   _lowerCamelCase    :List[Any]     				=     None






   @property
   def 	_lowerCAmelCase   (		self       :		int   )			->						Tuple:

         """simple docstring"""
         return self.feat_extract_tester.prepare_feat_extract_dict()






   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						List[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	self.feature_extraction_class(**self.feat_extract_dict   )
         self.assertTrue(hasattr(UpperCamelCase   ,     """feature_size"""   )   )
         self.assertTrue(hasattr(UpperCamelCase   ,     """sampling_rate"""   )   )
         self.assertTrue(hasattr(UpperCamelCase   ,     """padding_value"""   )   )






   def 	_lowerCAmelCase   (		self       :		Any   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			Dict   =	self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__      :			List[Any]   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			Union[str, Any]   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			Any   =	BatchFeature({input_name: speech_inputs}   )

         self.assertTrue(all(len(UpperCamelCase   ) == len(UpperCamelCase   ) for x, y in zip(UpperCamelCase   ,     processed_features[input_name]   )   )   )

         lowerCAmelCase__      :			List[str]   =	self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCamelCase   )
         lowerCAmelCase__      :			List[Any]   =	BatchFeature({input_name: speech_inputs}   ,     tensor_type="""np"""   )

         lowerCAmelCase__      :			Dict   =	processed_features[input_name]

         if len(batch_features_input.shape   ) < 3:
               lowerCAmelCase__      :			int   =	batch_features_input[:, :, None]

         self.assertTrue(
             batch_features_input.shape
             == (self.feat_extract_tester.batch_size, len(speech_inputs[0]   ), self.feat_extract_tester.feature_size)   )






   @require_torch
   def 	_lowerCAmelCase   (		self       :		Tuple   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCamelCase   )
         lowerCAmelCase__      :			str   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			Optional[int]   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			Union[str, Any]   =	BatchFeature({input_name: speech_inputs}   ,     tensor_type="""pt"""   )

         lowerCAmelCase__      :			str   =	processed_features[input_name]

         if len(batch_features_input.shape   ) < 3:
               lowerCAmelCase__      :			int   =	batch_features_input[:, :, None]

         self.assertTrue(
             batch_features_input.shape
             == (self.feat_extract_tester.batch_size, len(speech_inputs[0]   ), self.feat_extract_tester.feature_size)   )






   @require_tf
   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			List[str]   =	self.feat_extract_tester.prepare_inputs_for_common(equal_length=UpperCamelCase   )
         lowerCAmelCase__      :			str   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			Dict   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			Tuple   =	BatchFeature({input_name: speech_inputs}   ,     tensor_type="""tf"""   )

         lowerCAmelCase__      :			Union[str, Any]   =	processed_features[input_name]

         if len(batch_features_input.shape   ) < 3:
               lowerCAmelCase__      :			Optional[Any]   =	batch_features_input[:, :, None]

         self.assertTrue(
             batch_features_input.shape
             == (self.feat_extract_tester.batch_size, len(speech_inputs[0]   ), self.feat_extract_tester.feature_size)   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		Union[str, Any]=False   )			->						int:

         """simple docstring"""
         def _inputs_have_equal_length(UpperCamelCase       :		int   ):
               lowerCAmelCase__      :			Optional[Any]   =	len(input[0]   )
               for input_slice in input[1:]:
                     if len(UpperCamelCase   ) != length:
                           return False
               return True

         def _inputs_are_equal(UpperCamelCase       :		int   ,     UpperCamelCase       :		Dict   ):
               if len(UpperCamelCase   ) != len(UpperCamelCase   ):
                     return False

               for input_slice_a, input_slice_a in zip(UpperCamelCase   ,     UpperCamelCase   ):
                     if not np.allclose(np.asarray(UpperCamelCase   )   ,     np.asarray(UpperCamelCase   )   ,     atol=1E-3   ):
                           return False
               return True

         lowerCAmelCase__      :			str   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			Dict   =	self.feat_extract_tester.prepare_inputs_for_common(numpify=UpperCamelCase   )
         lowerCAmelCase__      :			str   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			str   =	BatchFeature({input_name: speech_inputs}   )

         lowerCAmelCase__      :			Any   =	self.feat_extract_tester.seq_length_diff
         lowerCAmelCase__      :			List[str]   =	self.feat_extract_tester.max_seq_length + pad_diff
         lowerCAmelCase__      :			Optional[int]   =	self.feat_extract_tester.min_seq_length
         lowerCAmelCase__      :			Optional[int]   =	self.feat_extract_tester.batch_size
         lowerCAmelCase__      :			str   =	self.feat_extract_tester.feature_size

         # test padding for List[int] + numpy
         lowerCAmelCase__      :			List[Any]   =	feat_extract.pad(UpperCamelCase   ,     padding=UpperCamelCase   )
         lowerCAmelCase__      :			int   =	input_a[input_name]

         lowerCAmelCase__      :			Tuple   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   )
         lowerCAmelCase__      :			Any   =	input_a[input_name]

         lowerCAmelCase__      :			List[str]   =	feat_extract.pad(UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[-1]   )   )
         lowerCAmelCase__      :			Dict   =	input_a[input_name]

         lowerCAmelCase__      :			str   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     return_tensors="""np"""   )
         lowerCAmelCase__      :			Optional[Any]   =	input_a[input_name]

         # max_length parameter has to be provided when setting `padding="max_length"`
         with self.assertRaises(UpperCamelCase   ):
               feat_extract.pad(UpperCamelCase   ,     padding="""max_length"""   )[input_name]

         lowerCAmelCase__      :			Optional[int]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=UpperCamelCase   ,     return_tensors="""np"""   )
         lowerCAmelCase__      :			int   =	input_a[input_name]

         self.assertFalse(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(_inputs_are_equal(UpperCamelCase   ,     UpperCamelCase   )   )
         self.assertTrue(len(input_a[0]   ) == pad_min_length   )
         self.assertTrue(len(input_a[1]   ) == pad_min_length + pad_diff   )
         self.assertTrue(input_a.shape[:2] == (batch_size, len(input_a[0]   ))   )
         self.assertTrue(input_a.shape[:2] == (batch_size, pad_max_length)   )

         if feature_size > 1:
               self.assertTrue(input_a.shape[2] == input_a.shape[2] == feature_size   )

         # test padding for `pad_to_multiple_of` for List[int] + numpy
         lowerCAmelCase__      :			Any   =	feat_extract.pad(UpperCamelCase   ,     pad_to_multiple_of=10   )
         lowerCAmelCase__      :			int   =	input_a[input_name]

         lowerCAmelCase__      :			Dict   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     pad_to_multiple_of=10   )
         lowerCAmelCase__      :			Union[str, Any]   =	input_a[input_name]

         lowerCAmelCase__      :			Optional[int]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     pad_to_multiple_of=10   ,     max_length=UpperCamelCase   )
         lowerCAmelCase__      :			str   =	input_a[input_name]

         lowerCAmelCase__      :			Union[str, Any]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     pad_to_multiple_of=10   ,     max_length=UpperCamelCase   ,     return_tensors="""np"""   ,     )
         lowerCAmelCase__      :			List[Any]   =	input_a[input_name]

         self.assertTrue(all(len(UpperCamelCase   ) % 10 == 0 for x in input_a   )   )
         self.assertTrue(_inputs_are_equal(UpperCamelCase   ,     UpperCamelCase   )   )

         lowerCAmelCase__      :			Optional[int]   =	pad_max_length if pad_max_length % 10 == 0 else (pad_max_length // 10 + 1) * 10
         self.assertTrue(all(len(UpperCamelCase   ) == expected_mult_pad_length for x in input_a   )   )
         self.assertEqual(input_a.shape[:2]   ,     (batch_size, expected_mult_pad_length)   )

         if feature_size > 1:
               self.assertTrue(input_a.shape[2] == feature_size   )

         # Check padding value is correct
         lowerCAmelCase__      :			str   =	(np.ones(self.feat_extract_tester.feature_size   ) * feat_extract.padding_value).sum()
         self.assertTrue(
             abs(np.asarray(input_a[0]   )[pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)   )
             < 1E-3   )
         self.assertTrue(
             abs(
                 np.asarray(input_a[1]   )[pad_min_length + pad_diff :].sum()
                 - padding_vector_sum * (pad_max_length - pad_min_length - pad_diff)   )
             < 1E-3   )
         self.assertTrue(
             abs(
                 np.asarray(input_a[2]   )[pad_min_length + 2 * pad_diff :].sum()
                 - padding_vector_sum * (pad_max_length - pad_min_length - 2 * pad_diff)   )
             < 1E-3   )
         self.assertTrue(
             abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (pad_max_length - pad_min_length)   ) < 1E-3   )
         self.assertTrue(
             abs(input_a[0, pad_min_length:].sum() - padding_vector_sum * (expected_mult_pad_length - pad_min_length)   )
             < 1E-3   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   ,     UpperCamelCase       :		int=False   )			->						List[str]:

         """simple docstring"""
         def _inputs_have_equal_length(UpperCamelCase       :		List[Any]   ):
               lowerCAmelCase__      :			List[str]   =	len(input[0]   )
               for input_slice in input[1:]:
                     if len(UpperCamelCase   ) != length:
                           return False
               return True

         def _inputs_are_equal(UpperCamelCase       :		Union[str, Any]   ,     UpperCamelCase       :		str   ):
               if len(UpperCamelCase   ) != len(UpperCamelCase   ):
                     return False

               for input_slice_a, input_slice_a in zip(UpperCamelCase   ,     UpperCamelCase   ):
                     if not np.allclose(np.asarray(UpperCamelCase   )   ,     np.asarray(UpperCamelCase   )   ,     atol=1E-3   ):
                           return False
               return True

         lowerCAmelCase__      :			Dict   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			Any   =	self.feat_extract_tester.prepare_inputs_for_common(numpify=UpperCamelCase   )
         lowerCAmelCase__      :			str   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			Optional[Any]   =	BatchFeature({input_name: speech_inputs}   )

         # truncate to smallest
         lowerCAmelCase__      :			str   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[0]   )   ,     truncation=UpperCamelCase   )
         lowerCAmelCase__      :			Dict   =	input_a[input_name]

         lowerCAmelCase__      :			str   =	feat_extract.pad(UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[0]   )   )
         lowerCAmelCase__      :			Optional[int]   =	input_a[input_name]

         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertFalse(_inputs_have_equal_length(UpperCamelCase   )   )

         # truncate to smallest with np
         lowerCAmelCase__      :			Tuple   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[0]   )   ,     return_tensors="""np"""   ,     truncation=UpperCamelCase   ,     )
         lowerCAmelCase__      :			List[Any]   =	input_a[input_name]

         lowerCAmelCase__      :			List[Any]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[0]   )   ,     return_tensors="""np"""   )
         lowerCAmelCase__      :			Union[str, Any]   =	input_a[input_name]

         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(input_a.shape[1] == len(speech_inputs[0]   )   )

         # since truncation forces padding to be smaller than longest input
         # function can't return `np.ndarray`, but has to return list
         self.assertFalse(_inputs_have_equal_length(UpperCamelCase   )   )

         # truncate to middle
         lowerCAmelCase__      :			List[Any]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[1]   )   ,     truncation=UpperCamelCase   ,     return_tensors="""np"""   ,     )
         lowerCAmelCase__      :			List[Any]   =	input_a[input_name]

         lowerCAmelCase__      :			List[Any]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[1]   )   ,     truncation=UpperCamelCase   )
         lowerCAmelCase__      :			List[str]   =	input_a[input_name]

         lowerCAmelCase__      :			Any   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[1]   )   ,     return_tensors="""np"""   )
         lowerCAmelCase__      :			Dict   =	input_a[input_name]

         self.assertTrue(input_a.shape[1] == len(speech_inputs[1]   )   )
         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(_inputs_are_equal(UpperCamelCase   ,     UpperCamelCase   )   )

         # since truncation forces padding to be smaller than longest input
         # function can't return `np.ndarray`, but has to return list
         self.assertFalse(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertTrue(len(input_a[-1]   ) == len(speech_inputs[-1]   )   )

         # padding has to be max_length when setting `truncation=True`
         with self.assertRaises(UpperCamelCase   ):
               feat_extract.pad(UpperCamelCase   ,     truncation=UpperCamelCase   )[input_name]

         # padding has to be max_length when setting `truncation=True`
         with self.assertRaises(UpperCamelCase   ):
               feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     truncation=UpperCamelCase   )[input_name]

         # padding has to be max_length when setting `truncation=True`
         with self.assertRaises(UpperCamelCase   ):
               feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     truncation=UpperCamelCase   )[input_name]

         # max_length parameter has to be provided when setting `truncation=True` and padding="max_length"
         with self.assertRaises(UpperCamelCase   ):
               feat_extract.pad(UpperCamelCase   ,     padding="""max_length"""   ,     truncation=UpperCamelCase   )[input_name]

         # test truncation for `pad_to_multiple_of` for List[int] + numpy
         lowerCAmelCase__      :			Any   =	12
         lowerCAmelCase__      :			Optional[int]   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[0]   )   ,     pad_to_multiple_of=UpperCamelCase   ,     truncation=UpperCamelCase   ,     )
         lowerCAmelCase__      :			List[Any]   =	input_a[input_name]

         lowerCAmelCase__      :			Any   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=len(speech_inputs[0]   )   ,     pad_to_multiple_of=UpperCamelCase   ,     )
         lowerCAmelCase__      :			Optional[Any]   =	input_a[input_name]

         # retrieve expected_length as multiple of pad_to_multiple_of
         lowerCAmelCase__      :			Union[str, Any]   =	len(speech_inputs[0]   )
         if expected_length % pad_to_multiple_of != 0:
               lowerCAmelCase__      :			Any   =	((len(speech_inputs[0]   ) // pad_to_multiple_of) + 1) * pad_to_multiple_of

         self.assertTrue(len(input_a[0]   ) == expected_length   )
         self.assertTrue(_inputs_have_equal_length(UpperCamelCase   )   )
         self.assertFalse(_inputs_have_equal_length(UpperCamelCase   )   )






   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Optional[int]:

         """simple docstring"""
         self._check_padding(numpify=UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						List[Any]:

         """simple docstring"""
         self._check_padding(numpify=UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Dict   )			->						Optional[int]:

         """simple docstring"""
         self._check_truncation(numpify=UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						Union[str, Any]:

         """simple docstring"""
         self._check_truncation(numpify=UpperCamelCase   )






   @require_torch
   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						str:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			str   =	self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__      :			str   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			Optional[int]   =	BatchFeature({input_name: speech_inputs}   )

         lowerCAmelCase__      :			Dict   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     return_tensors="""np"""   )[input_name]
         lowerCAmelCase__      :			str   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     return_tensors="""pt"""   )[input_name]

         self.assertTrue(abs(input_np.astype(np.floataa   ).sum() - input_pt.numpy().astype(np.floataa   ).sum()   ) < 1E-2   )






   @require_tf
   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						str:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	self.feature_extraction_class(**self.feat_extract_dict   )
         lowerCAmelCase__      :			Tuple   =	self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__      :			List[Any]   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			int   =	BatchFeature({input_name: speech_inputs}   )

         lowerCAmelCase__      :			int   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     return_tensors="""np"""   )[input_name]
         lowerCAmelCase__      :			Any   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     return_tensors="""tf"""   )[input_name]

         self.assertTrue(abs(input_np.astype(np.floataa   ).sum() - input_tf.numpy().astype(np.floataa   ).sum()   ) < 1E-2   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   )			->						int:

         """simple docstring"""
         lowerCAmelCase__      :			Any   =	self.feat_extract_dict
         lowerCAmelCase__      :			int   =	True
         lowerCAmelCase__      :			str   =	self.feature_extraction_class(**UpperCamelCase   )
         lowerCAmelCase__      :			int   =	self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__      :			List[Any]   =	[len(UpperCamelCase   ) for x in speech_inputs]
         lowerCAmelCase__      :			Optional[Any]   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			List[str]   =	BatchFeature({input_name: speech_inputs}   )

         lowerCAmelCase__      :			Tuple   =	feat_extract.pad(UpperCamelCase   ,     padding="""longest"""   ,     return_tensors="""np"""   )
         self.assertIn("""attention_mask"""   ,     UpperCamelCase   )
         self.assertListEqual(list(processed.attention_mask.shape   )   ,     list(processed[input_name].shape[:2]   )   )
         self.assertListEqual(processed.attention_mask.sum(-1   ).tolist()   ,     UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		Optional[int]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	self.feat_extract_dict
         lowerCAmelCase__      :			List[Any]   =	True
         lowerCAmelCase__      :			str   =	self.feature_extraction_class(**UpperCamelCase   )
         lowerCAmelCase__      :			int   =	self.feat_extract_tester.prepare_inputs_for_common()
         lowerCAmelCase__      :			Any   =	[len(UpperCamelCase   ) for x in speech_inputs]
         lowerCAmelCase__      :			int   =	feat_extract.model_input_names[0]

         lowerCAmelCase__      :			Optional[Any]   =	BatchFeature({input_name: speech_inputs}   )
         lowerCAmelCase__      :			Any   =	min(UpperCamelCase   )

         lowerCAmelCase__      :			Tuple   =	feat_extract.pad(
             UpperCamelCase   ,     padding="""max_length"""   ,     max_length=UpperCamelCase   ,     truncation=UpperCamelCase   ,     return_tensors="""np"""   )
         self.assertIn("""attention_mask"""   ,     UpperCamelCase   )
         self.assertListEqual(
             list(processed_pad.attention_mask.shape   )   ,     [processed_pad[input_name].shape[0], max_length]   )
         self.assertListEqual(
             processed_pad.attention_mask[:, :max_length].sum(-1   ).tolist()   ,     [max_length for x in speech_inputs]   )











"""simple docstring"""


import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

_A													=      logging.get_logger(__name__)


_A													=      {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

_A													=      {
    """vocab_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"""},
    """merges_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"""},
    """tokenizer_config_file""": {
        """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"""
    },
}

_A													=      {"""facebook/blenderbot-3B""": 1_2_8}



@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	lowercase_				(			)						->    Tuple:
      lowerCAmelCase__      :			int   =	(
          list(range(ord("""!"""				)    ,		ord("""~"""				) + 1				)				) + list(range(ord("""¡"""				)    ,		ord("""¬"""				) + 1				)				) + list(range(ord("""®"""				)    ,		ord("""ÿ"""				) + 1				)				)
      )
      lowerCAmelCase__      :			Any   =	bs[:]
      lowerCAmelCase__      :			Optional[int]   =	0
      for b in range(2**8				):
            if b not in bs:
                  bs.append(__UpperCAmelCase				)
                  cs.append(2**8 + n				)
                  n += 1
      lowerCAmelCase__      :			Dict   =	[chr(__UpperCAmelCase				) for n in cs]
      return dict(zip(__UpperCAmelCase    ,		__UpperCAmelCase				)				)



def 	lowercase_				(			__UpperCAmelCase				)						->    List[Any]:
      lowerCAmelCase__      :			List[Any]   =	set()
      lowerCAmelCase__      :			Union[str, Any]   =	word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)				)
            lowerCAmelCase__      :			Optional[Any]   =	char
      return pairs


class _lowerCamelCase    (					a_						):
   _lowerCamelCase    :Optional[Any]     				=     VOCAB_FILES_NAMES
   _lowerCamelCase    :List[Any]     				=     PRETRAINED_VOCAB_FILES_MAP
   _lowerCamelCase    :Union[str, Any]     				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _lowerCamelCase    :Optional[Any]     				=     ["input_ids", "attention_mask"]






   def __init__(		self       :		Any   ,     UpperCamelCase       :		str   ,     UpperCamelCase       :		Tuple   ,     UpperCamelCase       :		Any="replace"   ,     UpperCamelCase       :		Optional[Any]="<s>"   ,     UpperCamelCase       :		Union[str, Any]="</s>"   ,     UpperCamelCase       :		Optional[int]="</s>"   ,     UpperCamelCase       :		str="<s>"   ,     UpperCamelCase       :		int="<unk>"   ,     UpperCamelCase       :		int="<pad>"   ,     UpperCamelCase       :		Dict="<mask>"   ,     UpperCamelCase       :		Optional[int]=False   ,     **UpperCamelCase       :		Optional[Any]   ,     )			->						Any:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else bos_token
         lowerCAmelCase__      :			int   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else eos_token
         lowerCAmelCase__      :			Dict   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else sep_token
         lowerCAmelCase__      :			Union[str, Any]   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else cls_token
         lowerCAmelCase__      :			int   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else unk_token
         lowerCAmelCase__      :			Union[str, Any]   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         lowerCAmelCase__      :			Union[str, Any]   =	AddedToken(UpperCamelCase   ,     lstrip=UpperCamelCase   ,     rstrip=UpperCamelCase   ) if isinstance(UpperCamelCase   ,     UpperCamelCase   ) else mask_token

         super().__init__(
             errors=UpperCamelCase   ,     bos_token=UpperCamelCase   ,     eos_token=UpperCamelCase   ,     unk_token=UpperCamelCase   ,     sep_token=UpperCamelCase   ,     cls_token=UpperCamelCase   ,     pad_token=UpperCamelCase   ,     mask_token=UpperCamelCase   ,     add_prefix_space=UpperCamelCase   ,     **UpperCamelCase   ,     )

         with open(UpperCamelCase   ,     encoding="""utf-8"""   ) as vocab_handle:
               lowerCAmelCase__      :			Any   =	json.load(UpperCamelCase   )
         lowerCAmelCase__      :			Union[str, Any]   =	{v: k for k, v in self.encoder.items()}
         lowerCAmelCase__      :			Dict   =	errors  # how to handle errors in decoding
         lowerCAmelCase__      :			Union[str, Any]   =	bytes_to_unicode()
         lowerCAmelCase__      :			List[str]   =	{v: k for k, v in self.byte_encoder.items()}
         with open(UpperCamelCase   ,     encoding="""utf-8"""   ) as merges_handle:
               lowerCAmelCase__      :			Optional[int]   =	merges_handle.read().split("""\n"""   )[1:-1]
         lowerCAmelCase__      :			Dict   =	[tuple(merge.split()   ) for merge in bpe_merges]
         lowerCAmelCase__      :			Any   =	dict(zip(UpperCamelCase   ,     range(len(UpperCamelCase   )   )   )   )
         lowerCAmelCase__      :			Union[str, Any]   =	{}
         lowerCAmelCase__      :			Dict   =	add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         lowerCAmelCase__      :			Tuple   =	re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""   )






   @property
   # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
   def 	_lowerCAmelCase   (		self       :		List[str]   )			->						Union[str, Any]:

         """simple docstring"""
         return len(self.encoder   )






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   )			->						Optional[Any]:

         """simple docstring"""
         return dict(self.encoder   ,     **self.added_tokens_encoder   )






   def 	_lowerCAmelCase   (		self       :		List[str]   ,     UpperCamelCase       :		str   )			->						Union[str, Any]:

         """simple docstring"""
         if token in self.cache:
               return self.cache[token]
         lowerCAmelCase__      :			Union[str, Any]   =	tuple(UpperCamelCase   )
         lowerCAmelCase__      :			List[str]   =	get_pairs(UpperCamelCase   )

         if not pairs:
               return token

         while True:
               lowerCAmelCase__      :			List[str]   =	min(UpperCamelCase   ,     key=lambda UpperCamelCase   : self.bpe_ranks.get(UpperCamelCase   ,     float("""inf"""   )   )   )
               if bigram not in self.bpe_ranks:
                     break
               lowerCAmelCase__ ,   lowerCAmelCase__      :			str   =	bigram
               lowerCAmelCase__      :			List[str]   =	[]
               lowerCAmelCase__      :			List[str]   =	0
               while i < len(UpperCamelCase   ):
                     try:
                           lowerCAmelCase__      :			Optional[Any]   =	word.index(UpperCamelCase   ,     UpperCamelCase   )
                     except ValueError:
                           new_word.extend(word[i:]   )
                           break
                     else:
                           new_word.extend(word[i:j]   )
                           lowerCAmelCase__      :			List[str]   =	j

                     if word[i] == first and i < len(UpperCamelCase   ) - 1 and word[i + 1] == second:
                           new_word.append(first + second   )
                           i += 2
                     else:
                           new_word.append(word[i]   )
                           i += 1
               lowerCAmelCase__      :			List[Any]   =	tuple(UpperCamelCase   )
               lowerCAmelCase__      :			Tuple   =	new_word
               if len(UpperCamelCase   ) == 1:
                     break
               else:
                     lowerCAmelCase__      :			Any   =	get_pairs(UpperCamelCase   )
         lowerCAmelCase__      :			Tuple   =	""" """.join(UpperCamelCase   )
         lowerCAmelCase__      :			Tuple   =	word
         return word






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   ,     UpperCamelCase       :		List[str]   )			->						Optional[Any]:

         """simple docstring"""
         lowerCAmelCase__      :			Tuple   =	[]
         for token in re.findall(self.pat   ,     UpperCamelCase   ):
               lowerCAmelCase__      :			List[Any]   =	"""""".join(
                   self.byte_encoder[b] for b in token.encode("""utf-8"""   )   )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase   ).split(""" """   )   )
         return bpe_tokens






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   ,     UpperCamelCase       :		Union[str, Any]   )			->						Dict:

         """simple docstring"""
         return self.encoder.get(UpperCamelCase   ,     self.encoder.get(self.unk_token   )   )






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   ,     UpperCamelCase       :		Optional[Any]   )			->						Tuple:

         """simple docstring"""
         return self.decoder.get(UpperCamelCase   )






   def 	_lowerCAmelCase   (		self       :		List[Any]   ,     UpperCamelCase       :		Optional[int]   )			->						Optional[int]:

         """simple docstring"""
         lowerCAmelCase__      :			List[str]   =	"""""".join(UpperCamelCase   )
         lowerCAmelCase__      :			List[str]   =	bytearray([self.byte_decoder[c] for c in text]   ).decode("""utf-8"""   ,     errors=self.errors   )
         return text






   def 	_lowerCAmelCase   (		self       :		List[str]   ,     UpperCamelCase       :		str   ,     UpperCamelCase       :		Optional[str] = None   )			->						Tuple[str]:

         """simple docstring"""
         if not os.path.isdir(UpperCamelCase   ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""   )
               return
         lowerCAmelCase__      :			Union[str, Any]   =	os.path.join(
             UpperCamelCase   ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
         lowerCAmelCase__      :			int   =	os.path.join(
             UpperCamelCase   ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]   )

         with open(UpperCamelCase   ,     """w"""   ,     encoding="""utf-8"""   ) as f:
               f.write(json.dumps(self.encoder   ,     indent=2   ,     sort_keys=UpperCamelCase   ,     ensure_ascii=UpperCamelCase   ) + """\n"""   )

         lowerCAmelCase__      :			Optional[Any]   =	0
         with open(UpperCamelCase   ,     """w"""   ,     encoding="""utf-8"""   ) as writer:
               writer.write("""#version: 0.2\n"""   )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,     key=lambda UpperCamelCase   : kv[1]   ):
                     if index != token_index:
                           logger.warning(
                               f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                               """ Please check that the tokenizer is not corrupted!"""   )
                           lowerCAmelCase__      :			Dict   =	token_index
                     writer.write(""" """.join(UpperCamelCase   ) + """\n"""   )
                     index += 1

         return vocab_file, merge_file






   def 	_lowerCAmelCase   (		self       :		Dict   ,     UpperCamelCase       :		List[int]   ,     UpperCamelCase       :		Optional[List[int]] = None   ,     UpperCamelCase       :		bool = False   )			->						List[int]:

         """simple docstring"""
         if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=UpperCamelCase   ,     token_ids_a=UpperCamelCase   ,     already_has_special_tokens=UpperCamelCase   )

         if token_ids_a is None:
               return [1] + ([0] * len(UpperCamelCase   )) + [1]
         return [1] + ([0] * len(UpperCamelCase   )) + [1, 1] + ([0] * len(UpperCamelCase   )) + [1]






   def 	_lowerCAmelCase   (		self       :		Optional[int]   ,     UpperCamelCase       :		List[int]   ,     UpperCamelCase       :		Optional[List[int]] = None   )			->						List[int]:

         """simple docstring"""
         lowerCAmelCase__      :			Union[str, Any]   =	[self.sep_token_id]
         lowerCAmelCase__      :			List[str]   =	[self.cls_token_id]

         if token_ids_a is None:
               return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






   def 	_lowerCAmelCase   (		self       :		Union[str, Any]   ,     UpperCamelCase       :		Any   ,     UpperCamelCase       :		Optional[int]=False   ,     **UpperCamelCase       :		Union[str, Any]   )			->						str:

         """simple docstring"""
         lowerCAmelCase__      :			int   =	kwargs.pop("""add_prefix_space"""   ,     self.add_prefix_space   )
         if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase   ) > 0 and not text[0].isspace()):
               lowerCAmelCase__      :			Tuple   =	""" """ + text
         return (text, kwargs)






   def 	_lowerCAmelCase   (		self       :		Optional[Any]   ,     UpperCamelCase       :		List[int]   ,     UpperCamelCase       :		Optional[List[int]] = None   )			->						Any:

         """simple docstring"""
         return token_ids_a + [self.eos_token_id]






   def 	_lowerCAmelCase   (		self       :		str   ,     UpperCamelCase       :		"Conversation"   )			->						List[int]:

         """simple docstring"""
         lowerCAmelCase__      :			List[str]   =	[]
         for is_user, text in conversation.iter_texts():
               if is_user:
                     # We need to space prefix as it's being done within blenderbot
                     inputs.append(""" """ + text   )
               else:
                     # Generated responses should contain them already.
                     inputs.append(UpperCamelCase   )

         lowerCAmelCase__      :			Union[str, Any]   =	"""  """.join(UpperCamelCase   )
         lowerCAmelCase__      :			Union[str, Any]   =	self.encode(UpperCamelCase   )
         if len(UpperCamelCase   ) > self.model_max_length:
               lowerCAmelCase__      :			List[str]   =	input_ids[-self.model_max_length :]
               logger.warning(f"""Trimmed input from conversation as it was longer than {self.model_max_length} tokens."""   )
         return input_ids





