
"""simple docstring"""

import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_A						=			logging.get_logger(__name__)


_A						=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all BART models at https://huggingface.co/models?filter=bart
_A						=			{
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
}

_A						=			{
    'facebook/bart-base': 1_0_2_4,
    'facebook/bart-large': 1_0_2_4,
    'facebook/bart-large-mnli': 1_0_2_4,
    'facebook/bart-large-cnn': 1_0_2_4,
    'facebook/bart-large-xsum': 1_0_2_4,
    'yjernite/bart_eli5': 1_0_2_4,
}
@lru_cache()
def 		UpperCAmelCase (		):





		'''simple docstring'''


		lowerCamelCase					:      int          =   (
		    list(range(ord('!'      ),						ord('~'      ) + 1      )      ) + list(range(ord('¡'      ),						ord('¬'      ) + 1      )      ) + list(range(ord('®'      ),						ord('ÿ'      ) + 1      )      )
		)
		lowerCamelCase					:      Any          =   bs[:]
		lowerCamelCase					:      Tuple          =   0
		for b in range(2**8      ):
				if b not in bs:
						bs.append(a_      )
						cs.append(2**8 + n      )
						n += 1
		lowerCamelCase					:      Union[str, Any]          =   [chr(a_      ) for n in cs]
		return dict(zip(a_,						a_      )      )
def 		UpperCAmelCase (		a_      ):





		'''simple docstring'''


		lowerCamelCase					:      Optional[int]          =   set()
		lowerCamelCase					:      List[Any]          =   word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)      )
				lowerCamelCase					:      int          =   char
		return pairs






class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     VOCAB_FILES_NAMES
	lowercase_				=     PRETRAINED_VOCAB_FILES_MAP
	lowercase_				=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowercase_				=     ['input_ids', 'attention_mask']




	def __init__(			self		,			UpperCAmelCase_		,			UpperCAmelCase_		,			UpperCAmelCase_="replace"		,			UpperCAmelCase_="<s>"		,			UpperCAmelCase_="</s>"		,			UpperCAmelCase_="</s>"		,			UpperCAmelCase_="<s>"		,			UpperCAmelCase_="<unk>"		,			UpperCAmelCase_="<pad>"		,			UpperCAmelCase_="<mask>"		,			UpperCAmelCase_=False		,			**UpperCAmelCase_		,			)		->							Optional[Any]:
			lowerCamelCase					:      int          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else bos_token
			lowerCamelCase					:      Union[str, Any]          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else eos_token
			lowerCamelCase					:      Union[str, Any]          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else sep_token
			lowerCamelCase					:      str          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else cls_token
			lowerCamelCase					:      Tuple          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else unk_token
			lowerCamelCase					:      Any          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else pad_token

			# Mask token behave like a normal word, i.e. include the space before it
			lowerCamelCase					:      Dict          =   AddedToken(UpperCAmelCase_		,			lstrip=UpperCAmelCase_		,			rstrip=UpperCAmelCase_ ) if isinstance(UpperCAmelCase_		,			UpperCAmelCase_ ) else mask_token

			super().__init__(
			    errors=UpperCAmelCase_		,			bos_token=UpperCAmelCase_		,			eos_token=UpperCAmelCase_		,			unk_token=UpperCAmelCase_		,			sep_token=UpperCAmelCase_		,			cls_token=UpperCAmelCase_		,			pad_token=UpperCAmelCase_		,			mask_token=UpperCAmelCase_		,			add_prefix_space=UpperCAmelCase_		,			**UpperCAmelCase_		,			)

			with open(UpperCAmelCase_		,			encoding='utf-8' ) as vocab_handle:
					lowerCamelCase					:      Tuple          =   json.load(UpperCAmelCase_ )
			lowerCamelCase					:      Tuple          =   {v: k for k, v in self.encoder.items()}
			lowerCamelCase					:      Dict          =   errors  # how to handle errors in decoding
			lowerCamelCase					:      Tuple          =   bytes_to_unicode()
			lowerCamelCase					:      List[str]          =   {v: k for k, v in self.byte_encoder.items()}
			with open(UpperCAmelCase_		,			encoding='utf-8' ) as merges_handle:
					lowerCamelCase					:      List[str]          =   merges_handle.read().split('\n' )[1:-1]
			lowerCamelCase					:      Any          =   [tuple(merge.split() ) for merge in bpe_merges]
			lowerCamelCase					:      int          =   dict(zip(UpperCAmelCase_		,			range(len(UpperCAmelCase_ ) ) ) )
			lowerCamelCase					:      List[Any]          =   {}
			lowerCamelCase					:      Dict          =   add_prefix_space

			# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
			lowerCamelCase					:      Union[str, Any]          =   re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+' )




	@property
	def       _UpperCamelCase   (			self )		->							str:
			return len(self.encoder )




	def       _UpperCamelCase   (			self )		->							List[Any]:
			return dict(self.encoder		,			**self.added_tokens_encoder )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							int:
			if token in self.cache:
					return self.cache[token]
			lowerCamelCase					:      Optional[Any]          =   tuple(UpperCAmelCase_ )
			lowerCamelCase					:      Tuple          =   get_pairs(UpperCAmelCase_ )

			if not pairs:
					return token

			while True:
					lowerCamelCase					:      str          =   min(UpperCAmelCase_		,			key=lambda UpperCAmelCase_ : self.bpe_ranks.get(UpperCAmelCase_		,			float('inf' ) ) )
					if bigram not in self.bpe_ranks:
							break
					lowerCamelCase     ,   lowerCamelCase					:      str          =   bigram
					lowerCamelCase					:      List[Any]          =   []
					lowerCamelCase					:      int          =   0
					while i < len(UpperCAmelCase_ ):
							try:
									lowerCamelCase					:      str          =   word.index(UpperCAmelCase_		,			UpperCAmelCase_ )
							except ValueError:
									new_word.extend(word[i:] )
									break
							else:
									new_word.extend(word[i:j] )
									lowerCamelCase					:      Optional[int]          =   j

							if word[i] == first and i < len(UpperCAmelCase_ ) - 1 and word[i + 1] == second:
									new_word.append(first + second )
									i += 2
							else:
									new_word.append(word[i] )
									i += 1
					lowerCamelCase					:      Dict          =   tuple(UpperCAmelCase_ )
					lowerCamelCase					:      int          =   new_word
					if len(UpperCAmelCase_ ) == 1:
							break
					else:
							lowerCamelCase					:      Tuple          =   get_pairs(UpperCAmelCase_ )
			lowerCamelCase					:      Dict          =   ' '.join(UpperCAmelCase_ )
			lowerCamelCase					:      Optional[Any]          =   word
			return word




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							Optional[int]:
			lowerCamelCase					:      List[Any]          =   []
			for token in re.findall(self.pat		,			UpperCAmelCase_ ):
					lowerCamelCase					:      List[Any]          =   ''.join(
					    self.byte_encoder[b] for b in token.encode('utf-8' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
					bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase_ ).split(' ' ) )
			return bpe_tokens




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							Optional[int]:
			return self.encoder.get(UpperCAmelCase_		,			self.encoder.get(self.unk_token ) )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							str:
			return self.decoder.get(UpperCAmelCase_ )




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_ )		->							Dict:
			lowerCamelCase					:      List[Any]          =   ''.join(UpperCAmelCase_ )
			lowerCamelCase					:      str          =   bytearray([self.byte_decoder[c] for c in text] ).decode('utf-8'		,			errors=self.errors )
			return text




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							Tuple[str]:
			if not os.path.isdir(UpperCAmelCase_ ):
					logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
					return
			lowerCamelCase					:      str          =   os.path.join(
			    UpperCAmelCase_		,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
			lowerCamelCase					:      Optional[Any]          =   os.path.join(
			    UpperCAmelCase_		,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

			with open(UpperCAmelCase_		,			'w'		,			encoding='utf-8' ) as f:
					f.write(json.dumps(self.encoder		,			indent=2		,			sort_keys=UpperCAmelCase_		,			ensure_ascii=UpperCAmelCase_ ) + '\n' )

			lowerCamelCase					:      Any          =   0
			with open(UpperCAmelCase_		,			'w'		,			encoding='utf-8' ) as writer:
					writer.write('#version: 0.2\n' )
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,			key=lambda UpperCAmelCase_ : kv[1] ):
							if index != token_index:
									logger.warning(
									    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
									    ' Please check that the tokenizer is not corrupted!' )
									lowerCamelCase					:      List[str]          =   token_index
							writer.write(' '.join(UpperCAmelCase_ ) + '\n' )
							index += 1

			return vocab_file, merge_file




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							List[int]:
			if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
			lowerCamelCase					:      Optional[Any]          =   [self.cls_token_id]
			lowerCamelCase					:      Tuple          =   [self.sep_token_id]
			return cls + token_ids_a + sep + sep + token_ids_a + sep




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None		,			UpperCAmelCase_ = False )		->							List[int]:
			if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=UpperCAmelCase_		,			token_ids_a=UpperCAmelCase_		,			already_has_special_tokens=UpperCAmelCase_ )

			if token_ids_a is None:
					return [1] + ([0] * len(UpperCAmelCase_ )) + [1]
			return [1] + ([0] * len(UpperCAmelCase_ )) + [1, 1] + ([0] * len(UpperCAmelCase_ )) + [1]




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_ = None )		->							List[int]:
			lowerCamelCase					:      Tuple          =   [self.sep_token_id]
			lowerCamelCase					:      Any          =   [self.cls_token_id]

			if token_ids_a is None:
					return len(cls + token_ids_a + sep ) * [0]
			return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]




	def       _UpperCamelCase   (			self		,			UpperCAmelCase_		,			UpperCAmelCase_=False		,			**UpperCAmelCase_ )		->							Dict:
			lowerCamelCase					:      Tuple          =   kwargs.pop('add_prefix_space'		,			self.add_prefix_space )
			if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase_ ) > 0 and not text[0].isspace()):
					lowerCamelCase					:      str          =   ' ' + text
			return (text, kwargs)








"""simple docstring"""

import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_A						=			logging.get_logger(__name__)

_A						=			{
    'asapp/sew-d-tiny-100k': 'https://huggingface.co/asapp/sew-d-tiny-100k/resolve/main/config.json',
    # See all SEW-D models at https://huggingface.co/models?filter=sew-d
}






class 	_lowercase       (    __UpperCAmelCase		):
	lowercase_				=     'sew-d'




	def __init__(			self		,			UpperCAmelCase_=32		,			UpperCAmelCase_=768		,			UpperCAmelCase_=12		,			UpperCAmelCase_=12		,			UpperCAmelCase_=3072		,			UpperCAmelCase_=2		,			UpperCAmelCase_=512		,			UpperCAmelCase_=256		,			UpperCAmelCase_=True		,			UpperCAmelCase_=True		,			UpperCAmelCase_=("p2c", "c2p")		,			UpperCAmelCase_="layer_norm"		,			UpperCAmelCase_="gelu_python"		,			UpperCAmelCase_=0.1		,			UpperCAmelCase_=0.1		,			UpperCAmelCase_=0.1		,			UpperCAmelCase_=0.0		,			UpperCAmelCase_=0.1		,			UpperCAmelCase_=0.02		,			UpperCAmelCase_=1E-7		,			UpperCAmelCase_=1E-5		,			UpperCAmelCase_="group"		,			UpperCAmelCase_="gelu"		,			UpperCAmelCase_=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)		,			UpperCAmelCase_=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)		,			UpperCAmelCase_=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)		,			UpperCAmelCase_=False		,			UpperCAmelCase_=128		,			UpperCAmelCase_=16		,			UpperCAmelCase_=True		,			UpperCAmelCase_=0.05		,			UpperCAmelCase_=10		,			UpperCAmelCase_=2		,			UpperCAmelCase_=0.0		,			UpperCAmelCase_=10		,			UpperCAmelCase_=0		,			UpperCAmelCase_="mean"		,			UpperCAmelCase_=False		,			UpperCAmelCase_=False		,			UpperCAmelCase_=256		,			UpperCAmelCase_=0		,			UpperCAmelCase_=1		,			UpperCAmelCase_=2		,			**UpperCAmelCase_		,			)		->							Optional[Any]:
			super().__init__(**UpperCAmelCase_		,			pad_token_id=UpperCAmelCase_		,			bos_token_id=UpperCAmelCase_		,			eos_token_id=UpperCAmelCase_ )
			lowerCamelCase					:      Any          =   hidden_size
			lowerCamelCase					:      Any          =   feat_extract_norm
			lowerCamelCase					:      List[str]          =   feat_extract_activation
			lowerCamelCase					:      str          =   list(UpperCAmelCase_ )
			lowerCamelCase					:      Any          =   list(UpperCAmelCase_ )
			lowerCamelCase					:      str          =   list(UpperCAmelCase_ )
			lowerCamelCase					:      List[Any]          =   conv_bias
			lowerCamelCase					:      Optional[int]          =   num_conv_pos_embeddings
			lowerCamelCase					:      str          =   num_conv_pos_embedding_groups
			lowerCamelCase					:      Optional[int]          =   len(self.conv_dim )
			lowerCamelCase					:      Optional[int]          =   num_hidden_layers
			lowerCamelCase					:      Union[str, Any]          =   intermediate_size
			lowerCamelCase					:      str          =   squeeze_factor
			lowerCamelCase					:      Any          =   max_position_embeddings
			lowerCamelCase					:      List[Any]          =   position_buckets
			lowerCamelCase					:      Union[str, Any]          =   share_att_key
			lowerCamelCase					:      Optional[int]          =   relative_attention
			lowerCamelCase					:      Tuple          =   norm_rel_ebd
			lowerCamelCase					:      Union[str, Any]          =   list(UpperCAmelCase_ )
			lowerCamelCase					:      List[Any]          =   hidden_act
			lowerCamelCase					:      Optional[Any]          =   num_attention_heads
			lowerCamelCase					:      Tuple          =   hidden_dropout
			lowerCamelCase					:      List[Any]          =   attention_dropout
			lowerCamelCase					:      Optional[Any]          =   activation_dropout
			lowerCamelCase					:      List[str]          =   feat_proj_dropout
			lowerCamelCase					:      List[str]          =   final_dropout
			lowerCamelCase					:      str          =   layer_norm_eps
			lowerCamelCase					:      int          =   feature_layer_norm_eps
			lowerCamelCase					:      Optional[Any]          =   initializer_range
			lowerCamelCase					:      int          =   vocab_size

			if (
			    (len(self.conv_stride ) != self.num_feat_extract_layers)
			    or (len(self.conv_kernel ) != self.num_feat_extract_layers)
			    or (len(self.conv_dim ) != self.num_feat_extract_layers)
			):
					raise ValueError(
					    'Configuration for convolutional layers is incorrect.'
					    'It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,'
					    F"""but is `len(config.conv_dim) = {len(self.conv_dim )}`, `len(config.conv_stride)"""
					    F"""= {len(self.conv_stride )}`, `len(config.conv_kernel) = {len(self.conv_kernel )}`.""" )

			# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
			lowerCamelCase					:      Any          =   apply_spec_augment
			lowerCamelCase					:      Optional[int]          =   mask_time_prob
			lowerCamelCase					:      Optional[Any]          =   mask_time_length
			lowerCamelCase					:      str          =   mask_time_min_masks
			lowerCamelCase					:      List[Any]          =   mask_feature_prob
			lowerCamelCase					:      int          =   mask_feature_length
			lowerCamelCase					:      List[Any]          =   mask_feature_min_masks

			# ctc loss
			lowerCamelCase					:      Optional[Any]          =   ctc_loss_reduction
			lowerCamelCase					:      Union[str, Any]          =   ctc_zero_infinity

			# sequence classification
			lowerCamelCase					:      Optional[Any]          =   use_weighted_layer_sum
			lowerCamelCase					:      Dict          =   classifier_proj_size




	@property
	def       _UpperCamelCase   (			self )		->							Optional[Any]:
			return functools.reduce(operator.mul		,			self.conv_stride		,			1 )






