'''simple docstring'''




import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase												=logging.get_logger(__name__)

__UpperCAmelCase												={
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

__UpperCAmelCase												={
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

__UpperCAmelCase												={
    "vinai/phobert-base": 2_5_6,
    "vinai/phobert-large": 2_5_6,
}






def __lowerCAmelCase			(							UpperCamelCase__    )     ->   Tuple:
    __lowerCamelCase          =  set()
    __lowerCamelCase          =  word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)    )
        __lowerCamelCase          =  char

    __lowerCamelCase          =  set(UpperCamelCase__    )
    return pairs




class    a__				(       UpperCAmelCase__			):
     lowerCamelCase  :					Tuple							=VOCAB_FILES_NAMES
     lowerCamelCase  :					int							=PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase  :					Dict							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






     def __init__(			self		:		Optional[int]  ,    a		:		str  ,    a		:		Optional[int]  ,    a		:		Optional[Any]="<s>"  ,    a		:		Optional[int]="</s>"  ,    a		:		int="</s>"  ,    a		:		Tuple="<s>"  ,    a		:		Dict="<unk>"  ,    a		:		Optional[Any]="<pad>"  ,    a		:		Dict="<mask>"  ,    **a		:		Tuple  ,    ):





         """simple docstring"""



         super().__init__(
             bos_token=a  ,    eos_token=a  ,    unk_token=a  ,    sep_token=a  ,    cls_token=a  ,    pad_token=a  ,    mask_token=a  ,    **a  ,    )

         __lowerCamelCase          =  vocab_file
         __lowerCamelCase          =  merges_file

         __lowerCamelCase          =  {}
         __lowerCamelCase          =  0
         __lowerCamelCase          =  1
         __lowerCamelCase          =  2
         __lowerCamelCase          =  3

         self.add_from_file(a   )

         __lowerCamelCase          =  {v: k for k, v in self.encoder.items()}

         with open(a  ,    encoding='''utf-8'''   ) as merges_handle:
             __lowerCamelCase          =  merges_handle.read().split('''\n'''   )[:-1]
         __lowerCamelCase          =  [tuple(merge.split()[:-1]   ) for merge in merges]
         __lowerCamelCase          =  dict(zip(a  ,    range(len(a   )   )   )   )
         __lowerCamelCase          =  {}






     def        SCREAMING_SNAKE_CASE__       (			self		:		str  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None   ):





         """simple docstring"""



         if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         __lowerCamelCase          =  [self.cls_token_id]
         __lowerCamelCase          =  [self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None  ,    a		:		bool = False   ):





         """simple docstring"""



         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=a  ,    token_ids_a=a  ,    already_has_special_tokens=a   )

         if token_ids_a is None:
             return [1] + ([0] * len(a   )) + [1]
         return [1] + ([0] * len(a   )) + [1, 1] + ([0] * len(a   )) + [1]






     def        SCREAMING_SNAKE_CASE__       (			self		:		str  ,    a		:		List[int]  ,    a		:		Optional[List[int]] = None   ):





         """simple docstring"""



         __lowerCamelCase          =  [self.sep_token_id]
         __lowerCamelCase          =  [self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep   ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]






     @property
     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         return len(self.encoder   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]   ):





         """simple docstring"""



         return dict(self.encoder  ,    **self.added_tokens_encoder   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]  ,    a		:		Dict   ):





         """simple docstring"""



         if token in self.cache:
             return self.cache[token]
         __lowerCamelCase          =  tuple(a   )
         __lowerCamelCase          =  tuple(list(word[:-1]   ) + [word[-1] + '''</w>''']   )
         __lowerCamelCase          =  get_pairs(a   )

         if not pairs:
             return token

         while True:
             __lowerCamelCase          =  min(a  ,    key=lambda a   : self.bpe_ranks.get(a  ,    float('''inf'''   )   )   )
             if bigram not in self.bpe_ranks:
                 break
             __lowerCamelCase     ,  __lowerCamelCase          =  bigram
             __lowerCamelCase          =  []
             __lowerCamelCase          =  0
             while i < len(a   ):
                 try:
                     __lowerCamelCase          =  word.index(a  ,    a   )
                 except ValueError:
                     new_word.extend(word[i:]   )
                     break
                 else:
                     new_word.extend(word[i:j]   )
                     __lowerCamelCase          =  j

                 if word[i] == first and i < len(a   ) - 1 and word[i + 1] == second:
                     new_word.append(first + second   )
                     i += 2
                 else:
                     new_word.append(word[i]   )
                     i += 1
             __lowerCamelCase          =  tuple(a   )
             __lowerCamelCase          =  new_word
             if len(a   ) == 1:
                 break
             else:
                 __lowerCamelCase          =  get_pairs(a   )
         __lowerCamelCase          =  '''@@ '''.join(a   )
         __lowerCamelCase          =  word[:-4]
         __lowerCamelCase          =  word
         return word






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple  ,    a		:		Tuple   ):





         """simple docstring"""



         __lowerCamelCase          =  []

         __lowerCamelCase          =  re.findall(R'''\S+\n?'''  ,    a   )

         for token in words:
             split_tokens.extend(list(self.bpe(a   ).split(''' '''   )   )   )
         return split_tokens






     def        SCREAMING_SNAKE_CASE__       (			self		:		Union[str, Any]  ,    a		:		Tuple   ):





         """simple docstring"""



         return self.encoder.get(a  ,    self.encoder.get(self.unk_token   )   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[str]  ,    a		:		Tuple   ):





         """simple docstring"""



         return self.decoder.get(a  ,    self.unk_token   )






     def        SCREAMING_SNAKE_CASE__       (			self		:		int  ,    a		:		Optional[Any]   ):





         """simple docstring"""



         __lowerCamelCase          =  ''' '''.join(a   ).replace('''@@ '''  ,    ''''''   ).strip()
         return out_string






     def        SCREAMING_SNAKE_CASE__       (			self		:		Tuple  ,    a		:		str  ,    a		:		Optional[str] = None   ):





         """simple docstring"""



         if not os.path.isdir(a   ):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""   )
             return
         __lowerCamelCase          =  os.path.join(
             a  ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
         __lowerCamelCase          =  os.path.join(
             a  ,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']   )

         if os.path.abspath(self.vocab_file   ) != os.path.abspath(a   ):
             copyfile(self.vocab_file  ,    a   )

         if os.path.abspath(self.merges_file   ) != os.path.abspath(a   ):
             copyfile(self.merges_file  ,    a   )

         return out_vocab_file, out_merge_file






     def        SCREAMING_SNAKE_CASE__       (			self		:		List[Any]  ,    a		:		Optional[Any]   ):





         """simple docstring"""



         if isinstance(a  ,    a   ):
             try:
                 with open(a  ,    '''r'''  ,    encoding='''utf-8'''   ) as fd:
                     self.add_from_file(a   )
             except FileNotFoundError as fnfe:
                 raise fnfe
             except UnicodeError:
                 raise Exception(f"""Incorrect encoding detected in {f}, please rebuild the dataset"""   )
             return

         __lowerCamelCase          =  f.readlines()
         for lineTmp in lines:
             __lowerCamelCase          =  lineTmp.strip()
             __lowerCamelCase          =  line.rfind(''' '''   )
             if idx == -1:
                 raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''   )
             __lowerCamelCase          =  line[:idx]
             __lowerCamelCase          =  len(self.encoder   )

'''simple docstring'''




import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase												=logging.get_logger(__name__)

__UpperCAmelCase												={
    "microsoft/unispeech-sat-base-100h-libri-ft": (
        "https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}




class    a__				(       UpperCAmelCase__			):
     lowerCamelCase  :					List[Any]							="unispeech-sat"






     def __init__(			self		:		Dict  ,    a		:		str=32  ,    a		:		Any=7_68  ,    a		:		Optional[Any]=12  ,    a		:		Optional[int]=12  ,    a		:		int=30_72  ,    a		:		int="gelu"  ,    a		:		Dict=0.1  ,    a		:		Dict=0.1  ,    a		:		List[Any]=0.1  ,    a		:		Tuple=0.0  ,    a		:		Optional[Any]=0.0  ,    a		:		Tuple=0.1  ,    a		:		List[Any]=0.1  ,    a		:		str=0.02  ,    a		:		List[Any]=1e-5  ,    a		:		int="group"  ,    a		:		Union[str, Any]="gelu"  ,    a		:		Optional[int]=(5_12, 5_12, 5_12, 5_12, 5_12, 5_12, 5_12)  ,    a		:		List[Any]=(5, 2, 2, 2, 2, 2, 2)  ,    a		:		int=(10, 3, 3, 3, 3, 2, 2)  ,    a		:		Optional[Any]=False  ,    a		:		Any=1_28  ,    a		:		Tuple=16  ,    a		:		str=False  ,    a		:		Optional[Any]=True  ,    a		:		Dict=0.05  ,    a		:		List[Any]=10  ,    a		:		Any=2  ,    a		:		Optional[Any]=0.0  ,    a		:		Optional[Any]=10  ,    a		:		Any=0  ,    a		:		Any=3_20  ,    a		:		str=2  ,    a		:		List[str]=0.1  ,    a		:		List[str]=1_00  ,    a		:		List[str]=2_56  ,    a		:		str=2_56  ,    a		:		Dict=0.1  ,    a		:		Optional[Any]="mean"  ,    a		:		str=False  ,    a		:		Tuple=False  ,    a		:		Optional[Any]=2_56  ,    a		:		int=(5_12, 5_12, 5_12, 5_12, 15_00)  ,    a		:		int=(5, 3, 3, 1, 1)  ,    a		:		Any=(1, 2, 3, 1, 1)  ,    a		:		Union[str, Any]=5_12  ,    a		:		Optional[int]=0  ,    a		:		Optional[int]=1  ,    a		:		Optional[int]=2  ,    a		:		int=5_04  ,    **a		:		Dict  ,    ):





         """simple docstring"""



         super().__init__(**a  ,    pad_token_id=a  ,    bos_token_id=a  ,    eos_token_id=a   )
         __lowerCamelCase          =  hidden_size
         __lowerCamelCase          =  feat_extract_norm
         __lowerCamelCase          =  feat_extract_activation
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  conv_bias
         __lowerCamelCase          =  num_conv_pos_embeddings
         __lowerCamelCase          =  num_conv_pos_embedding_groups
         __lowerCamelCase          =  len(self.conv_dim   )
         __lowerCamelCase          =  num_hidden_layers
         __lowerCamelCase          =  intermediate_size
         __lowerCamelCase          =  hidden_act
         __lowerCamelCase          =  num_attention_heads
         __lowerCamelCase          =  hidden_dropout
         __lowerCamelCase          =  attention_dropout
         __lowerCamelCase          =  activation_dropout
         __lowerCamelCase          =  feat_proj_dropout
         __lowerCamelCase          =  final_dropout
         __lowerCamelCase          =  layerdrop
         __lowerCamelCase          =  layer_norm_eps
         __lowerCamelCase          =  initializer_range
         __lowerCamelCase          =  vocab_size
         __lowerCamelCase          =  num_clusters
         __lowerCamelCase          =  do_stable_layer_norm
         __lowerCamelCase          =  use_weighted_layer_sum

         if (
             (len(self.conv_stride   ) != self.num_feat_extract_layers)
             or (len(self.conv_kernel   ) != self.num_feat_extract_layers)
             or (len(self.conv_dim   ) != self.num_feat_extract_layers)
         ):
             raise ValueError(
                 '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                 ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                 f""" {len(self.conv_dim   )}`, `len(config.conv_stride) = {len(self.conv_stride   )}`,"""
                 f""" `len(config.conv_kernel) = {len(self.conv_kernel   )}`."""   )

         # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
         __lowerCamelCase          =  apply_spec_augment
         __lowerCamelCase          =  mask_time_prob
         __lowerCamelCase          =  mask_time_length
         __lowerCamelCase          =  mask_time_min_masks
         __lowerCamelCase          =  mask_feature_prob
         __lowerCamelCase          =  mask_feature_length
         __lowerCamelCase          =  mask_feature_min_masks

         # parameters for pretraining with codevector quantized representations
         __lowerCamelCase          =  num_codevectors_per_group
         __lowerCamelCase          =  num_codevector_groups
         __lowerCamelCase          =  contrastive_logits_temperature
         __lowerCamelCase          =  feat_quantizer_dropout
         __lowerCamelCase          =  num_negatives
         __lowerCamelCase          =  codevector_dim
         __lowerCamelCase          =  proj_codevector_dim
         __lowerCamelCase          =  diversity_loss_weight

         # ctc loss
         __lowerCamelCase          =  ctc_loss_reduction
         __lowerCamelCase          =  ctc_zero_infinity

         # SequenceClassification-specific parameter. Feel free to ignore for other classes.
         __lowerCamelCase          =  classifier_proj_size

         # XVector-specific parameters. Feel free to ignore for other classes.
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  list(a   )
         __lowerCamelCase          =  xvector_output_dim






     @property
     def        SCREAMING_SNAKE_CASE__       (			self		:		Any   ):





         """simple docstring"""



         return functools.reduce(operator.mul  ,    self.conv_stride  ,    1   )
