



"""simple docstring"""






import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__      =    logging.get_logger(__name__)

lowerCAmelCase__      =    {
    '''vocab_file''': '''vocab.txt''',
    '''merges_file''': '''bpe.codes''',
}

lowerCAmelCase__      =    {
    '''vocab_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt''',
    },
    '''merges_file''': {
        '''vinai/phobert-base''': '''https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes''',
        '''vinai/phobert-large''': '''https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes''',
    },
}

lowerCAmelCase__      =    {
    '''vinai/phobert-base''': 256,
    '''vinai/phobert-large''': 256,
}






def 	snake_case_  (				A_ :		Dict						):


       '''simple docstring'''





       _lowerCamelCase						:    Optional[int]         =			set()
       _lowerCamelCase						:    List[Any]         =			word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)						)
              _lowerCamelCase						:    Tuple         =			char

       _lowerCamelCase						:    Optional[Any]         =			set(A_						)
       return pairs







class 	__snake_case	(     _lowercase):
  snake_case__    :       str        =						VOCAB_FILES_NAMES
  snake_case__    :       Tuple        =						PRETRAINED_VOCAB_FILES_MAP
  snake_case__    :       int        =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




  def __init__(			self					:						Union[str, Any]						, __lowerCAmelCase					:						List[Any]						, __lowerCAmelCase					:						Union[str, Any]						, __lowerCAmelCase					:						Tuple="<s>"						, __lowerCAmelCase					:						str="</s>"						, __lowerCAmelCase					:						Optional[int]="</s>"						, __lowerCAmelCase					:						List[str]="<s>"						, __lowerCAmelCase					:						str="<unk>"						, __lowerCAmelCase					:						List[str]="<pad>"						, __lowerCAmelCase					:						Any="<mask>"						, **__lowerCAmelCase					:						List[Any]						, ):






         """simple docstring"""






         super().__init__(
             bos_token=__lowerCAmelCase						, eos_token=__lowerCAmelCase						, unk_token=__lowerCAmelCase						, sep_token=__lowerCAmelCase						, cls_token=__lowerCAmelCase						, pad_token=__lowerCAmelCase						, mask_token=__lowerCAmelCase						, **__lowerCAmelCase						, )

         _lowerCamelCase						:    str         =			vocab_file
         _lowerCamelCase						:    List[str]         =			merges_file

         _lowerCamelCase						:    Tuple         =			{}
         _lowerCamelCase						:    int         =			0
         _lowerCamelCase						:    List[Any]         =			1
         _lowerCamelCase						:    str         =			2
         _lowerCamelCase						:    int         =			3

         self.add_from_file(__lowerCAmelCase		)

         _lowerCamelCase						:    Any         =			{v: k for k, v in self.encoder.items()}

         with open(__lowerCAmelCase						, encoding='''utf-8'''		) as merges_handle:
                _lowerCamelCase						:    str         =			merges_handle.read().split('''\n'''		)[:-1]
         _lowerCamelCase						:    str         =			[tuple(merge.split()[:-1]		) for merge in merges]
         _lowerCamelCase						:    Tuple         =			dict(zip(__lowerCAmelCase						, range(len(__lowerCAmelCase		)		)		)		)
         _lowerCamelCase						:    Optional[Any]         =			{}




  def 						SCREAMING_SNAKE_CASE    (			self					:						int						, __lowerCAmelCase					:						List[int]						, __lowerCAmelCase					:						Optional[List[int]] = None		):






         """simple docstring"""






         if token_ids_a is None:
                return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         _lowerCamelCase						:    Optional[Any]         =			[self.cls_token_id]
         _lowerCamelCase						:    Any         =			[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]						, __lowerCAmelCase					:						List[int]						, __lowerCAmelCase					:						Optional[List[int]] = None						, __lowerCAmelCase					:						bool = False		):






         """simple docstring"""






         if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=__lowerCAmelCase						, token_ids_a=__lowerCAmelCase						, already_has_special_tokens=__lowerCAmelCase		)

         if token_ids_a is None:
                return [1] + ([0] * len(__lowerCAmelCase		)) + [1]
         return [1] + ([0] * len(__lowerCAmelCase		)) + [1, 1] + ([0] * len(__lowerCAmelCase		)) + [1]




  def 						SCREAMING_SNAKE_CASE    (			self					:						str						, __lowerCAmelCase					:						List[int]						, __lowerCAmelCase					:						Optional[List[int]] = None		):






         """simple docstring"""






         _lowerCamelCase						:    int         =			[self.sep_token_id]
         _lowerCamelCase						:    Union[str, Any]         =			[self.cls_token_id]

         if token_ids_a is None:
                return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]




  @property
  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[Any]		):






         """simple docstring"""






         return len(self.encoder		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[Any]		):






         """simple docstring"""






         return dict(self.encoder						, **self.added_tokens_encoder		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Any						, __lowerCAmelCase					:						Dict		):






         """simple docstring"""






         if token in self.cache:
                return self.cache[token]
         _lowerCamelCase						:    Dict         =			tuple(__lowerCAmelCase		)
         _lowerCamelCase						:    Optional[int]         =			tuple(list(word[:-1]		) + [word[-1] + '''</w>''']		)
         _lowerCamelCase						:    Optional[Any]         =			get_pairs(__lowerCAmelCase		)

         if not pairs:
                return token

         while True:
                _lowerCamelCase						:    List[Any]         =			min(__lowerCAmelCase						, key=lambda __lowerCAmelCase		: self.bpe_ranks.get(__lowerCAmelCase						, float('''inf'''		)		)		)
                if bigram not in self.bpe_ranks:
                       break
                _lowerCamelCase      ,   _lowerCamelCase						:    Any         =			bigram
                _lowerCamelCase						:    Optional[Any]         =			[]
                _lowerCamelCase						:    str         =			0
                while i < len(__lowerCAmelCase		):
                       try:
                              _lowerCamelCase						:    Optional[Any]         =			word.index(__lowerCAmelCase						, __lowerCAmelCase		)
                       except ValueError:
                              new_word.extend(word[i:]		)
                              break
                       else:
                              new_word.extend(word[i:j]		)
                              _lowerCamelCase						:    List[Any]         =			j

                       if word[i] == first and i < len(__lowerCAmelCase		) - 1 and word[i + 1] == second:
                              new_word.append(first + second		)
                              i += 2
                       else:
                              new_word.append(word[i]		)
                              i += 1
                _lowerCamelCase						:    Any         =			tuple(__lowerCAmelCase		)
                _lowerCamelCase						:    List[Any]         =			new_word
                if len(__lowerCAmelCase		) == 1:
                       break
                else:
                       _lowerCamelCase						:    int         =			get_pairs(__lowerCAmelCase		)
         _lowerCamelCase						:    int         =			'''@@ '''.join(__lowerCAmelCase		)
         _lowerCamelCase						:    Optional[Any]         =			word[:-4]
         _lowerCamelCase						:    List[Any]         =			word
         return word




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple						, __lowerCAmelCase					:						List[Any]		):






         """simple docstring"""






         _lowerCamelCase						:    Tuple         =			[]

         _lowerCamelCase						:    Optional[Any]         =			re.findall(R'''\S+\n?'''						, __lowerCAmelCase		)

         for token in words:
                split_tokens.extend(list(self.bpe(__lowerCAmelCase		).split(''' '''		)		)		)
         return split_tokens




  def 						SCREAMING_SNAKE_CASE    (			self					:						int						, __lowerCAmelCase					:						Dict		):






         """simple docstring"""






         return self.encoder.get(__lowerCAmelCase						, self.encoder.get(self.unk_token		)		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[int]						, __lowerCAmelCase					:						int		):






         """simple docstring"""






         return self.decoder.get(__lowerCAmelCase						, self.unk_token		)




  def 						SCREAMING_SNAKE_CASE    (			self					:						Optional[Any]						, __lowerCAmelCase					:						Union[str, Any]		):






         """simple docstring"""






         _lowerCamelCase						:    str         =			''' '''.join(__lowerCAmelCase		).replace('''@@ '''						, ''''''		).strip()
         return out_string




  def 						SCREAMING_SNAKE_CASE    (			self					:						Tuple						, __lowerCAmelCase					:						str						, __lowerCAmelCase					:						Optional[str] = None		):






         """simple docstring"""






         if not os.path.isdir(__lowerCAmelCase		):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''		)
                return
         _lowerCamelCase						:    List[str]         =			os.path.join(
             __lowerCAmelCase						, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)
         _lowerCamelCase						:    Optional[Any]         =			os.path.join(
             __lowerCAmelCase						, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']		)

         if os.path.abspath(self.vocab_file		) != os.path.abspath(__lowerCAmelCase		):
                copyfile(self.vocab_file						, __lowerCAmelCase		)

         if os.path.abspath(self.merges_file		) != os.path.abspath(__lowerCAmelCase		):
                copyfile(self.merges_file						, __lowerCAmelCase		)

         return out_vocab_file, out_merge_file




  def 						SCREAMING_SNAKE_CASE    (			self					:						str						, __lowerCAmelCase					:						List[str]		):






         """simple docstring"""






         if isinstance(__lowerCAmelCase						, __lowerCAmelCase		):
                try:
                       with open(__lowerCAmelCase						, '''r'''						, encoding='''utf-8'''		) as fd:
                              self.add_from_file(__lowerCAmelCase		)
                except FileNotFoundError as fnfe:
                       raise fnfe
                except UnicodeError:
                       raise Exception(f'''Incorrect encoding detected in {f}, please rebuild the dataset'''		)
                return

         _lowerCamelCase						:    List[str]         =			f.readlines()
         for lineTmp in lines:
                _lowerCamelCase						:    str         =			lineTmp.strip()
                _lowerCamelCase						:    Optional[Any]         =			line.rfind(''' '''		)
                if idx == -1:
                       raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''		)
                _lowerCamelCase						:    int         =			line[:idx]
                _lowerCamelCase						:    Any         =			len(self.encoder		)







"""simple docstring"""






import argparse


lowerCAmelCase__      =    '''docs/source/_static/js/custom.js'''






def 	snake_case_  (				A_ :		List[str]						):


       '''simple docstring'''





       with open(A_,    encoding='''utf-8''',    newline='''\n'''						) as f:
              _lowerCamelCase						:    int         =			f.readlines()
       _lowerCamelCase						:    List[str]         =			0

       # First let's put the right version
       while not lines[index].startswith('''const stableVersion ='''						):
              index += 1
       _lowerCamelCase						:    List[Any]         =			F'''const stableVersion = "v{version}"\n'''

       # Then update the dictionary
       while not lines[index].startswith('''const versionMapping = {'''						):
              index += 1

       # We go until the end
       while not lines[index].startswith('''}'''						):
              index += 1
       # We add the new version at the end
       lines[index - 1] += F'''    "v{version}": "v{version}",\n'''

       with open(A_,    '''w''',    encoding='''utf-8''',    newline='''\n'''						) as f:
              f.writelines(A_						)


if __name__ == "__main__":
   lowerCAmelCase__      =    argparse.ArgumentParser()
   parser.add_argument('''--version''', help='''Release version.''')
   lowerCAmelCase__      =    parser.parse_args()
   update_custom_js(args.version)


