





import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase             =							logging.get_logger(__name__)

_UpperCAmelCase             =							{
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

_UpperCAmelCase             =							{
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

_UpperCAmelCase             =							{
    'vinai/phobert-base': 256,
    'vinai/phobert-large': 256,
}






def 	_SCREAMING_SNAKE_CASE		(      SCREAMING_SNAKE_CASE		:Tuple   )      ->   Optional[Any]:
 __lowerCAmelCase					:	int      =       set()
 __lowerCAmelCase					:	Any      =       word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)   )
  __lowerCAmelCase					:	Any      =       char

 __lowerCAmelCase					:	List[Any]      =       set(SCREAMING_SNAKE_CASE   )
 return pairs







class        snake_case_    (		__lowercase						):
  A_      							=   VOCAB_FILES_NAMES
  A_      							=   PRETRAINED_VOCAB_FILES_MAP
  A_      							=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






  def __init__(							self			:					List[str]       ,  _snake_case			:					Union[str, Any]       ,  _snake_case			:					List[Any]       ,  _snake_case			:					Dict="<s>"       ,  _snake_case			:					Any="</s>"       ,  _snake_case			:					Dict="</s>"       ,  _snake_case			:					Dict="<s>"       ,  _snake_case			:					int="<unk>"       ,  _snake_case			:					Optional[int]="<pad>"       ,  _snake_case			:					Dict="<mask>"       ,  **_snake_case			:					Union[str, Any]       ,  )->int:






   '''simple docstring'''

   super().__init__(
       bos_token=_snake_case       ,  eos_token=_snake_case       ,  unk_token=_snake_case       ,  sep_token=_snake_case       ,  cls_token=_snake_case       ,  pad_token=_snake_case       ,  mask_token=_snake_case       ,  **_snake_case       ,  )

   __lowerCAmelCase					:	List[Any]      =       vocab_file
   __lowerCAmelCase					:	List[Any]      =       merges_file

   __lowerCAmelCase					:	Optional[int]      =       {}
   __lowerCAmelCase					:	Tuple      =       0
   __lowerCAmelCase					:	Union[str, Any]      =       1
   __lowerCAmelCase					:	Tuple      =       2
   __lowerCAmelCase					:	List[str]      =       3

   self.add_from_file(_snake_case      )

   __lowerCAmelCase					:	Dict      =       {v: k for k, v in self.encoder.items()}

   with open(_snake_case       ,  encoding="""utf-8"""      ) as merges_handle:
    __lowerCAmelCase					:	Tuple      =       merges_handle.read().split("""\n"""      )[:-1]
   __lowerCAmelCase					:	int      =       [tuple(merge.split()[:-1]      ) for merge in merges]
   __lowerCAmelCase					:	Any      =       dict(zip(_snake_case       ,  range(len(_snake_case      )      )      )      )
   __lowerCAmelCase					:	Dict      =       {}






  def 				UpperCAmelCase__						(							self			:					int       ,  _snake_case			:					List[int]       ,  _snake_case			:					Optional[List[int]] = None      )->List[int]:






   '''simple docstring'''

   if token_ids_a is None:
    return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
   __lowerCAmelCase					:	int      =       [self.cls_token_id]
   __lowerCAmelCase					:	List[Any]      =       [self.sep_token_id]
   return cls + token_ids_a + sep + sep + token_ids_a + sep






  def 				UpperCAmelCase__						(							self			:					Dict       ,  _snake_case			:					List[int]       ,  _snake_case			:					Optional[List[int]] = None       ,  _snake_case			:					bool = False      )->List[int]:






   '''simple docstring'''

   if already_has_special_tokens:
    return super().get_special_tokens_mask(
        token_ids_a=_snake_case       ,  token_ids_a=_snake_case       ,  already_has_special_tokens=_snake_case      )

   if token_ids_a is None:
    return [1] + ([0] * len(_snake_case      )) + [1]
   return [1] + ([0] * len(_snake_case      )) + [1, 1] + ([0] * len(_snake_case      )) + [1]






  def 				UpperCAmelCase__						(							self			:					int       ,  _snake_case			:					List[int]       ,  _snake_case			:					Optional[List[int]] = None      )->List[int]:






   '''simple docstring'''

   __lowerCAmelCase					:	Optional[Any]      =       [self.sep_token_id]
   __lowerCAmelCase					:	List[str]      =       [self.cls_token_id]

   if token_ids_a is None:
    return len(cls + token_ids_a + sep      ) * [0]
   return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]






  @property
  def 				UpperCAmelCase__						(							self			:					Tuple      )->int:






   '''simple docstring'''

   return len(self.encoder      )






  def 				UpperCAmelCase__						(							self			:					List[str]      )->Optional[Any]:






   '''simple docstring'''

   return dict(self.encoder       ,  **self.added_tokens_encoder      )






  def 				UpperCAmelCase__						(							self			:					Union[str, Any]       ,  _snake_case			:					Optional[Any]      )->Optional[int]:






   '''simple docstring'''

   if token in self.cache:
    return self.cache[token]
   __lowerCAmelCase					:	List[str]      =       tuple(_snake_case      )
   __lowerCAmelCase					:	str      =       tuple(list(word[:-1]      ) + [word[-1] + """</w>"""]      )
   __lowerCAmelCase					:	Dict      =       get_pairs(_snake_case      )

   if not pairs:
    return token

   while True:
    __lowerCAmelCase					:	Union[str, Any]      =       min(_snake_case       ,  key=lambda _snake_case      : self.bpe_ranks.get(_snake_case       ,  float("""inf"""      )      )      )
    if bigram not in self.bpe_ranks:
     break
    __lowerCAmelCase ,					__lowerCAmelCase					:	Optional[Any]      =       bigram
    __lowerCAmelCase					:	List[str]      =       []
    __lowerCAmelCase					:	Any      =       0
    while i < len(_snake_case      ):
     try:
      __lowerCAmelCase					:	Optional[int]      =       word.index(_snake_case       ,  _snake_case      )
     except ValueError:
      new_word.extend(word[i:]      )
      break
     else:
      new_word.extend(word[i:j]      )
      __lowerCAmelCase					:	Any      =       j

     if word[i] == first and i < len(_snake_case      ) - 1 and word[i + 1] == second:
      new_word.append(first + second      )
      i += 2
     else:
      new_word.append(word[i]      )
      i += 1
    __lowerCAmelCase					:	int      =       tuple(_snake_case      )
    __lowerCAmelCase					:	Any      =       new_word
    if len(_snake_case      ) == 1:
     break
    else:
     __lowerCAmelCase					:	Any      =       get_pairs(_snake_case      )
   __lowerCAmelCase					:	str      =       """@@ """.join(_snake_case      )
   __lowerCAmelCase					:	List[Any]      =       word[:-4]
   __lowerCAmelCase					:	List[str]      =       word
   return word






  def 				UpperCAmelCase__						(							self			:					List[Any]       ,  _snake_case			:					Optional[int]      )->Dict:






   '''simple docstring'''

   __lowerCAmelCase					:	Union[str, Any]      =       []

   __lowerCAmelCase					:	Tuple      =       re.findall(R"""\S+\n?"""       ,  _snake_case      )

   for token in words:
    split_tokens.extend(list(self.bpe(_snake_case      ).split(""" """      )      )      )
   return split_tokens






  def 				UpperCAmelCase__						(							self			:					Any       ,  _snake_case			:					Union[str, Any]      )->int:






   '''simple docstring'''

   return self.encoder.get(_snake_case       ,  self.encoder.get(self.unk_token      )      )






  def 				UpperCAmelCase__						(							self			:					Optional[Any]       ,  _snake_case			:					Any      )->Optional[int]:






   '''simple docstring'''

   return self.decoder.get(_snake_case       ,  self.unk_token      )






  def 				UpperCAmelCase__						(							self			:					Optional[Any]       ,  _snake_case			:					Optional[int]      )->Tuple:






   '''simple docstring'''

   __lowerCAmelCase					:	Optional[int]      =       """ """.join(_snake_case      ).replace("""@@ """       ,  """"""      ).strip()
   return out_string






  def 				UpperCAmelCase__						(							self			:					Tuple       ,  _snake_case			:					str       ,  _snake_case			:					Optional[str] = None      )->Tuple[str]:






   '''simple docstring'''

   if not os.path.isdir(_snake_case      ):
    logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
    return
   __lowerCAmelCase					:	List[Any]      =       os.path.join(
       _snake_case       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
   __lowerCAmelCase					:	str      =       os.path.join(
       _snake_case       ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )

   if os.path.abspath(self.vocab_file      ) != os.path.abspath(_snake_case      ):
    copyfile(self.vocab_file       ,  _snake_case      )

   if os.path.abspath(self.merges_file      ) != os.path.abspath(_snake_case      ):
    copyfile(self.merges_file       ,  _snake_case      )

   return out_vocab_file, out_merge_file






  def 				UpperCAmelCase__						(							self			:					Optional[Any]       ,  _snake_case			:					Optional[int]      )->List[Any]:






   '''simple docstring'''

   if isinstance(_snake_case       ,  _snake_case      ):
    try:
     with open(_snake_case       ,  """r"""       ,  encoding="""utf-8"""      ) as fd:
      self.add_from_file(_snake_case      )
    except FileNotFoundError as fnfe:
     raise fnfe
    except UnicodeError:
     raise Exception(F'''Incorrect encoding detected in {f}, please rebuild the dataset'''      )
    return

   __lowerCAmelCase					:	Union[str, Any]      =       f.readlines()
   for lineTmp in lines:
    __lowerCAmelCase					:	int      =       lineTmp.strip()
    __lowerCAmelCase					:	Dict      =       line.rfind(""" """      )
    if idx == -1:
     raise ValueError("""Incorrect dictionary format, expected '<token> <cnt>'"""      )
    __lowerCAmelCase					:	Tuple      =       line[:idx]
    __lowerCAmelCase					:	Tuple      =       len(self.encoder      )






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


_UpperCAmelCase             =							{
    'configuration_gpt_bigcode': ['GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GPTBigCodeConfig'],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCAmelCase             =							[
          'GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST',
          'GPTBigCodeForSequenceClassification',
          'GPTBigCodeForTokenClassification',
          'GPTBigCodeForCausalLM',
          'GPTBigCodeModel',
          'GPTBigCodePreTrainedModel',
      ]

if TYPE_CHECKING:
      from .configuration_gpt_bigcode import GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTBigCodeConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_gpt_bigcode import (
                GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST,
                GPTBigCodeForCausalLM,
                GPTBigCodeForSequenceClassification,
                GPTBigCodeForTokenClassification,
                GPTBigCodeModel,
                GPTBigCodePreTrainedModel,
            )


else:
      import sys

      _UpperCAmelCase             =							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)