


import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase      :int			    =       logging.get_logger(__name__)

__UpperCAmelCase      :Tuple			    =       {
    'vocab_file': 'vocab.txt',
    'merges_file': 'bpe.codes',
}

__UpperCAmelCase      :List[str]			    =       {
    'vocab_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt',
    },
    'merges_file': {
        'vinai/phobert-base': 'https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes',
        'vinai/phobert-large': 'https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes',
    },
}

__UpperCAmelCase      :str			    =       {
    'vinai/phobert-base': 2_5_6,
    'vinai/phobert-large': 2_5_6,
}




def      _a     (						_lowercase :     List[str]  ):





     '''simple docstring'''






     __UpperCAmelCase     :			str  							=      set()
     __UpperCAmelCase     :			Optional[int]  							=      word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)  )
          __UpperCAmelCase     :			str  							=      char

     __UpperCAmelCase     :			int  							=      set(UpperCamelCase__  )
     return pairs



class      a    (		_a	):


     """simple docstring"""






     SCREAMING_SNAKE_CASE	:			Optional[Any]							       =       VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE	:			Any							       =       PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE	:			Any							       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

     def __init__(    self					:		Any							,			snake_case					:		Optional[Any]							,			snake_case					:		Tuple							,			snake_case					:		List[Any]="<s>"							,			snake_case					:		List[str]="</s>"							,			snake_case					:		List[Any]="</s>"							,			snake_case					:		Any="<s>"							,			snake_case					:		int="<unk>"							,			snake_case					:		List[Any]="<pad>"							,			snake_case					:		List[str]="<mask>"							,			**snake_case					:		List[Any]							,			)						->							Any:
          super().__init__(
              bos_token=snake_case							,			eos_token=snake_case							,			unk_token=snake_case							,			sep_token=snake_case							,			cls_token=snake_case							,			pad_token=snake_case							,			mask_token=snake_case							,			**snake_case							,			)

          __UpperCAmelCase     :			List[Any]  							=      vocab_file
          __UpperCAmelCase     :			str  							=      merges_file

          __UpperCAmelCase     :			Optional[int]  							=      {}
          __UpperCAmelCase     :			List[str]  							=      0
          __UpperCAmelCase     :			Union[str, Any]  							=      1
          __UpperCAmelCase     :			str  							=      2
          __UpperCAmelCase     :			int  							=      3

          self.add_from_file(snake_case      )

          __UpperCAmelCase     :			int  							=      {v: k for k, v in self.encoder.items()}

          with open(snake_case							,			encoding='''utf-8'''      ) as merges_handle:
               __UpperCAmelCase     :			Optional[Any]  							=      merges_handle.read().split('''\n'''      )[:-1]
          __UpperCAmelCase     :			List[Any]  							=      [tuple(merge.split()[:-1]      ) for merge in merges]
          __UpperCAmelCase     :			Any  							=      dict(zip(snake_case							,			range(len(snake_case      )      )      )      )
          __UpperCAmelCase     :			List[str]  							=      {}

     def 			lowerCamelCase__			(    self					:		List[str]							,			snake_case					:		Optional[int]							,			snake_case					:		Any = None      )						->							List[int]:

          if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          __UpperCAmelCase     :			Tuple  							=      [self.cls_token_id]
          __UpperCAmelCase     :			List[Any]  							=      [self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep

     def 			lowerCamelCase__			(    self					:		Optional[int]							,			snake_case					:		List[Any]							,			snake_case					:		Tuple = None							,			snake_case					:		Union[str, Any] = False      )						->							List[int]:

          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=snake_case							,			token_ids_a=snake_case							,			already_has_special_tokens=snake_case      )

          if token_ids_a is None:
               return [1] + ([0] * len(snake_case      )) + [1]
          return [1] + ([0] * len(snake_case      )) + [1, 1] + ([0] * len(snake_case      )) + [1]

     def 			lowerCamelCase__			(    self					:		List[str]							,			snake_case					:		int							,			snake_case					:		Any = None      )						->							List[int]:
          __UpperCAmelCase     :			List[str]  							=      [self.sep_token_id]
          __UpperCAmelCase     :			List[str]  							=      [self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep      ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]

     @property
     def 			lowerCamelCase__			(    self					:		List[Any]      )						->							List[str]:
          return len(self.encoder      )

     def 			lowerCamelCase__			(    self					:		Dict      )						->							Union[str, Any]:
          return dict(self.encoder							,			**self.added_tokens_encoder      )

     def 			lowerCamelCase__			(    self					:		int							,			snake_case					:		Dict      )						->							List[Any]:
          if token in self.cache:
               return self.cache[token]
          __UpperCAmelCase     :			str  							=      tuple(snake_case      )
          __UpperCAmelCase     :			List[str]  							=      tuple(list(word[:-1]      ) + [word[-1] + '''</w>''']      )
          __UpperCAmelCase     :			List[Any]  							=      get_pairs(snake_case      )

          if not pairs:
               return token

          while True:
               __UpperCAmelCase     :			Union[str, Any]  							=      min(snake_case							,			key=lambda snake_case      : self.bpe_ranks.get(snake_case							,			float('''inf'''      )      )      )
               if bigram not in self.bpe_ranks:
                    break
               __UpperCAmelCase     :			Any  							=      bigram
               __UpperCAmelCase     :			int  							=      []
               __UpperCAmelCase     :			str  							=      0
               while i < len(snake_case      ):
                    try:
                         __UpperCAmelCase     :			Tuple  							=      word.index(snake_case							,			snake_case      )
                    except ValueError:
                         new_word.extend(word[i:]      )
                         break
                    else:
                         new_word.extend(word[i:j]      )
                         __UpperCAmelCase     :			Union[str, Any]  							=      j

                    if word[i] == first and i < len(snake_case      ) - 1 and word[i + 1] == second:
                         new_word.append(first + second      )
                         i += 2
                    else:
                         new_word.append(word[i]      )
                         i += 1
               __UpperCAmelCase     :			int  							=      tuple(snake_case      )
               __UpperCAmelCase     :			Optional[Any]  							=      new_word
               if len(snake_case      ) == 1:
                    break
               else:
                    __UpperCAmelCase     :			Dict  							=      get_pairs(snake_case      )
          __UpperCAmelCase     :			Optional[Any]  							=      '''@@ '''.join(snake_case      )
          __UpperCAmelCase     :			str  							=      word[:-4]
          __UpperCAmelCase     :			Any  							=      word
          return word

     def 			lowerCamelCase__			(    self					:		List[str]							,			snake_case					:		Any      )						->							Dict:
          __UpperCAmelCase     :			Union[str, Any]  							=      []

          __UpperCAmelCase     :			Union[str, Any]  							=      re.findall(r'''\S+\n?'''							,			snake_case      )

          for token in words:
               split_tokens.extend(list(self.bpe(snake_case      ).split(''' '''      )      )      )
          return split_tokens

     def 			lowerCamelCase__			(    self					:		Union[str, Any]							,			snake_case					:		str      )						->							Optional[Any]:
          return self.encoder.get(snake_case							,			self.encoder.get(self.unk_token      )      )

     def 			lowerCamelCase__			(    self					:		Optional[int]							,			snake_case					:		Union[str, Any]      )						->							List[str]:
          return self.decoder.get(snake_case							,			self.unk_token      )

     def 			lowerCamelCase__			(    self					:		int							,			snake_case					:		str      )						->							str:
          __UpperCAmelCase     :			Optional[Any]  							=      ''' '''.join(snake_case      ).replace('''@@ '''							,			''''''      ).strip()
          return out_string

     def 			lowerCamelCase__			(    self					:		Any							,			snake_case					:		Dict							,			snake_case					:		List[Any] = None      )						->							Tuple[str]:
          if not os.path.isdir(snake_case      ):
               logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
               return
          __UpperCAmelCase     :			Dict  							=      os.path.join(
              snake_case							,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )
          __UpperCAmelCase     :			Optional[Any]  							=      os.path.join(
              snake_case							,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']      )

          if os.path.abspath(self.vocab_file      ) != os.path.abspath(snake_case      ):
               copyfile(self.vocab_file							,			snake_case      )

          if os.path.abspath(self.merges_file      ) != os.path.abspath(snake_case      ):
               copyfile(self.merges_file							,			snake_case      )

          return out_vocab_file, out_merge_file






     def 			lowerCamelCase__			(    self					:		Optional[Any]							,			snake_case					:		Tuple      )						->							Any:
          if isinstance(snake_case							,			snake_case      ):
               try:
                    with open(snake_case							,			'''r'''							,			encoding='''utf-8'''      ) as fd:
                         self.add_from_file(snake_case      )
               except FileNotFoundError as fnfe:
                    raise fnfe
               except UnicodeError:
                    raise Exception(f'Incorrect encoding detected in {f}, please rebuild the dataset'      )
               return

          __UpperCAmelCase     :			int  							=      f.readlines()
          for lineTmp in lines:
               __UpperCAmelCase     :			Optional[int]  							=      lineTmp.strip()
               __UpperCAmelCase     :			str  							=      line.rfind(''' '''      )
               if idx == -1:
                    raise ValueError('''Incorrect dictionary format, expected \'<token> <cnt>\''''      )
               __UpperCAmelCase     :			str  							=      line[:idx]
               __UpperCAmelCase     :			str  							=      len(self.encoder      )



'''simple docstring'''



import os
import sys
import unittest


__UpperCAmelCase      :Optional[int]			    =       os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
__UpperCAmelCase      :Dict			    =       os.path.join(git_repo_path, "src", "diffusers")



class      a    (		unittest.TestCase	):


     """simple docstring"""






     def 			lowerCamelCase__			(    self					:		Any      )						->							int:
          __UpperCAmelCase     :			Optional[Any]  							=      find_backend('''    if not is_torch_available():'''      )
          self.assertEqual(snake_case							,			'''torch'''      )

          # backend_with_underscore = find_backend("    if not is_tensorflow_text_available():")
          # self.assertEqual(backend_with_underscore, "tensorflow_text")

          __UpperCAmelCase     :			Union[str, Any]  							=      find_backend('''    if not (is_torch_available() and is_transformers_available()):'''      )
          self.assertEqual(snake_case							,			'''torch_and_transformers'''      )

          # double_backend_with_underscore = find_backend(
          #    "    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"
          # )
          # self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text")

          __UpperCAmelCase     :			List[str]  							=      find_backend(
              '''    if not (is_torch_available() and is_transformers_available() and is_onnx_available()):'''      )
          self.assertEqual(snake_case							,			'''torch_and_transformers_and_onnx'''      )

     def 			lowerCamelCase__			(    self					:		Optional[int]      )						->							int:
          __UpperCAmelCase     :			Tuple  							=      read_init()
          # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
          self.assertIn('''torch'''							,			snake_case      )
          self.assertIn('''torch_and_transformers'''							,			snake_case      )
          self.assertIn('''flax_and_transformers'''							,			snake_case      )
          self.assertIn('''torch_and_transformers_and_onnx'''							,			snake_case      )

          # Likewise, we can't assert on the exact content of a key
          self.assertIn('''UNet2DModel'''							,			objects['''torch''']      )
          self.assertIn('''FlaxUNet2DConditionModel'''							,			objects['''flax''']      )
          self.assertIn('''StableDiffusionPipeline'''							,			objects['''torch_and_transformers''']      )
          self.assertIn('''FlaxStableDiffusionPipeline'''							,			objects['''flax_and_transformers''']      )
          self.assertIn('''LMSDiscreteScheduler'''							,			objects['''torch_and_scipy''']      )
          self.assertIn('''OnnxStableDiffusionPipeline'''							,			objects['''torch_and_transformers_and_onnx''']      )

     def 			lowerCamelCase__			(    self					:		Optional[int]      )						->							List[Any]:
          __UpperCAmelCase     :			str  							=      create_dummy_object('''CONSTANT'''							,			'''\'torch\''''      )
          self.assertEqual(snake_case							,			'''\nCONSTANT = None\n'''      )

          __UpperCAmelCase     :			Union[str, Any]  							=      create_dummy_object('''function'''							,			'''\'torch\''''      )
          self.assertEqual(
              snake_case							,			'''\ndef function(*args, **kwargs):\n    requires_backends(function, \'torch\')\n'''      )

          __UpperCAmelCase     :			Optional[int]  							=      '''
class FakeClass(metaclass=DummyObject):
    _backends = \'torch\'

    def __init__(self, *args, **kwargs):
        requires_backends(self, \'torch\')

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, \'torch\')

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, \'torch\')
'''
          __UpperCAmelCase     :			Optional[Any]  							=      create_dummy_object('''FakeClass'''							,			'''\'torch\''''      )
          self.assertEqual(snake_case							,			snake_case      )






     def 			lowerCamelCase__			(    self					:		int      )						->							List[Any]:
          __UpperCAmelCase     :			List[str]  							=      '''# This file is autogenerated by the command `make fix-copies`, do not edit.
from ..utils import DummyObject, requires_backends


CONSTANT = None


def function(*args, **kwargs):
    requires_backends(function, ["torch"])


class FakeClass(metaclass=DummyObject):
    _backends = ["torch"]

    def __init__(self, *args, **kwargs):
        requires_backends(self, ["torch"])

    @classmethod
    def from_config(cls, *args, **kwargs):
        requires_backends(cls, ["torch"])

    @classmethod
    def from_pretrained(cls, *args, **kwargs):
        requires_backends(cls, ["torch"])
'''
          __UpperCAmelCase     :			Optional[int]  							=      create_dummy_files({'''torch''': ['''CONSTANT''', '''function''', '''FakeClass''']}      )
          self.assertEqual(dummy_files['''torch''']							,			snake_case      )