






'''simple docstring'''
import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible






class        __SCREAMING_SNAKE_CASE (     unittest.TestCase			):




 def        __magic_name__		( self		:							Any )		->  List[str]:
       SCREAMING_SNAKE_CASE__       :			Dict													=[
           '''safety_checker/pytorch_model.bin''',
           '''safety_checker/model.safetensors''',
           '''vae/diffusion_pytorch_model.bin''',
           '''vae/diffusion_pytorch_model.safetensors''',
           '''text_encoder/pytorch_model.bin''',
           '''text_encoder/model.safetensors''',
           '''unet/diffusion_pytorch_model.bin''',
           '''unet/diffusion_pytorch_model.safetensors''',
       ]
       self.assertTrue(is_safetensors_compatible(__lowercase ) )




 def        __magic_name__		( self		:							Tuple )		->  str:
       SCREAMING_SNAKE_CASE__       :			Tuple													=[
           '''unet/diffusion_pytorch_model.bin''',
           '''unet/diffusion_pytorch_model.safetensors''',
       ]
       self.assertTrue(is_safetensors_compatible(__lowercase ) )




 def        __magic_name__		( self		:							Tuple )		->  int:
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=[
           '''safety_checker/pytorch_model.bin''',
           '''safety_checker/model.safetensors''',
           '''vae/diffusion_pytorch_model.bin''',
           '''vae/diffusion_pytorch_model.safetensors''',
           '''text_encoder/pytorch_model.bin''',
           '''text_encoder/model.safetensors''',
           '''unet/diffusion_pytorch_model.bin''',
           # Removed: 'unet/diffusion_pytorch_model.safetensors',
       ]
       self.assertFalse(is_safetensors_compatible(__lowercase ) )




 def        __magic_name__		( self		:							str )		->  int:
       SCREAMING_SNAKE_CASE__       :			int													=[
           '''text_encoder/pytorch_model.bin''',
           '''text_encoder/model.safetensors''',
       ]
       self.assertTrue(is_safetensors_compatible(__lowercase ) )




 def        __magic_name__		( self		:							List[str] )		->  Tuple:
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													=[
           '''safety_checker/pytorch_model.bin''',
           '''safety_checker/model.safetensors''',
           '''vae/diffusion_pytorch_model.bin''',
           '''vae/diffusion_pytorch_model.safetensors''',
           '''text_encoder/pytorch_model.bin''',
           # Removed: 'text_encoder/model.safetensors',
           '''unet/diffusion_pytorch_model.bin''',
           '''unet/diffusion_pytorch_model.safetensors''',
       ]
       self.assertFalse(is_safetensors_compatible(__lowercase ) )




 def        __magic_name__		( self		:							Optional[Any] )		->  Optional[Any]:
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=[
           '''safety_checker/pytorch_model.fp16.bin''',
           '''safety_checker/model.fp16.safetensors''',
           '''vae/diffusion_pytorch_model.fp16.bin''',
           '''vae/diffusion_pytorch_model.fp16.safetensors''',
           '''text_encoder/pytorch_model.fp16.bin''',
           '''text_encoder/model.fp16.safetensors''',
           '''unet/diffusion_pytorch_model.fp16.bin''',
           '''unet/diffusion_pytorch_model.fp16.safetensors''',
       ]
       SCREAMING_SNAKE_CASE__       :			int													='''fp16'''
       self.assertTrue(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )




 def        __magic_name__		( self		:							Any )		->  Tuple:
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=[
           '''unet/diffusion_pytorch_model.fp16.bin''',
           '''unet/diffusion_pytorch_model.fp16.safetensors''',
       ]
       SCREAMING_SNAKE_CASE__       :			Dict													='''fp16'''
       self.assertTrue(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )




 def        __magic_name__		( self		:							List[str] )		->  Any:
       # pass variant but use the non-variant filenames
       SCREAMING_SNAKE_CASE__       :			List[str]													=[
           '''unet/diffusion_pytorch_model.bin''',
           '''unet/diffusion_pytorch_model.safetensors''',
       ]
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													='''fp16'''
       self.assertTrue(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )




 def        __magic_name__		( self		:							Any )		->  int:
       SCREAMING_SNAKE_CASE__       :			List[str]													=[
           '''safety_checker/pytorch_model.fp16.bin''',
           '''safety_checker/model.fp16.safetensors''',
           '''vae/diffusion_pytorch_model.fp16.bin''',
           '''vae/diffusion_pytorch_model.fp16.safetensors''',
           '''text_encoder/pytorch_model.fp16.bin''',
           '''text_encoder/model.fp16.safetensors''',
           '''unet/diffusion_pytorch_model.fp16.bin''',
           # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
       ]
       SCREAMING_SNAKE_CASE__       :			int													='''fp16'''
       self.assertFalse(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )




 def        __magic_name__		( self		:							Any )		->  Union[str, Any]:
       SCREAMING_SNAKE_CASE__       :			Tuple													=[
           '''text_encoder/pytorch_model.fp16.bin''',
           '''text_encoder/model.fp16.safetensors''',
       ]
       SCREAMING_SNAKE_CASE__       :			Optional[Any]													='''fp16'''
       self.assertTrue(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )




 def        __magic_name__		( self		:							Any )		->  Optional[Any]:
       # pass variant but use the non-variant filenames
       SCREAMING_SNAKE_CASE__       :			Dict													=[
           '''text_encoder/pytorch_model.bin''',
           '''text_encoder/model.safetensors''',
       ]
       SCREAMING_SNAKE_CASE__       :			str													='''fp16'''
       self.assertTrue(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )




 def        __magic_name__		( self		:							Optional[int] )		->  int:
       SCREAMING_SNAKE_CASE__       :			int													=[
           '''safety_checker/pytorch_model.fp16.bin''',
           '''safety_checker/model.fp16.safetensors''',
           '''vae/diffusion_pytorch_model.fp16.bin''',
           '''vae/diffusion_pytorch_model.fp16.safetensors''',
           '''text_encoder/pytorch_model.fp16.bin''',
           # 'text_encoder/model.fp16.safetensors',
           '''unet/diffusion_pytorch_model.fp16.bin''',
           '''unet/diffusion_pytorch_model.fp16.safetensors''',
       ]
       SCREAMING_SNAKE_CASE__       :			Any													='''fp16'''
       self.assertFalse(is_safetensors_compatible(__lowercase   , variant=__lowercase ) )







'''simple docstring'''
import json
import os
from typing import Dict, List, Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_					=	logging.get_logger(__name__)


a_					=	{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

a_					=	{
    'vocab_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/vocab.json'
    },
    'merges_file': {
        'facebook/blenderbot_small-90M': 'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/merges.txt'
    },
    'tokenizer_config_file': {
        'facebook/blenderbot_small-90M': (
            'https://huggingface.co/facebook/blenderbot_small-90M/resolve/main/tokenizer_config.json'
        )
    },
}

a_					=	{'facebook/blenderbot_small-90M': 5_1_2}





def 	_a(					UpperCamelCase__			:  Dict							):




      '''simple docstring'''






      SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=set()
      SCREAMING_SNAKE_CASE__       :			Optional[Any]													=word[0]
      for char in word[1:]:
            pairs.add((prev_char, char)							)
            SCREAMING_SNAKE_CASE__       :			Optional[Any]													=char

      SCREAMING_SNAKE_CASE__       :			Any													=set(UpperCamelCase__							)
      return pairs







class        __SCREAMING_SNAKE_CASE (     lowerCamelCase			):
 snake_case_												=							VOCAB_FILES_NAMES
 snake_case_												=							PRETRAINED_VOCAB_FILES_MAP
 snake_case_												=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 snake_case_												=							["""input_ids""", """attention_mask"""]




 def __init__( self		:							Tuple   , __lowercase		:							int   , __lowercase		:							Optional[int]   , __lowercase		:							List[str]="__start__"   , __lowercase		:							Union[str, Any]="__end__"   , __lowercase		:							str="__unk__"   , __lowercase		:							Union[str, Any]="__null__"   , **__lowercase		:							List[str]   , )		->  Optional[int]:
       super().__init__(unk_token=__lowercase   , bos_token=__lowercase   , eos_token=__lowercase   , pad_token=__lowercase   , **__lowercase )

       with open(__lowercase   , encoding='''utf-8''' ) as vocab_handle:
             SCREAMING_SNAKE_CASE__       :			Any													=json.load(__lowercase )
       SCREAMING_SNAKE_CASE__       :			Dict													={v: k for k, v in self.encoder.items()}
       with open(__lowercase   , encoding='''utf-8''' ) as merges_handle:
             SCREAMING_SNAKE_CASE__       :			int													=merges_handle.read().split('''\n''' )[1:-1]
       SCREAMING_SNAKE_CASE__       :			List[Any]													=[tuple(merge.split() ) for merge in merges]
       SCREAMING_SNAKE_CASE__       :			int													=dict(zip(__lowercase   , range(len(__lowercase ) ) ) )
       SCREAMING_SNAKE_CASE__       :			Dict													={}




 @property
 def        __magic_name__		( self		:							Any )		->  int:
       return len(self.encoder )




 def        __magic_name__		( self		:							Tuple )		->  Dict:
       return dict(self.encoder   , **self.added_tokens_encoder )




 def        __magic_name__		( self		:							Optional[Any]   , __lowercase		:							str )		->  str:
       if token in self.cache:
             return self.cache[token]
       SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=re.sub('''([.,!?()])'''   , r''' \1'''   , __lowercase )
       SCREAMING_SNAKE_CASE__       :			int													=re.sub('''(\')'''   , r''' \1 '''   , __lowercase )
       SCREAMING_SNAKE_CASE__       :			Dict													=re.sub(r'''\s{2,}'''   , ''' '''   , __lowercase )
       if "\n" in token:
             SCREAMING_SNAKE_CASE__       :			List[str]													=token.replace('''\n'''   , ''' __newln__''' )

       SCREAMING_SNAKE_CASE__       :			Dict													=token.split(''' ''' )
       SCREAMING_SNAKE_CASE__       :			Dict													=[]
       for token in tokens:
             if not len(__lowercase ):
                   continue

             SCREAMING_SNAKE_CASE__       :			Union[str, Any]													=token.lower()
             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=tuple(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Optional[int]													=tuple(list(word[:-1] ) + [word[-1] + '''</w>'''] )
             SCREAMING_SNAKE_CASE__       :			Optional[Any]													=get_pairs(__lowercase )

             if not pairs:
                   words.append(__lowercase )
                   continue

             while True:
                   SCREAMING_SNAKE_CASE__       :			int													=min(__lowercase   , key=lambda __lowercase : self.bpe_ranks.get(__lowercase   , float('''inf''' ) ) )
                   if bigram not in self.bpe_ranks:
                         break
                   SCREAMING_SNAKE_CASE__						,		SCREAMING_SNAKE_CASE__       :			Any													=bigram
                   SCREAMING_SNAKE_CASE__       :			List[Any]													=[]
                   SCREAMING_SNAKE_CASE__       :			Tuple													=0

                   while i < len(__lowercase ):
                         try:
                               SCREAMING_SNAKE_CASE__       :			Optional[Any]													=word.index(__lowercase   , __lowercase )
                               new_word.extend(word[i:j] )
                               SCREAMING_SNAKE_CASE__       :			Optional[int]													=j
                         except ValueError:
                               new_word.extend(word[i:] )
                               break

                         if word[i] == first and i < len(__lowercase ) - 1 and word[i + 1] == second:
                               new_word.append(first + second )
                               i += 2
                         else:
                               new_word.append(word[i] )
                               i += 1
                   SCREAMING_SNAKE_CASE__       :			int													=tuple(__lowercase )
                   SCREAMING_SNAKE_CASE__       :			int													=new_word
                   if len(__lowercase ) == 1:
                         break
                   else:
                         SCREAMING_SNAKE_CASE__       :			Any													=get_pairs(__lowercase )
             SCREAMING_SNAKE_CASE__       :			Optional[int]													='''@@ '''.join(__lowercase )
             SCREAMING_SNAKE_CASE__       :			List[Any]													=word[:-4]

             SCREAMING_SNAKE_CASE__       :			str													=word
             words.append(__lowercase )
       return " ".join(__lowercase )




 def        __magic_name__		( self		:							str   , __lowercase		:							str )		->  List[str]:
       SCREAMING_SNAKE_CASE__       :			Dict													=[]

       SCREAMING_SNAKE_CASE__       :			List[Any]													=re.findall(r'''\S+\n?'''   , __lowercase )

       for token in words:
             split_tokens.extend(list(self.bpe(__lowercase ).split(''' ''' ) ) )
       return split_tokens




 def        __magic_name__		( self		:							List[Any]   , __lowercase		:							str )		->  int:
       SCREAMING_SNAKE_CASE__       :			int													=token.lower()
       return self.encoder.get(__lowercase   , self.encoder.get(self.unk_token ) )




 def        __magic_name__		( self		:							int   , __lowercase		:							int )		->  str:
       return self.decoder.get(__lowercase   , self.unk_token )




 def        __magic_name__		( self		:							List[Any]   , __lowercase		:							List[str] )		->  str:
       SCREAMING_SNAKE_CASE__       :			Optional[int]													=''' '''.join(__lowercase ).replace('''@@ '''   , '''''' ).strip()
       return out_string




 def        __magic_name__		( self		:							Tuple   , __lowercase		:							str   , __lowercase		:							Optional[str] = None )		->  Tuple[str]:
       if not os.path.isdir(__lowercase ):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory" )
             return
       SCREAMING_SNAKE_CASE__       :			str													=os.path.join(
           __lowercase   , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
       SCREAMING_SNAKE_CASE__       :			int													=os.path.join(
           __lowercase   , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

       with open(__lowercase   , '''w'''   , encoding='''utf-8''' ) as f:
             f.write(json.dumps(self.encoder   , indent=2   , sort_keys=__lowercase   , ensure_ascii=__lowercase ) + '''\n''' )

       SCREAMING_SNAKE_CASE__       :			List[str]													=0
       with open(__lowercase   , '''w'''   , encoding='''utf-8''' ) as writer:
             writer.write('''#version: 0.2\n''' )
             for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   , key=lambda __lowercase : kv[1] ):
                   if index != token_index:
                         logger.warning(
                             F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                             ''' Please check that the tokenizer is not corrupted!''' )
                         SCREAMING_SNAKE_CASE__       :			Any													=token_index
                   writer.write(''' '''.join(__lowercase ) + '''\n''' )
                   index += 1

       return vocab_file, merge_file