




'''simple docstring'''


import collections
import os
import re
from pathlib import Path


_snake_case				:				Tuple       							=		'src/transformers'


# Matches is_xxx_available()
_snake_case				:				Any       							=		re.compile(R'is\_([a-z_]*)_available()')
# Catches a one-line _import_struct = {xxx}
_snake_case				:				Any       							=		re.compile(R'^_import_structure\s+=\s+\{([^\}]+)\}')
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
_snake_case				:				Optional[int]       							=		re.compile(R'\s+"\S*":\s+\[([^\]]*)\]')
# Catches a line if not is_foo_available
_snake_case				:				Optional[Any]       							=		re.compile(R'^\s*if\s+not\s+is\_[a-z_]*\_available\(\)')
# Catches a line _import_struct["bla"].append("foo")
_snake_case				:				str       							=		re.compile(R'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
_snake_case				:				Any       							=		re.compile(R'^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]')
# Catches a line with an object between quotes and a comma:     "MyModel",
_snake_case				:				Dict       							=		re.compile(R'^\s+"([^"]+)",')
# Catches a line with objects between brackets only:    ["foo", "bar"],
_snake_case				:				str       							=		re.compile(R'^\s+\[([^\]]+)\]')
# Catches a line with from foo import bar, bla, boo
_snake_case				:				str       							=		re.compile(R'\s+from\s+\S*\s+import\s+([^\(\s].*)\n')
# Catches a line with try:
_snake_case				:				Optional[Any]       							=		re.compile(R'^\s*try:')
# Catches a line with else:
_snake_case				:				str       							=		re.compile(R'^\s*else:')
def 		snake_case_   (UpperCamelCase	:  int				):





							'''simple docstring'''


							if _re_test_backend.search(UpperCamelCase				) is None:
														return None
							_a     					=    [b[0] for b in _re_backend.findall(UpperCamelCase				)]
							backends.sort()
							return "_and_".join(UpperCamelCase				)
def 		snake_case_   (UpperCamelCase	:  str				):





							'''simple docstring'''


							with open(UpperCamelCase			,						'''r'''			,						encoding='''utf-8'''			,						newline='''\n'''				) as f:
														_a     					=    f.readlines()

							_a     					=    0
							while line_index < len(UpperCamelCase				) and not lines[line_index].startswith('''_import_structure = {'''				):
														line_index += 1

							# If this is a traditional init, just return.
							if line_index >= len(UpperCamelCase				):
														return None

							# First grab the objects without a specific backend in _import_structure
							_a     					=    []
							while not lines[line_index].startswith('''if TYPE_CHECKING'''				) and find_backend(lines[line_index]				) is None:
														_a     					=    lines[line_index]
														# If we have everything on a single line, let's deal with it.
														if _re_one_line_import_struct.search(UpperCamelCase				):
																					_a     					=    _re_one_line_import_struct.search(UpperCamelCase				).groups()[0]
																					_a     					=    re.findall(R'''\[([^\]]+)\]'''			,						UpperCamelCase				)
																					for imp in imports:
																												objects.extend([obj[1:-1] for obj in imp.split(''', '''				)]				)
																					line_index += 1
																					continue
														_a     					=    _re_import_struct_key_value.search(UpperCamelCase				)
														if single_line_import_search is not None:
																					_a     					=    [obj[1:-1] for obj in single_line_import_search.groups()[0].split(''', '''				) if len(UpperCamelCase				) > 0]
																					objects.extend(UpperCamelCase				)
														elif line.startswith(''' ''' * 8 + '''"'''				):
																					objects.append(line[9:-3]				)
														line_index += 1

							_a     					=    {'''none''': objects}
							# Let's continue with backend-specific objects in _import_structure
							while not lines[line_index].startswith('''if TYPE_CHECKING'''				):
														# If the line is an if not is_backend_available, we grab all objects associated.
														_a     					=    find_backend(lines[line_index]				)
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]				) is None:
																					_a     					=    None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]				) is None:
																												line_index += 1

																					line_index += 1

																					_a     					=    []
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]				) <= 1 or lines[line_index].startswith(''' ''' * 4				):
																												_a     					=    lines[line_index]
																												if _re_import_struct_add_one.search(UpperCamelCase				) is not None:
																																			objects.append(_re_import_struct_add_one.search(UpperCamelCase				).groups()[0]				)
																												elif _re_import_struct_add_many.search(UpperCamelCase				) is not None:
																																			_a     					=    _re_import_struct_add_many.search(UpperCamelCase				).groups()[0].split(''', '''				)
																																			_a     					=    [obj[1:-1] for obj in imports if len(UpperCamelCase				) > 0]
																																			objects.extend(UpperCamelCase				)
																												elif _re_between_brackets.search(UpperCamelCase				) is not None:
																																			_a     					=    _re_between_brackets.search(UpperCamelCase				).groups()[0].split(''', '''				)
																																			_a     					=    [obj[1:-1] for obj in imports if len(UpperCamelCase				) > 0]
																																			objects.extend(UpperCamelCase				)
																												elif _re_quote_object.search(UpperCamelCase				) is not None:
																																			objects.append(_re_quote_object.search(UpperCamelCase				).groups()[0]				)
																												elif line.startswith(''' ''' * 8 + '''"'''				):
																																			objects.append(line[9:-3]				)
																												elif line.startswith(''' ''' * 12 + '''"'''				):
																																			objects.append(line[13:-3]				)
																												line_index += 1

																					_a     					=    objects
														else:
																					line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
							_a     					=    []
							while (
							    line_index < len(UpperCamelCase				)
							    and find_backend(lines[line_index]				) is None
							    and not lines[line_index].startswith('''else'''				)
							):
														_a     					=    lines[line_index]
														_a     					=    _re_import.search(UpperCamelCase				)
														if single_line_import_search is not None:
																					objects.extend(single_line_import_search.groups()[0].split(''', '''				)				)
														elif line.startswith(''' ''' * 8				):
																					objects.append(line[8:-2]				)
														line_index += 1

							_a     					=    {'''none''': objects}
							# Let's continue with backend-specific objects
							while line_index < len(UpperCamelCase				):
														# If the line is an if is_backend_available, we grab all objects associated.
														_a     					=    find_backend(lines[line_index]				)
														# Check if the backend declaration is inside a try block:
														if _re_try.search(lines[line_index - 1]				) is None:
																					_a     					=    None

														if backend is not None:
																					line_index += 1

																					# Scroll until we hit the else block of try-except-else
																					while _re_else.search(lines[line_index]				) is None:
																												line_index += 1

																					line_index += 1

																					_a     					=    []
																					# Until we unindent, add backend objects to the list
																					while len(lines[line_index]				) <= 1 or lines[line_index].startswith(''' ''' * 8				):
																												_a     					=    lines[line_index]
																												_a     					=    _re_import.search(UpperCamelCase				)
																												if single_line_import_search is not None:
																																			objects.extend(single_line_import_search.groups()[0].split(''', '''				)				)
																												elif line.startswith(''' ''' * 12				):
																																			objects.append(line[12:-2]				)
																												line_index += 1

																					_a     					=    objects
														else:
																					line_index += 1

							return import_dict_objects, type_hint_objects
def 		snake_case_   (UpperCamelCase	:  Dict			,						UpperCamelCase	:  Tuple				):





							'''simple docstring'''


							def find_duplicates(UpperCamelCase	:  Dict				):
														return [k for k, v in collections.Counter(UpperCamelCase				).items() if v > 1]

							if list(import_dict_objects.keys()				) != list(type_hint_objects.keys()				):
														return ["Both sides of the init do not have the same backends!"]

							_a     					=    []
							for key in import_dict_objects.keys():
														_a     					=    find_duplicates(import_dict_objects[key]				)
														if duplicate_imports:
																					errors.append(f'Duplicate _import_structure definitions for: {duplicate_imports}'				)
														_a     					=    find_duplicates(type_hint_objects[key]				)
														if duplicate_type_hints:
																					errors.append(f'Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}'				)

														if sorted(set(import_dict_objects[key]				)				) != sorted(set(type_hint_objects[key]				)				):
																					_a     					=    '''base imports''' if key == '''none''' else f'{key} backend'
																					errors.append(f'Differences for {name}:'				)
																					for a in type_hint_objects[key]:
																												if a not in import_dict_objects[key]:
																																			errors.append(f'  {a} in TYPE_HINT but not in _import_structure.'				)
																					for a in import_dict_objects[key]:
																												if a not in type_hint_objects[key]:
																																			errors.append(f'  {a} in _import_structure but not in TYPE_HINT.'				)
							return errors
def 		snake_case_   ():





							'''simple docstring'''


							_a     					=    []
							for root, _, files in os.walk(UpperCamelCase				):
														if "__init__.py" in files:
																					_a     					=    os.path.join(UpperCamelCase			,						'''__init__.py'''				)
																					_a     					=    parse_init(UpperCamelCase				)
																					if objects is not None:
																												_a     					=    analyze_results(*UpperCamelCase				)
																												if len(UpperCamelCase				) > 0:
																																			_a     					=    f'Problem in {fname}, both halves do not define the same objects.\n{errors[0]}'
																																			failures.append('''\n'''.join(UpperCamelCase				)				)
							if len(UpperCamelCase				) > 0:
														raise ValueError('''\n\n'''.join(UpperCamelCase				)				)
def 		snake_case_   ():





							'''simple docstring'''


							_a     					=    []
							for path, directories, files in os.walk(UpperCamelCase				):
														for folder in directories:
																					# Ignore private modules
																					if folder.startswith('''_'''				):
																												directories.remove(UpperCamelCase				)
																												continue
																					# Ignore leftovers from branches (empty folders apart from pycache)
																					if len(list((Path(UpperCamelCase				) / folder).glob('''*.py'''				)				)				) == 0:
																												continue
																					_a     					=    str((Path(UpperCamelCase				) / folder).relative_to(UpperCamelCase				)				)
																					_a     					=    short_path.replace(os.path.sep			,						'''.'''				)
																					submodules.append(UpperCamelCase				)
														for fname in files:
																					if fname == "__init__.py":
																												continue
																					_a     					=    str((Path(UpperCamelCase				) / fname).relative_to(UpperCamelCase				)				)
																					_a     					=    short_path.replace('''.py'''			,						''''''				).replace(os.path.sep			,						'''.'''				)
																					if len(submodule.split('''.'''				)				) == 1:
																												submodules.append(UpperCamelCase				)
							return submodules


_snake_case				:				str       							=		[
    'convert_pytorch_checkpoint_to_tf2',
    'modeling_flax_pytorch_utils',
    'models.esm.openfold_utils',
]
def 		snake_case_   ():





							'''simple docstring'''


							from transformers.utils import direct_transformers_import

							_a     					=    direct_transformers_import(UpperCamelCase				)

							_a     					=    set(transformers._import_structure.keys()				)
							# This contains all the base keys of the _import_structure object defined in the init, but if the user is missing
							# some optional dependencies, they may not have all of them. Thus we read the init to read all additions and
							# (potentiall re-) add them.
							with open(os.path.join(UpperCamelCase			,						'''__init__.py'''				)			,						'''r'''				) as f:
														_a     					=    f.read()
							import_structure_keys.update(set(re.findall(R'''import_structure\[\"([^\"]*)\"\]'''			,						UpperCamelCase				)				)				)

							_a     					=    [
							    module
							    for module in get_transformers_submodules()
							    if module not in IGNORE_SUBMODULES and module not in import_structure_keys
							]

							if len(UpperCamelCase				) > 0:
														_a     					=    '''\n'''.join(f'- {module}' for module in module_not_registered				)
														raise ValueError(
														    '''The following submodules are not properly registed in the main init of Transformers:\n'''
														    f'{list_of_modules}\n'
														    '''Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'''				)


if __name__ == "__main__":
				check_all_inits()
				check_submodules()







'''simple docstring'''


import os
import re
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case				:				str       							=		logging.get_logger(__name__)

_snake_case				:				Tuple       							=		{'vocab_file': 'spiece.model'}

_snake_case				:				Optional[int]       							=		{
    'vocab_file': {
        'google/bigbird-roberta-base': 'https://huggingface.co/google/bigbird-roberta-base/resolve/main/spiece.model',
        'google/bigbird-roberta-large': (
            'https://huggingface.co/google/bigbird-roberta-large/resolve/main/spiece.model'
        ),
        'google/bigbird-base-trivia-itc': (
            'https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/spiece.model'
        ),
    }
}

_snake_case				:				Tuple       							=		{
    'google/bigbird-roberta-base': 4096,
    'google/bigbird-roberta-large': 4096,
    'google/bigbird-base-trivia-itc': 4096,
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    ['input_ids', 'attention_mask']
							lowercase_          =    []







							def __init__(  self	:  Optional[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[int]="<unk>"							,      lowerCAmelCase_	:  Tuple="<s>"							,      lowerCAmelCase_	:  List[str]="</s>"							,      lowerCAmelCase_	:  int="<pad>"							,      lowerCAmelCase_	:  Union[str, Any]="[SEP]"							,      lowerCAmelCase_	:  Dict="[MASK]"							,      lowerCAmelCase_	:  Optional[int]="[CLS]"							,      lowerCAmelCase_	:  Optional[Dict[str, Any]] = None							,      **lowerCAmelCase_	:  Union[str, Any]							,      )      ->							None:


														"""simple docstring"""


														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else bos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else eos_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else unk_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else pad_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else cls_token
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else sep_token

														# Mask token behave like a normal word, i.e. include the space before it
														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else mask_token

														_a     					=    {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      sp_model_kwargs=self.sp_model_kwargs							,      **lowerCAmelCase_							,      )

														_a     					=    vocab_file

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(lowerCAmelCase_ )







							@property
							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							int:


														"""simple docstring"""


														return self.sp_model.get_piece_size()







							def        __lowerCAmelCase      (  self	:  str )      ->							List[str]:


														"""simple docstring"""


														_a     					=    {self.convert_ids_to_tokens(lowerCAmelCase_ ): i for i in range(self.vocab_size )}
														vocab.update(self.added_tokens_encoder )
														return vocab







							def __getstate__(  self	:  str )      ->							Tuple:


														"""simple docstring"""


														_a     					=    self.__dict__.copy()
														_a     					=    None
														return state







							def __setstate__(  self	:  List[str]							,      lowerCAmelCase_	:  Any )      ->							Dict:


														"""simple docstring"""


														_a     					=    d

														# for backward compatibility
														if not hasattr(self							,      '''sp_model_kwargs''' ):
																					_a     					=    {}

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(self.vocab_file )







							def        __lowerCAmelCase      (  self	:  Optional[int]							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														return self.sp_model.encode(lowerCAmelCase_							,      out_type=lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  List[str] )      ->							int:


														"""simple docstring"""


														return self.sp_model.piece_to_id(lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  Tuple )      ->							str:


														"""simple docstring"""


														_a     					=    self.sp_model.IdToPiece(lowerCAmelCase_ )
														return token







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Optional[Any]:


														"""simple docstring"""


														_a     					=    []
														_a     					=    ''''''
														_a     					=    False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(lowerCAmelCase_ ) + token
																												_a     					=    True
																												_a     					=    []
																					else:
																												current_sub_tokens.append(lowerCAmelCase_ )
																												_a     					=    False
														out_string += self.sp_model.decode(lowerCAmelCase_ )
														return out_string.strip()







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  bool = False							,      lowerCAmelCase_	:  bool = None							,      lowerCAmelCase_	:  bool = True							,      **lowerCAmelCase_	:  Tuple							,      )      ->							str:


														"""simple docstring"""


														_a     					=    kwargs.pop('''use_source_tokenizer'''							,      lowerCAmelCase_ )

														_a     					=    self.convert_ids_to_tokens(lowerCAmelCase_							,      skip_special_tokens=lowerCAmelCase_ )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														_a     					=    []
														_a     					=    []
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_ ) )
																																			_a     					=    []
																												sub_texts.append(lowerCAmelCase_ )
																					else:
																												current_sub_text.append(lowerCAmelCase_ )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(lowerCAmelCase_ ) )

														# Mimic the behavior of the Rust tokenizer:
														# No space before [MASK] and [SEP]
														if spaces_between_special_tokens:
																					_a     					=    re.sub(R''' (\[(MASK|SEP)\])'''							,      R'''\1'''							,      ''' '''.join(lowerCAmelCase_ ) )
														else:
																					_a     					=    ''''''.join(lowerCAmelCase_ )

														_a     					=    (
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					_a     					=    self.clean_up_tokenization(lowerCAmelCase_ )
																					return clean_text
														else:
																					return text







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														if not os.path.isdir(lowerCAmelCase_ ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
																					return
														_a     					=    os.path.join(
														    lowerCAmelCase_							,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

														if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase_ ) and os.path.isfile(self.vocab_file ):
																					copyfile(self.vocab_file							,      lowerCAmelCase_ )
														elif not os.path.isfile(self.vocab_file ):
																					with open(lowerCAmelCase_							,      '''wb''' ) as fi:
																												_a     					=    self.sp_model.serialized_model_proto()
																												fi.write(lowerCAmelCase_ )

														return (out_vocab_file,)







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														_a     					=    [self.sep_token_id]
														return cls + token_ids_a + sep + token_ids_a + sep







							def        __lowerCAmelCase      (  self	:  List[Any]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None							,      lowerCAmelCase_	:  bool = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														if token_ids_a is None:
																					return [1] + ([0] * len(lowerCAmelCase_ )) + [1]
														return [1] + ([0] * len(lowerCAmelCase_ )) + [1] + ([0] * len(lowerCAmelCase_ )) + [1]







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]

