






"""simple docstring"""







import argparse
import os
import re

import packaging.version


_UpperCamelCase					      =  'examples/'
_UpperCamelCase					      =  {
    'examples': (re.compile(R"""^check_min_version\(\"[^\"]+\"\)\s*$""", re.MULTILINE), 'check_min_version("VERSION")\n'),
    'init': (re.compile(R"""^__version__\s+=\s+\"([^\"]+)\"\s*$""", re.MULTILINE), '__version__ = "VERSION"\n'),
    'setup': (re.compile(R"""^(\s*)version\s*=\s*\"[^\"]+\",""", re.MULTILINE), R'\1version="VERSION",'),
    'doc': (re.compile(R"""^(\s*)release\s*=\s*\"[^\"]+\"$""", re.MULTILINE), 'release = "VERSION"\n'),
}
_UpperCamelCase					      =  {
    'init': 'src/transformers/__init__.py',
    'setup': 'setup.py',
}
_UpperCamelCase					      =  'README.md'

def _a			(		_snake_case				,	_snake_case				,	_snake_case							):







      """simple docstring"""






      with open(UpperCamelCase__				,	"""r"""				,	encoding="""utf-8"""				,	newline="""\n"""							) as f:
            UpperCAmelCase          =	f.read()
      UpperCAmelCase          =	REPLACE_PATTERNS[pattern]
      UpperCAmelCase          =	replace.replace("""VERSION"""				,	UpperCamelCase__							)
      UpperCAmelCase          =	re_pattern.sub(UpperCamelCase__				,	UpperCamelCase__							)
      with open(UpperCamelCase__				,	"""w"""				,	encoding="""utf-8"""				,	newline="""\n"""							) as f:
            f.write(UpperCamelCase__							)

def _a			(		_snake_case							):







      """simple docstring"""






      for folder, directories, fnames in os.walk(UpperCamelCase__							):
            # Removing some of the folders with non-actively maintained examples from the walk
            if "research_projects" in directories:
                  directories.remove("""research_projects"""							)
            if "legacy" in directories:
                  directories.remove("""legacy"""							)
            for fname in fnames:
                  if fname.endswith(""".py"""							):
                        update_version_in_file(os.path.join(UpperCamelCase__				,	UpperCamelCase__							)				,	UpperCamelCase__				,	pattern="""examples"""							)

def _a			(		_snake_case				,	_snake_case=False							):







      """simple docstring"""






      for pattern, fname in REPLACE_FILES.items():
            update_version_in_file(UpperCamelCase__				,	UpperCamelCase__				,	UpperCamelCase__							)
      if not patch:
            update_version_in_examples(UpperCamelCase__							)

def _a			(		):







      """simple docstring"""






      UpperCAmelCase          =	'''🤗 Transformers currently provides the following architectures'''
      UpperCAmelCase          =	'''1. Want to contribute a new model?'''
      with open(UpperCamelCase__				,	"""r"""				,	encoding="""utf-8"""				,	newline="""\n"""							) as f:
            UpperCAmelCase          =	f.readlines()

      # Find the start of the list.
      UpperCAmelCase          =	0
      while not lines[start_index].startswith(_start_prompt							):
            start_index += 1
      start_index += 1

      UpperCAmelCase          =	start_index
      # Update the lines in the model list.
      while not lines[index].startswith(_end_prompt							):
            if lines[index].startswith("""1."""							):
                  UpperCAmelCase          =	lines[index].replace(
                      """https://huggingface.co/docs/transformers/main/model_doc"""				,	"""https://huggingface.co/docs/transformers/model_doc"""				,	)
            index += 1

      with open(UpperCamelCase__				,	"""w"""				,	encoding="""utf-8"""				,	newline="""\n"""							) as f:
            f.writelines(UpperCamelCase__							)

def _a			(		):







      """simple docstring"""






      with open(REPLACE_FILES["""init"""]				,	"""r"""							) as f:
            UpperCAmelCase          =	f.read()
      UpperCAmelCase          =	REPLACE_PATTERNS['''init'''][0].search(UpperCamelCase__							).groups()[0]
      return packaging.version.parse(UpperCamelCase__							)

def _a			(		_snake_case=False							):







      """simple docstring"""






      UpperCAmelCase          =	get_version()
      if patch and default_version.is_devrelease:
            raise ValueError("""Can\'t create a patch version from the dev branch, checkout a released version!"""							)
      if default_version.is_devrelease:
            UpperCAmelCase          =	default_version.base_version
      elif patch:
            UpperCAmelCase          =	F'''{default_version.major}.{default_version.minor}.{default_version.micro + 1}'''
      else:
            UpperCAmelCase          =	F'''{default_version.major}.{default_version.minor + 1}.0'''

      # Now let's ask nicely if that's the right one.
      UpperCAmelCase          =	input(F'''Which version are you releasing? [{default_version}]'''							)
      if len(UpperCamelCase__							) == 0:
            UpperCAmelCase          =	default_version

      print(F'''Updating version to {version}.'''							)
      global_version_update(UpperCamelCase__				,	patch=UpperCamelCase__							)
      if not patch:
            print("""Cleaning main README, don\'t forget to run `make fix-copies`."""							)
            clean_main_ref_in_model_list()

def _a			(		):







      """simple docstring"""






      UpperCAmelCase          =	get_version()
      UpperCAmelCase          =	F'''{current_version.major}.{current_version.minor + 1}.0.dev0'''
      UpperCAmelCase          =	current_version.base_version

      # Check with the user we got that right.
      UpperCAmelCase          =	input(F'''Which version are we developing now? [{dev_version}]'''							)
      if len(UpperCamelCase__							) == 0:
            UpperCAmelCase          =	dev_version

      print(F'''Updating version to {version}.'''							)
      global_version_update(UpperCamelCase__							)
      print("""Cleaning main README, don\'t forget to run `make fix-copies`."""							)
      clean_main_ref_in_model_list()


if __name__ == "__main__":
  _UpperCamelCase					      =  argparse.ArgumentParser()
  parser.add_argument("""--post_release""", action="""store_true""", help="""Whether this is pre or post release.""")
  parser.add_argument("""--patch""", action="""store_true""", help="""Whether or not this is a patch release.""")
  _UpperCamelCase					      =  parser.parse_args()
  if not args.post_release:
    pre_release_work(patch=args.patch)
  elif args.patch:
    print("""Nothing to do after a patch :-)""")
  else:
    post_release_work()











"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_pegasus import PegasusTokenizer
else:
  _UpperCamelCase					      =  None


_UpperCamelCase					      =  logging.get_logger(__name__)


_UpperCamelCase					      =  """▁"""

_UpperCamelCase					      =  {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase					      =  {
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""},
    """tokenizer_file""": {
        """google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/tokenizer.json"""
    },
}

_UpperCamelCase					      =  {
    """google/pegasus-xsum""": 512,
}







class  lowerCamelCase__ (	snake_case			):
    SCREAMING_SNAKE_CASE		=      VOCAB_FILES_NAMES
    SCREAMING_SNAKE_CASE		=      PRETRAINED_VOCAB_FILES_MAP
    SCREAMING_SNAKE_CASE		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    SCREAMING_SNAKE_CASE		=      PegasusTokenizer
    SCREAMING_SNAKE_CASE		=      ['''input_ids''', '''attention_mask''']






    def __init__(			self     ,A=None     ,A=None     ,A="<pad>"     ,A="</s>"     ,A="<unk>"     ,A="<mask_2>"     ,A="<mask_1>"     ,A=None     ,A=103     ,**A     ,):
          UpperCAmelCase          =	offset

          if additional_special_tokens is not None:
                if not isinstance(A     ,A ):
                      raise TypeError(
                          F'''additional_special_tokens should be of type {type(A )}, but is'''
                          F''' {type(A )}''' )

                UpperCAmelCase          =	(
                    ([mask_token_sent] + additional_special_tokens)
                    if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                    else additional_special_tokens
                )
                # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                additional_special_tokens_extended += [
                    F'''<unk_{i}>''' for i in range(len(A )     ,self.offset - 1 )
                ]

                if len(set(A ) ) != len(A ):
                      raise ValueError(
                          """Please make sure that the provided additional_special_tokens do not contain an incorrectly"""
                          F''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.''' )
                UpperCAmelCase          =	additional_special_tokens_extended
          else:
                UpperCAmelCase          =	[mask_token_sent] if mask_token_sent is not None else []
                additional_special_tokens += [F'''<unk_{i}>''' for i in range(2     ,self.offset )]

          super().__init__(
              A     ,tokenizer_file=A     ,pad_token=A     ,eos_token=A     ,unk_token=A     ,mask_token=A     ,mask_token_sent=A     ,offset=A     ,additional_special_tokens=A     ,**A     ,)
          UpperCAmelCase          =	vocab_file
          UpperCAmelCase          =	False if not self.vocab_file else True






    def     _UpperCamelCase							(			self     ,A ):
          UpperCAmelCase          =	set(self.all_special_ids )  # call it once instead of inside list comp
          all_special_ids.remove(self.unk_token_id )  # <unk> is only sometimes special

          if all_special_ids != set(range(len(self.additional_special_tokens ) + 3 ) ):
                raise ValueError(
                    """There should be 3 special tokens: mask_token, pad_token, and eos_token +"""
                    F''' {len(self.additional_special_tokens )} additional_special_tokens, but got {all_special_ids}''' )

          return [1 if x in all_special_ids else 0 for x in seq]






    def     _UpperCamelCase							(			self     ,A     ,A = None     ,A = False ):
          if already_has_special_tokens:
                return self._special_token_mask(A )
          elif token_ids_a is None:
                return self._special_token_mask(A ) + [1]
          else:
                return self._special_token_mask(token_ids_a + token_ids_a ) + [1]






    def     _UpperCamelCase							(			self     ,A     ,A=None ):
          if token_ids_a is None:
                return token_ids_a + [self.eos_token_id]
          # We don't expect to process pairs, but leave the pair logic for API consistency
          return token_ids_a + token_ids_a + [self.eos_token_id]






    def     _UpperCamelCase							(			self     ,A     ,A = None ):
          if not self.can_save_slow_tokenizer:
                raise ValueError(
                    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                    """tokenizer.""" )

          if not os.path.isdir(A ):
                logger.error(F'''Vocabulary path ({save_directory}) should be a directory''' )
                return
          UpperCAmelCase          =	os.path.join(
              A     ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

          if os.path.abspath(self.vocab_file ) != os.path.abspath(A ):
                copyfile(self.vocab_file     ,A )

          return (out_vocab_file,)



