


import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map







def SCREAMING_SNAKE_CASE    (	lowercase_						)       ->	List[Tuple[int, ...]]:



       """simple docstring"""

       A__							=   []
       if isinstance(lowercase_			,    lowercase_						):
              for v in tree.values():
                     shapes.extend(_fetch_dims(lowercase_						)						)
       elif isinstance(lowercase_			,    (list, tuple)						):
              for t in tree:
                     shapes.extend(_fetch_dims(lowercase_						)						)
       elif isinstance(lowercase_			,    torch.Tensor						):
              shapes.append(tree.shape						)
       else:
              raise ValueError('''Not supported'''						)

       return shapes







@torch.jit.ignore
def SCREAMING_SNAKE_CASE    (	lowercase_			,    lowercase_						)       ->	Tuple[int, ...]:



       """simple docstring"""

       A__							=   []
       for d in reversed(lowercase_						):
              idx.append(flat_idx % d						)
              A__							=   flat_idx // d

       return tuple(reversed(lowercase_						)						)







@torch.jit.ignore
def SCREAMING_SNAKE_CASE    (	lowercase_			,    lowercase_			,    lowercase_			,    lowercase_ = None			,    lowercase_ = None			,    )       ->	List[Tuple[slice, ...]]:



       """simple docstring"""

       def reduce_edge_list(lowercase_						) -> None:
              A__							=   True
              for i in range(len(lowercase_						)						):
                     A__							=   -1 * (i + 1)
                     l[reversed_idx] &= tally
                     A__							=   l[reversed_idx]

       if start_edges is None:
              A__							=   [s == 0 for s in start]
              reduce_edge_list(lowercase_						)
       if end_edges is None:
              A__							=   [e == (d - 1) for e, d in zip(lowercase_			,    lowercase_						)]
              reduce_edge_list(lowercase_						)

       # Base cases. Either start/end are empty and we're done, or the final,
       # one-dimensional tensor can be simply sliced
       if len(lowercase_						) == 0:
              return [()]
       elif len(lowercase_						) == 1:
              return [(slice(start[0]			,    end[0] + 1						),)]

       A__							=   []
       A__							=   []

       # Dimensions common to start and end can be selected directly
       for s, e in zip(lowercase_			,    lowercase_						):
              if s == e:
                     path_list.append(slice(lowercase_			,    s + 1						)						)
              else:
                     break

       A__							=   tuple(lowercase_						)
       A__							=   len(lowercase_						)

       # start == end, and we're done
       if divergence_idx == len(lowercase_						):
              return [path]

       def upper() -> Tuple[Tuple[slice, ...], ...]:
              assert start_edges is not None
              assert end_edges is not None

              A__							=   start[divergence_idx]
              return tuple(
                  path + (slice(lowercase_			,    sdi + 1						),) + s
                  for s in _get_minimal_slice_set(
                      start[divergence_idx + 1 :]			,    [d - 1 for d in dims[divergence_idx + 1 :]]			,    dims[divergence_idx + 1 :]			,    start_edges=start_edges[divergence_idx + 1 :]			,    end_edges=[True for _ in end_edges[divergence_idx + 1 :]]			,    )						)

       def lower() -> Tuple[Tuple[slice, ...], ...]:
              assert start_edges is not None
              assert end_edges is not None

              A__							=   end[divergence_idx]
              return tuple(
                  path + (slice(lowercase_			,    edi + 1						),) + s
                  for s in _get_minimal_slice_set(
                      [0 for _ in start[divergence_idx + 1 :]]			,    end[divergence_idx + 1 :]			,    dims[divergence_idx + 1 :]			,    start_edges=[True for _ in start_edges[divergence_idx + 1 :]]			,    end_edges=end_edges[divergence_idx + 1 :]			,    )						)

       # If both start and end are at the edges of the subtree rooted at
       # divergence_idx, we can just select the whole subtree at once
       if start_edges[divergence_idx] and end_edges[divergence_idx]:
              slices.append(path + (slice(start[divergence_idx]			,    end[divergence_idx] + 1						),)						)
       # If just start is at the edge, we can grab almost all of the subtree,
       # treating only the ragged bottom edge as an edge case
       elif start_edges[divergence_idx]:
              slices.append(path + (slice(start[divergence_idx]			,    end[divergence_idx]						),)						)
              slices.extend(lower()						)
       # Analogous to the previous case, but the top is ragged this time
       elif end_edges[divergence_idx]:
              slices.extend(upper()						)
              slices.append(path + (slice(start[divergence_idx] + 1			,    end[divergence_idx] + 1						),)						)
       # If both sides of the range are ragged, we need to handle both sides
       # separately. If there's contiguous meat in between them, we can index it
       # in one big chunk
       else:
              slices.extend(upper()						)
              A__							=   end[divergence_idx] - start[divergence_idx]
              if middle_ground > 1:
                     slices.append(path + (slice(start[divergence_idx] + 1			,    end[divergence_idx]						),)						)
              slices.extend(lower()						)

       return slices







@torch.jit.ignore
def SCREAMING_SNAKE_CASE    (	lowercase_			,    lowercase_			,    lowercase_			,    lowercase_						)       ->	torch.Tensor:



       """simple docstring"""

       A__							=   t.shape[:no_batch_dims]
       A__							=   list(_flat_idx_to_idx(lowercase_			,    lowercase_						)						)
       # _get_minimal_slice_set is inclusive
       A__							=   list(_flat_idx_to_idx(flat_end - 1			,    lowercase_						)						)

       # Get an ordered list of slices to perform
       A__							=   _get_minimal_slice_set(
           lowercase_			,    lowercase_			,    lowercase_			,    )

       A__							=   [t[s] for s in slices]

       return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]						) for s in sliced_tensors]						)







def SCREAMING_SNAKE_CASE    (	lowercase_			,    lowercase_			,    lowercase_			,    lowercase_			,    lowercase_ = False			,    lowercase_ = None			,    lowercase_ = False			,    )       ->	Any:



       """simple docstring"""

       if not (len(lowercase_						) > 0):
              raise ValueError('''Must provide at least one input'''						)

       A__							=   [shape[:no_batch_dims] for shape in _fetch_dims(lowercase_						)]
       A__							=   tuple([max(lowercase_						) for s in zip(*lowercase_						)]						)

       def _prep_inputs(lowercase_						) -> torch.Tensor:
              if not low_mem:
                     if not sum(t.shape[:no_batch_dims]						) == no_batch_dims:
                            A__							=   t.expand(orig_batch_dims + t.shape[no_batch_dims:]						)
                     A__							=   t.reshape(-1			,    *t.shape[no_batch_dims:]						)
              else:
                     A__							=   t.expand(orig_batch_dims + t.shape[no_batch_dims:]						)
              return t

       A__							=   tensor_tree_map(_prep_inputs			,    lowercase_						)
       A__							=   None
       if _out is not None:
              A__							=   tensor_tree_map(lambda lowercase_						: t.view([-1] + list(t.shape[no_batch_dims:]						)						)			,    _out						)

       A__							=   1
       for d in orig_batch_dims:
              flat_batch_dim *= d

       A__							=   flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

       def _select_chunk(lowercase_						) -> torch.Tensor:
              return t[i : i + chunk_size] if t.shape[0] != 1 else t

       A__							=   0
       A__							=   prepped_outputs
       for _ in range(lowercase_						):
              # Chunk the input
              if not low_mem:
                     A__							=   _select_chunk
              else:
                     A__							=   partial(
                         _chunk_slice			,    flat_start=lowercase_			,    flat_end=min(lowercase_			,    i + chunk_size						)			,    no_batch_dims=len(lowercase_						)			,    )

              A__							=   tensor_tree_map(lowercase_			,    lowercase_						)

              # Run the layer on the chunk
              A__							=   layer(**lowercase_						)

              # Allocate space for the output
              if out is None:
                     A__							=   tensor_tree_map(lambda lowercase_						: t.new_zeros((flat_batch_dim,) + t.shape[1:]						)			,    lowercase_						)

              # Put the chunk in its pre-allocated space
              if isinstance(lowercase_			,    lowercase_						):

                     def assign(lowercase_			,    lowercase_						) -> None:
                            for k, v in da.items():
                                   if isinstance(lowercase_			,    lowercase_						):
                                          assign(lowercase_			,    da[k]						)
                                   else:
                                          if _add_into_out:
                                                 v[i : i + chunk_size] += da[k]
                                          else:
                                                 A__							=   da[k]

                     assign(lowercase_			,    lowercase_						)
              elif isinstance(lowercase_			,    lowercase_						):
                     for xa, xa in zip(lowercase_			,    lowercase_						):
                            if _add_into_out:
                                   xa[i : i + chunk_size] += xa
                            else:
                                   A__							=   xa
              elif isinstance(lowercase_			,    torch.Tensor						):
                     if _add_into_out:
                            out[i : i + chunk_size] += output_chunk
                     else:
                            A__							=   output_chunk
              else:
                     raise ValueError('''Not supported'''						)

              i += chunk_size

       A__							=   tensor_tree_map(lambda lowercase_						: t.view(orig_batch_dims + t.shape[1:]						)			,    lowercase_						)

       return out


class     UpperCamelCase_   :

    '''simple docstring'''






    def __init__(      self				:     str ,			UpperCAmelCase__				:     int = 512 ,			)						->Union[str, Any]:





           '''simple docstring'''

           A__							=   max_chunk_size
           A__							=   None
           A__							=   None


    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any] ,			UpperCAmelCase__				:     Callable ,			UpperCAmelCase__				:     tuple ,			UpperCAmelCase__				:     int)						->int:





           '''simple docstring'''

           logging.info('''Tuning chunk size...''')

           if min_chunk_size >= self.max_chunk_size:
                  return min_chunk_size

           A__							=   [2**l for l in range(int(math.log(self.max_chunk_size ,			2)) + 1)]
           A__							=   [c for c in candidates if c > min_chunk_size]
           A__							=   [min_chunk_size] + candidates
           candidates[-1] += 4

           def test_chunk_size(UpperCAmelCase__				:     int) -> bool:
                  try:
                         with torch.no_grad():
                                fn(*UpperCAmelCase__ ,			chunk_size=UpperCAmelCase__)
                         return True
                  except RuntimeError:
                         return False

           A__							=   0
           A__							=   len(UpperCAmelCase__) - 1
           while i > min_viable_chunk_size_index:
                  A__							=   test_chunk_size(candidates[i])
                  if not viable:
                         A__							=   (min_viable_chunk_size_index + i) // 2
                  else:
                         A__							=   i
                         A__							=   (i + len(UpperCAmelCase__) - 1) // 2

           return candidates[min_viable_chunk_size_index]


    def 	SCREAMING_SNAKE_CASE							(      self				:     Tuple ,			UpperCAmelCase__				:     Iterable ,			UpperCAmelCase__				:     Iterable)						->bool:





           '''simple docstring'''

           A__							=   True
           for aa, aa in zip(UpperCAmelCase__ ,			UpperCAmelCase__):
                  assert type(UpperCAmelCase__) == type(UpperCAmelCase__)
                  if isinstance(UpperCAmelCase__ ,			(list, tuple)):
                         consistent &= self._compare_arg_caches(UpperCAmelCase__ ,			UpperCAmelCase__)
                  elif isinstance(UpperCAmelCase__ ,			UpperCAmelCase__):
                         A__							=   [v for _, v in sorted(aa.items() ,			key=lambda UpperCAmelCase__: x[0])]
                         A__							=   [v for _, v in sorted(aa.items() ,			key=lambda UpperCAmelCase__: x[0])]
                         consistent &= self._compare_arg_caches(UpperCAmelCase__ ,			UpperCAmelCase__)
                  else:
                         consistent &= aa == aa

           return consistent





    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any] ,			UpperCAmelCase__				:     Callable ,			UpperCAmelCase__				:     tuple ,			UpperCAmelCase__				:     int ,			)						->int:





           '''simple docstring'''

           A__							=   True
           A__							=   tree_map(lambda UpperCAmelCase__: a.shape if isinstance(UpperCAmelCase__ ,			torch.Tensor) else a ,			UpperCAmelCase__ ,			UpperCAmelCase__)
           if self.cached_arg_data is not None:
                  # If args have changed shape/value, we need to re-tune
                  assert len(self.cached_arg_data) == len(UpperCAmelCase__)
                  A__							=   self._compare_arg_caches(self.cached_arg_data ,			UpperCAmelCase__)
           else:
                  # Otherwise, we can reuse the precomputed value
                  A__							=   False

           if not consistent:
                  A__							=   self._determine_favorable_chunk_size(
                      UpperCAmelCase__ ,			UpperCAmelCase__ ,			UpperCAmelCase__ ,			)
                  A__							=   arg_data

           assert self.cached_chunk_size is not None

           return self.cached_chunk_size








import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase     :		Union[str, Any]								=       """▁"""

_lowerCamelCase     :		Optional[Any]								=       {"""vocab_file""": """spiece.model"""}

_lowerCamelCase     :		str								=       {
    """vocab_file""": {"""google/pegasus-xsum""": """https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model"""}
}

_lowerCamelCase     :		List[str]								=       {
    """google/pegasus-xsum""": 512,
}


_lowerCamelCase     :		int								=       logging.get_logger(__name__)


class     UpperCamelCase_   (			UpperCAmelCase__					):

    '''simple docstring'''






    UpperCAmelCase__							  =				VOCAB_FILES_NAMES

    UpperCAmelCase__							  =				VOCAB_FILES_NAMES
    UpperCAmelCase__							  =				PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__							  =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase__							  =				['''input_ids''', '''attention_mask''']


    def __init__(      self				:     Optional[int] ,			UpperCAmelCase__				:     Tuple ,			UpperCAmelCase__				:     str="<pad>" ,			UpperCAmelCase__				:     Optional[Any]="</s>" ,			UpperCAmelCase__				:     Any="<unk>" ,			UpperCAmelCase__				:     Union[str, Any]="<mask_2>" ,			UpperCAmelCase__				:     List[str]="<mask_1>" ,			UpperCAmelCase__				:     Dict=None ,			UpperCAmelCase__				:     Optional[Any]=103 ,			UpperCAmelCase__				:     Optional[Dict[str, Any]] = None ,			**UpperCAmelCase__				:     Dict ,			)						->None:





           '''simple docstring'''

           A__							=   offset
           if additional_special_tokens is not None:
                  if not isinstance(UpperCAmelCase__ ,			UpperCAmelCase__):
                         raise TypeError(
                             f"""additional_special_tokens should be of type {type(UpperCAmelCase__)}, but is"""
                             f""" {type(UpperCAmelCase__)}""")

                  A__							=   (
                      ([mask_token_sent] + additional_special_tokens)
                      if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
                      else additional_special_tokens
                  )
                  # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
                  additional_special_tokens_extended += [
                      f"""<unk_{i}>""" for i in range(len(UpperCAmelCase__) ,			self.offset - 1)
                  ]

                  if len(set(UpperCAmelCase__)) != len(UpperCAmelCase__):
                         raise ValueError(
                             '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
                             f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.""")
                  A__							=   additional_special_tokens_extended
           else:
                  A__							=   [mask_token_sent] if mask_token_sent is not None else []
                  additional_special_tokens += [f"""<unk_{i}>""" for i in range(2 ,			self.offset)]

           A__							=   {} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               eos_token=UpperCAmelCase__ ,			unk_token=UpperCAmelCase__ ,			mask_token=UpperCAmelCase__ ,			pad_token=UpperCAmelCase__ ,			mask_token_sent=UpperCAmelCase__ ,			offset=UpperCAmelCase__ ,			additional_special_tokens=UpperCAmelCase__ ,			sp_model_kwargs=self.sp_model_kwargs ,			**UpperCAmelCase__ ,			)
           A__							=   mask_token_sent
           A__							=   vocab_file
           A__							=   spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(UpperCAmelCase__)

           # add special tokens to encoder dict
           A__							=   {
               0: self.pad_token,
               1: self.eos_token,
           }

           if self.mask_token_sent is not None:
                  self.encoder.update(
                      {
                          2: self.mask_token_sent,
                          3: self.mask_token,
                      })

           if self.offset > 0:
                  # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
                  # mask_token_sent is already added to list -> so start at 1
                  self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1 ,			self.offset - 1)})

           A__							=   {v: k for k, v in self.encoder.items()}


    @property
    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any])						->int:





           '''simple docstring'''

           return len(self.sp_model) + self.offset


    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[Any])						->Dict[str, int]:





           '''simple docstring'''

           A__							=   {self.convert_ids_to_tokens(UpperCAmelCase__): i for i in range(self.vocab_size)}
           vocab.update(self.added_tokens_encoder)
           return vocab


    def __getstate__(      self				:     Any)						->Union[str, Any]:





           '''simple docstring'''

           A__							=   self.__dict__.copy()
           A__							=   None
           return state


    def __setstate__(      self				:     int ,			UpperCAmelCase__				:     Optional[int])						->Optional[int]:





           '''simple docstring'''

           A__							=   d

           # for backward compatibility
           if not hasattr(self ,			'''sp_model_kwargs'''):
                  A__							=   {}

           A__							=   spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(self.vocab_file)


    def 	SCREAMING_SNAKE_CASE							(      self				:     str ,			UpperCAmelCase__				:     str)						->List[str]:





           '''simple docstring'''

           return self.sp_model.encode(UpperCAmelCase__ ,			out_type=UpperCAmelCase__)


    def 	SCREAMING_SNAKE_CASE							(      self				:     Any ,			UpperCAmelCase__				:     str)						->int:





           '''simple docstring'''

           if token in self.decoder:
                  return self.decoder[token]
           elif token in self.added_tokens_decoder:
                  return self.added_tokens_decoder[token]
           A__							=   self.sp_model.piece_to_id(UpperCAmelCase__)
           return sp_id + self.offset


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[Any] ,			UpperCAmelCase__				:     int)						->str:





           '''simple docstring'''

           if index in self.encoder:
                  return self.encoder[index]
           elif index in self.added_tokens_encoder:
                  return self.added_tokens_encoder[index]
           else:
                  A__							=   self.sp_model.IdToPiece(index - self.offset)
           return token


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[str] ,			UpperCAmelCase__				:     int)						->Optional[int]:





           '''simple docstring'''

           A__							=   []
           A__							=   ''''''
           for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                         out_string += self.sp_model.decode(UpperCAmelCase__) + token
                         A__							=   []
                  else:
                         current_sub_tokens.append(UpperCAmelCase__)
           out_string += self.sp_model.decode(UpperCAmelCase__)
           return out_string.strip()


    def 	SCREAMING_SNAKE_CASE							(      self				:     Union[str, Any] ,			UpperCAmelCase__				:     Optional[int]=False)						->Union[str, Any]:





           '''simple docstring'''

           return 1


    def 	SCREAMING_SNAKE_CASE							(      self				:     int ,			UpperCAmelCase__				:     Union[str, Any])						->Optional[Any]:





           '''simple docstring'''

           A__							=   set(self.all_special_ids)  # call it once instead of inside list comp
           all_special_ids.remove(self.unk_token_id)  # <unk> is only sometimes special

           return [1 if x in all_special_ids else 0 for x in seq]


    def 	SCREAMING_SNAKE_CASE							(      self				:     int ,			UpperCAmelCase__				:     List ,			UpperCAmelCase__				:     Optional[List] = None ,			UpperCAmelCase__				:     bool = False)						->List[int]:





           '''simple docstring'''

           if already_has_special_tokens:
                  return self._special_token_mask(UpperCAmelCase__)
           elif token_ids_a is None:
                  return self._special_token_mask(UpperCAmelCase__) + [1]
           else:
                  return self._special_token_mask(token_ids_a + token_ids_a) + [1]


    def 	SCREAMING_SNAKE_CASE							(      self				:     Tuple ,			UpperCAmelCase__				:     List[str] ,			UpperCAmelCase__				:     List[str]=None)						->List[int]:





           '''simple docstring'''

           if token_ids_a is None:
                  return token_ids_a + [self.eos_token_id]
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return token_ids_a + token_ids_a + [self.eos_token_id]





    def 	SCREAMING_SNAKE_CASE							(      self				:     str ,			UpperCAmelCase__				:     str ,			UpperCAmelCase__				:     Optional[str] = None)						->Tuple[str]:





           '''simple docstring'''

           if not os.path.isdir(UpperCAmelCase__):
                  logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
                  return
           A__							=   os.path.join(
               UpperCAmelCase__ ,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCAmelCase__) and os.path.isfile(self.vocab_file):
                  copyfile(self.vocab_file ,			UpperCAmelCase__)
           elif not os.path.isfile(self.vocab_file):
                  with open(UpperCAmelCase__ ,			'''wb''') as fi:
                         A__							=   self.sp_model.serialized_model_proto()
                         fi.write(UpperCAmelCase__)

           return (out_vocab_file,)


