

'''simple docstring'''
import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
    import PIL.Image

    from .features import FeatureType


_snake_case         =  None
_snake_case         =  '<' if sys.byteorder == 'little' else '>'
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
_snake_case         =  [
    np.dtype('|b1'),
    np.dtype('|u1'),
    np.dtype('<u2'),
    np.dtype('>u2'),
    np.dtype('<i2'),
    np.dtype('>i2'),
    np.dtype('<u4'),
    np.dtype('>u4'),
    np.dtype('<i4'),
    np.dtype('>i4'),
    np.dtype('<f4'),
    np.dtype('>f4'),
    np.dtype('<f8'),
    np.dtype('>f8'),
]


@dataclass
class        a__  :
   _SCREAMING_SNAKE_CASE   :					bool     	=							True
   _SCREAMING_SNAKE_CASE   :					Optional[str]     	=							None
   # Automatically constructed
   _SCREAMING_SNAKE_CASE   :					ClassVar[str]     	=							"PIL.Image.Image"
   _SCREAMING_SNAKE_CASE   :					ClassVar[Any]     	=							pa.struct({'bytes': pa.binary(), 'path': pa.string()}  )
   _SCREAMING_SNAKE_CASE   :					str     	=							field(default='Image'  ,				init=lowerCamelCase_  ,				repr=lowerCamelCase_  )


   def __call__(			self  ):





      """simple docstring"""





      return self.pa_type


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if config.PIL_AVAILABLE:
         import PIL.Image
      else:
         raise ImportError("To support encoding images, please install 'Pillow'."  )

      if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
         _lowercase      :				List[Any]      					=       np.array(_UpperCamelCase  )

      if isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
         return {"path": value, "bytes": None}
      elif isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
         return {"path": None, "bytes": value}
      elif isinstance(_UpperCamelCase      ,    np.ndarray  ):
         # convert the image array to PNG/TIFF bytes
         return encode_np_array(_UpperCamelCase  )
      elif isinstance(_UpperCamelCase      ,    PIL.Image.Image  ):
         # convert the PIL image to bytes (default format is PNG/TIFF)
         return encode_pil_image(_UpperCamelCase  )
      elif value.get("path"  ) is not None and os.path.isfile(value["path"]  ):
         # we set "bytes": None to not duplicate the data if they're already available locally
         return {"bytes": None, "path": value.get("path"  )}
      elif value.get("bytes"  ) is not None or value.get("path"  ) is not None:
         # store the image bytes, and path is used to infer the image format using the file extension
         return {"bytes": value.get("bytes"  ), "path": value.get("path"  )}
      else:
         raise ValueError(
             f'''An image sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.'''  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      if not self.decode:
         raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead."  )

      if config.PIL_AVAILABLE:
         import PIL.Image
      else:
         raise ImportError("To support decoding images, please install 'Pillow'."  )

      if token_per_repo_id is None:
         _lowercase      :				List[Any]      					=       {}

      _lowercase					,				_lowercase      :				List[str]      					=       value["path"], value["bytes"]
      if bytes_ is None:
         if path is None:
            raise ValueError(f'''An image should have one of \'path\' or \'bytes\' but both are None in {value}.'''  )
         else:
            if is_local_path(_UpperCamelCase  ):
               _lowercase      :				Optional[int]      					=       PIL.Image.open(_UpperCamelCase  )
            else:
               _lowercase      :				str      					=       path.split("::"  )[-1]
               try:
                  _lowercase      :				int      					=       string_to_dict(_UpperCamelCase      ,    config.HUB_DATASETS_URL  )["repo_id"]
                  _lowercase      :				Tuple      					=       token_per_repo_id.get(_UpperCamelCase  )
               except ValueError:
                  _lowercase      :				Tuple      					=       None
               with xopen(_UpperCamelCase      ,    "rb"      ,    use_auth_token=_UpperCamelCase  ) as f:
                  _lowercase      :				Union[str, Any]      					=       BytesIO(f.read()  )
               _lowercase      :				int      					=       PIL.Image.open(bytes_  )
      else:
         _lowercase      :				Dict      					=       PIL.Image.open(BytesIO(bytes_  )  )
      image.load()  # to avoid "Too many open files" errors
      return image


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      from .features import Value

      return (
          self
          if self.decode
          else {
              "bytes": Value("binary"  ),
              "path": Value("string"  ),
          }
      )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if pa.types.is_string(storage.type  ):
         _lowercase      :				str      					=       pa.array([None] * len(_UpperCamelCase  )      ,    type=pa.binary()  )
         _lowercase      :				Dict      					=       pa.StructArray.from_arrays([bytes_array, storage]      ,    ["bytes", "path"]      ,    mask=storage.is_null()  )
      elif pa.types.is_binary(storage.type  ):
         _lowercase      :				Optional[int]      					=       pa.array([None] * len(_UpperCamelCase  )      ,    type=pa.string()  )
         _lowercase      :				Dict      					=       pa.StructArray.from_arrays([storage, path_array]      ,    ["bytes", "path"]      ,    mask=storage.is_null()  )
      elif pa.types.is_struct(storage.type  ):
         if storage.type.get_field_index("bytes"  ) >= 0:
            _lowercase      :				Optional[int]      					=       storage.field("bytes"  )
         else:
            _lowercase      :				Optional[Any]      					=       pa.array([None] * len(_UpperCamelCase  )      ,    type=pa.binary()  )
         if storage.type.get_field_index("path"  ) >= 0:
            _lowercase      :				Optional[int]      					=       storage.field("path"  )
         else:
            _lowercase      :				List[Any]      					=       pa.array([None] * len(_UpperCamelCase  )      ,    type=pa.string()  )
         _lowercase      :				Dict      					=       pa.StructArray.from_arrays([bytes_array, path_array]      ,    ["bytes", "path"]      ,    mask=storage.is_null()  )
      elif pa.types.is_list(storage.type  ):
         _lowercase      :				List[Any]      					=       pa.array(
             [encode_np_array(np.array(_UpperCamelCase  )  )["bytes"] if arr is not None else None for arr in storage.to_pylist()]      ,    type=pa.binary()      ,    )
         _lowercase      :				Tuple      					=       pa.array([None] * len(_UpperCamelCase  )      ,    type=pa.string()  )
         _lowercase      :				Optional[Any]      					=       pa.StructArray.from_arrays(
             [bytes_array, path_array]      ,    ["bytes", "path"]      ,    mask=bytes_array.is_null()  )
      return array_cast(_UpperCamelCase      ,    self.pa_type  )






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      @no_op_if_value_is_null
      def path_to_bytes(_UpperCamelCase  ):
         with xopen(_UpperCamelCase      ,    "rb"  ) as f:
            _lowercase      :				int      					=       f.read()
         return bytes_

      _lowercase      :				List[str]      					=       pa.array(
          [
              (path_to_bytes(x["path"]  ) if x["bytes"] is None else x["bytes"]) if x is not None else None
              for x in storage.to_pylist()
          ]      ,    type=pa.binary()      ,    )
      _lowercase      :				Dict      					=       pa.array(
          [os.path.basename(_UpperCamelCase  ) if path is not None else None for path in storage.field("path"  ).to_pylist()]      ,    type=pa.string()      ,    )
      _lowercase      :				Union[str, Any]      					=       pa.StructArray.from_arrays([bytes_array, path_array]      ,    ["bytes", "path"]      ,    mask=bytes_array.is_null()  )
      return array_cast(_UpperCamelCase      ,    self.pa_type  )







def  _A		(     )      ->     List[str]:
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError("To support encoding images, please install 'Pillow'."					)

   global _IMAGE_COMPRESSION_FORMATS
   if _IMAGE_COMPRESSION_FORMATS is None:
      PIL.Image.init()
      _lowercase      :				List[str]      					=       list(set(PIL.Image.OPEN.keys()					) & set(PIL.Image.SAVE.keys()					)					)
   return _IMAGE_COMPRESSION_FORMATS







def  _A		(     snake_case					)      ->     bytes:
   _lowercase      :				Any      					=       BytesIO()
   if image.format in list_image_compression_formats():
      _lowercase      :				Tuple      					=       image.format
   else:
      _lowercase      :				Union[str, Any]      					=       "PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
   image.save(snake_case		,							format=snake_case					)
   return buffer.getvalue()







def  _A		(     snake_case					)      ->     dict:
   if hasattr(snake_case		,							"filename"					) and image.filename != "":
      return {"path": image.filename, "bytes": None}
   else:
      return {"path": None, "bytes": image_to_bytes(snake_case					)}







def  _A		(     snake_case					)      ->     dict:
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError("To support encoding images, please install 'Pillow'."					)

   _lowercase      :				Optional[int]      					=       array.dtype
   _lowercase      :				Tuple      					=       dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
   _lowercase      :				int      					=       dtype.kind
   _lowercase      :				Dict      					=       dtype.itemsize

   _lowercase      :				Union[str, Any]      					=       None

   # Multi-channel array case (only np.dtype("|u1") is allowed)
   if array.shape[2:]:
      _lowercase      :				Any      					=       np.dtype("|u1"					)
      if dtype_kind not in ["u", "i"]:
         raise TypeError(
             F'''Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays.'''					)
      if dtype is not dest_dtype:
         warnings.warn(F'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''''					)
    # Exact match
   elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
      _lowercase      :				List[str]      					=       dtype
   else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
      while dtype_itemsize >= 1:
         _lowercase      :				Optional[Any]      					=       dtype_byteorder + dtype_kind + str(snake_case					)
         _lowercase      :				List[str]      					=       np.dtype(snake_case					)
         if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
            warnings.warn(F'''Downcasting array dtype {dtype} to {dest_dtype} to be compatible with \'Pillow\''''					)
            break
         else:
            dtype_itemsize //= 2

   if dest_dtype is None:
      raise TypeError(
          F'''Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}'''					)
   _lowercase      :				Optional[Any]      					=       PIL.Image.fromarray(array.astype(snake_case					)					)
   return {"path": None, "bytes": image_to_bytes(snake_case					)}







def  _A		(     snake_case					)      ->     List[dict]:
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError("To support encoding images, please install 'Pillow'."					)

   if objs:
      _lowercase					,				_lowercase      :				List[Any]      					=       first_non_null_value(snake_case					)
      if isinstance(snake_case		,							snake_case					):
         return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
      if isinstance(snake_case		,							np.ndarray					):
         _lowercase      :				Union[str, Any]      					=       no_op_if_value_is_null(snake_case					)
         return [obj_to_image_dict_func(snake_case					) for obj in objs]
      elif isinstance(snake_case		,							PIL.Image.Image					):
         _lowercase      :				Union[str, Any]      					=       no_op_if_value_is_null(snake_case					)
         return [obj_to_image_dict_func(snake_case					) for obj in objs]
      else:
         return objs
   else:
      return objs









'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case         =  '▁'

_snake_case         =  {'vocab_file': 'spiece.model'}

_snake_case         =  {
    'vocab_file': {'google/pegasus-xsum': 'https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'}
}

_snake_case         =  {
    'google/pegasus-xsum': 512,
}


_snake_case         =  logging.get_logger(__name__)


class        a__  (							lowerCamelCase_  ):
   _SCREAMING_SNAKE_CASE   :					str     	=							VOCAB_FILES_NAMES

   _SCREAMING_SNAKE_CASE   :					Tuple     	=							VOCAB_FILES_NAMES
   _SCREAMING_SNAKE_CASE   :					Any     	=							PRETRAINED_VOCAB_FILES_MAP
   _SCREAMING_SNAKE_CASE   :					Tuple     	=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _SCREAMING_SNAKE_CASE   :					Union[str, Any]     	=							['input_ids', 'attention_mask']


   def __init__(			self      ,    _UpperCamelCase      ,    _UpperCamelCase="<pad>"      ,    _UpperCamelCase="</s>"      ,    _UpperCamelCase="<unk>"      ,    _UpperCamelCase="<mask_2>"      ,    _UpperCamelCase="<mask_1>"      ,    _UpperCamelCase=None      ,    _UpperCamelCase=103      ,    _UpperCamelCase = None      ,    **_UpperCamelCase      ,    ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       offset
      if additional_special_tokens is not None:
         if not isinstance(_UpperCamelCase      ,    _UpperCamelCase  ):
            raise TypeError(
                f'''additional_special_tokens should be of type {type(_UpperCamelCase  )}, but is'''
                f''' {type(_UpperCamelCase  )}'''  )

         _lowercase      :				Dict      					=       (
             ([mask_token_sent] + additional_special_tokens)
             if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
             else additional_special_tokens
         )
         # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
         additional_special_tokens_extended += [
             f'''<unk_{i}>''' for i in range(len(_UpperCamelCase  )      ,    self.offset - 1  )
         ]

         if len(set(_UpperCamelCase  )  ) != len(_UpperCamelCase  ):
            raise ValueError(
                "Please make sure that the provided additional_special_tokens do not contain an incorrectly"
                f''' shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}.'''  )
         _lowercase      :				List[str]      					=       additional_special_tokens_extended
      else:
         _lowercase      :				Dict      					=       [mask_token_sent] if mask_token_sent is not None else []
         additional_special_tokens += [f'''<unk_{i}>''' for i in range(2      ,    self.offset  )]

      _lowercase      :				int      					=       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          eos_token=_UpperCamelCase      ,    unk_token=_UpperCamelCase      ,    mask_token=_UpperCamelCase      ,    pad_token=_UpperCamelCase      ,    mask_token_sent=_UpperCamelCase      ,    offset=_UpperCamelCase      ,    additional_special_tokens=_UpperCamelCase      ,    sp_model_kwargs=self.sp_model_kwargs      ,    **_UpperCamelCase      ,    )
      _lowercase      :				Union[str, Any]      					=       mask_token_sent
      _lowercase      :				str      					=       vocab_file
      _lowercase      :				Dict      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(_UpperCamelCase  )

      # add special tokens to encoder dict
      _lowercase      :				Dict[int, str]      					=       {
          0: self.pad_token,
          1: self.eos_token,
      }

      if self.mask_token_sent is not None:
         self.encoder.update(
             {
                 2: self.mask_token_sent,
                 3: self.mask_token,
             }  )

      if self.offset > 0:
         # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
         # mask_token_sent is already added to list -> so start at 1
         self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1      ,    self.offset - 1  )}  )

      _lowercase      :				Dict[str, int]      					=       {v: k for k, v in self.encoder.items()}


   @property
   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      return len(self.sp_model  ) + self.offset


   def 						_lowerCamelCase       (			self  ):





      """simple docstring"""





      _lowercase      :				Tuple      					=       {self.convert_ids_to_tokens(_UpperCamelCase  ): i for i in range(self.vocab_size  )}
      vocab.update(self.added_tokens_encoder  )
      return vocab


   def __getstate__(			self  ):





      """simple docstring"""





      _lowercase      :				Optional[Any]      					=       self.__dict__.copy()
      _lowercase      :				Union[str, Any]      					=       None
      return state


   def __setstate__(			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       d

      # for backward compatibility
      if not hasattr(self      ,    "sp_model_kwargs"  ):
         _lowercase      :				List[Any]      					=       {}

      _lowercase      :				Any      					=       spm.SentencePieceProcessor(**self.sp_model_kwargs  )
      self.sp_model.Load(self.vocab_file  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      return self.sp_model.encode(_UpperCamelCase      ,    out_type=_UpperCamelCase  )


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if token in self.decoder:
         return self.decoder[token]
      elif token in self.added_tokens_decoder:
         return self.added_tokens_decoder[token]
      _lowercase      :				int      					=       self.sp_model.piece_to_id(_UpperCamelCase  )
      return sp_id + self.offset


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      if index in self.encoder:
         return self.encoder[index]
      elif index in self.added_tokens_encoder:
         return self.added_tokens_encoder[index]
      else:
         _lowercase      :				Any      					=       self.sp_model.IdToPiece(index - self.offset  )
      return token


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				Optional[int]      					=       []
      _lowercase      :				Optional[Any]      					=       ""
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            out_string += self.sp_model.decode(_UpperCamelCase  ) + token
            _lowercase      :				Tuple      					=       []
         else:
            current_sub_tokens.append(_UpperCamelCase  )
      out_string += self.sp_model.decode(_UpperCamelCase  )
      return out_string.strip()


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase=False  ):





      """simple docstring"""





      return 1


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase  ):





      """simple docstring"""





      _lowercase      :				int      					=       set(self.all_special_ids  )  # call it once instead of inside list comp
      all_special_ids.remove(self.unk_token_id  )  # <unk> is only sometimes special

      return [1 if x in all_special_ids else 0 for x in seq]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None      ,    _UpperCamelCase = False  ):





      """simple docstring"""





      if already_has_special_tokens:
         return self._special_token_mask(_UpperCamelCase  )
      elif token_ids_a is None:
         return self._special_token_mask(_UpperCamelCase  ) + [1]
      else:
         return self._special_token_mask(token_ids_a + token_ids_a  ) + [1]


   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase=None  ):





      """simple docstring"""





      if token_ids_a is None:
         return token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return token_ids_a + token_ids_a + [self.eos_token_id]






   def 						_lowerCamelCase       (			self      ,    _UpperCamelCase      ,    _UpperCamelCase = None  ):





      """simple docstring"""





      if not os.path.isdir(_UpperCamelCase  ):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
         return
      _lowercase      :				List[Any]      					=       os.path.join(
          _UpperCamelCase      ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

      if os.path.abspath(self.vocab_file  ) != os.path.abspath(_UpperCamelCase  ) and os.path.isfile(self.vocab_file  ):
         copyfile(self.vocab_file      ,    _UpperCamelCase  )
      elif not os.path.isfile(self.vocab_file  ):
         with open(_UpperCamelCase      ,    "wb"  ) as fi:
            _lowercase      :				Optional[int]      					=       self.sp_model.serialized_model_proto()
            fi.write(_UpperCamelCase  )

      return (out_vocab_file,)






