


"""simple docstring"""







import warnings
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase    =				logging.get_logger(__name__)






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							int     =	["""input_ids""", """attention_mask"""]






    def __init__( self    :   Optional[int]    ,			__UpperCAmelCase    :   Optional[int]="</s>"    ,			__UpperCAmelCase    :   Union[str, Any]="<unk>"    ,			__UpperCAmelCase    :   Tuple="<pad>"    ,			__UpperCAmelCase    :   Dict=125    ,			__UpperCAmelCase    :   List[Any]=None    ,			**__UpperCAmelCase    :   Optional[int]    ,			):
      # Add extra_ids to the special token list
      if extra_ids > 0 and additional_special_tokens is None:
        a						:		List[Any]											=      [f'''<extra_id_{i}>''' for i in range(__UpperCAmelCase)]
      elif extra_ids > 0 and additional_special_tokens is not None:
        # Check that we have the right number of extra_id special tokens
        a						:		Optional[Any]											=      len(set(filter(lambda __UpperCAmelCase: bool("extra_id" in str(__UpperCAmelCase))    ,			__UpperCAmelCase)))
        if extra_tokens != extra_ids:
          raise ValueError(
              f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
              " provided to ByT5Tokenizer. In this case the additional_special_tokens must include the"
              " extra_ids tokens")

      a						:		List[str]											=      AddedToken(__UpperCAmelCase    ,			lstrip=__UpperCAmelCase    ,			rstrip=__UpperCAmelCase) if isinstance(__UpperCAmelCase    ,			__UpperCAmelCase) else pad_token
      a						:		Tuple											=      AddedToken(__UpperCAmelCase    ,			lstrip=__UpperCAmelCase    ,			rstrip=__UpperCAmelCase) if isinstance(__UpperCAmelCase    ,			__UpperCAmelCase) else eos_token
      a						:		List[str]											=      AddedToken(__UpperCAmelCase    ,			lstrip=__UpperCAmelCase    ,			rstrip=__UpperCAmelCase) if isinstance(__UpperCAmelCase    ,			__UpperCAmelCase) else unk_token

      super().__init__(
          eos_token=__UpperCAmelCase    ,			unk_token=__UpperCAmelCase    ,			pad_token=__UpperCAmelCase    ,			extra_ids=__UpperCAmelCase    ,			additional_special_tokens=__UpperCAmelCase    ,			**__UpperCAmelCase    ,			)

      a						:		Union[str, Any]											=      extra_ids

      a						:		Dict											=      2**8  # utf is 8 bits

      # define special tokens dict
      a						:		Dict[int, str]											=      {
          self.pad_token: 0,
          self.eos_token: 1,
          self.unk_token: 2,
      }
      a						:		Union[str, Any]											=      len(self.special_tokens_encoder)
      a						:		int											=      len(__UpperCAmelCase)
      for i, token in enumerate(__UpperCAmelCase):
        a						:		str											=      self.vocab_size + i - n
      a						:		Dict[str, int]											=      {v: k for k, v in self.special_tokens_encoder.items()}






    @property
    def      __snake_case							( self    :   str):
      return self._utf_vocab_size + self._num_special_tokens + self._extra_ids






    def      __snake_case							( self    :   Optional[Any]    ,			__UpperCAmelCase    :   List[int]    ,			__UpperCAmelCase    :   Optional[List[int]] = None    ,			__UpperCAmelCase    :   bool = False):
      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__UpperCAmelCase    ,			token_ids_a=__UpperCAmelCase    ,			already_has_special_tokens=__UpperCAmelCase)

      # normal case: some special tokens
      if token_ids_a is None:
        return ([0] * len(__UpperCAmelCase)) + [1]
      return ([0] * len(__UpperCAmelCase)) + [1] + ([0] * len(__UpperCAmelCase)) + [1]






    def      __snake_case							( self    :   int    ,			__UpperCAmelCase    :   List[int]):
      if len(__UpperCAmelCase) > 0 and token_ids[-1] == self.eos_token_id:
        warnings.warn(
            f'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
            " eos tokens being added.")
        return token_ids
      else:
        return token_ids + [self.eos_token_id]






    def      __snake_case							( self    :   List[Any]    ,			__UpperCAmelCase    :   List[int]    ,			__UpperCAmelCase    :   Optional[List[int]] = None):
      a						:		Union[str, Any]											=      [self.eos_token_id]

      if token_ids_a is None:
        return len(token_ids_a + eos) * [0]
      return len(token_ids_a + eos + token_ids_a + eos) * [0]






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   List[int]    ,			__UpperCAmelCase    :   Optional[List[int]] = None):
      a						:		Optional[int]											=      self._add_eos_if_not_present(__UpperCAmelCase)
      if token_ids_a is None:
        return token_ids_a
      else:
        a						:		str											=      self._add_eos_if_not_present(__UpperCAmelCase)
        return token_ids_a + token_ids_a






    def      __snake_case							( self    :   Dict    ,			__UpperCAmelCase    :   str):
      a						:		List[str]											=      [chr(__UpperCAmelCase) for i in text.encode("utf-8")]
      return tokens






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   int):
      if token in self.special_tokens_encoder:
        a						:		List[str]											=      self.special_tokens_encoder[token]
      elif token in self.added_tokens_encoder:
        a						:		List[Any]											=      self.added_tokens_encoder[token]
      elif len(__UpperCAmelCase) != 1:
        a						:		Dict											=      self.unk_token_id
      else:
        a						:		Any											=      ord(__UpperCAmelCase) + self._num_special_tokens
      return token_id






    def      __snake_case							( self    :   Dict    ,			__UpperCAmelCase    :   Union[str, Any]):
      if index in self.special_tokens_decoder:
        a						:		Union[str, Any]											=      self.special_tokens_decoder[index]
      else:
        a						:		int											=      chr(index - self._num_special_tokens)
      return token






    def      __snake_case							( self    :   List[Any]    ,			__UpperCAmelCase    :   Any):
      a						:		Optional[Any]											=      B""
      for token in tokens:
        if token in self.special_tokens_decoder:
          a						:		List[Any]											=      self.special_tokens_decoder[token].encode("utf-8")
        elif token in self.added_tokens_decoder:
          a						:		Union[str, Any]											=      self.special_tokens_decoder[token].encode("utf-8")
        elif token in self.special_tokens_encoder:
          a						:		Optional[Any]											=      token.encode("utf-8")
        elif token in self.added_tokens_encoder:
          a						:		Optional[int]											=      token.encode("utf-8")
        else:
          a						:		Optional[Any]											=      bytes([ord(__UpperCAmelCase)])
        bstring += tok_string
      a						:		Tuple											=      bstring.decode("utf-8"    ,			errors="ignore")
      return string






    def      __snake_case							( self    :   Dict    ,			__UpperCAmelCase    :   str    ,			__UpperCAmelCase    :   Optional[str] = None):
      return ()










"""simple docstring"""







import warnings

from transformers import AutoTokenizer
from transformers.utils import is_torch_available
from transformers.utils.generic import ExplicitEnum

from ...processing_utils import ProcessorMixin


if is_torch_available():
 import torch






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							str     =	"""char"""
    UpperCAmelCase	:							Optional[Any]     =	"""bpe"""
    UpperCAmelCase	:							Optional[Any]     =	"""wp"""






__lowercase    =				(DecodeType.CHARACTER, DecodeType.BPE, DecodeType.WORDPIECE)






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase	:							Optional[Any]     =	["""image_processor""", """char_tokenizer"""]
    UpperCAmelCase	:							Optional[Any]     =	"""ViTImageProcessor"""
    UpperCAmelCase	:							List[Any]     =	"""MgpstrTokenizer"""






    def __init__( self    :   List[Any]    ,			__UpperCAmelCase    :   Union[str, Any]=None    ,			__UpperCAmelCase    :   Dict=None    ,			**__UpperCAmelCase    :   str):
      a						:		Tuple											=      None
      if "feature_extractor" in kwargs:
        warnings.warn(
            "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
            " instead."    ,			__UpperCAmelCase    ,			)
        a						:		List[str]											=      kwargs.pop("feature_extractor")

      a						:		Dict											=      image_processor if image_processor is not None else feature_extractor
      if image_processor is None:
        raise ValueError("You need to specify an `image_processor`.")
      if tokenizer is None:
        raise ValueError("You need to specify a `tokenizer`.")

      a						:		Union[str, Any]											=      tokenizer
      a						:		int											=      AutoTokenizer.from_pretrained("gpt2")
      a						:		str											=      AutoTokenizer.from_pretrained("bert-base-uncased")

      super().__init__(__UpperCAmelCase    ,			__UpperCAmelCase)






    def __call__( self    :   str    ,			__UpperCAmelCase    :   Dict=None    ,			__UpperCAmelCase    :   Tuple=None    ,			__UpperCAmelCase    :   Union[str, Any]=None    ,			**__UpperCAmelCase    :   int):
      if images is None and text is None:
        raise ValueError("You need to specify either an `images` or `text` input to process.")

      if images is not None:
        a						:		List[str]											=      self.image_processor(__UpperCAmelCase    ,			return_tensors=__UpperCAmelCase    ,			**__UpperCAmelCase)
      if text is not None:
        a						:		Optional[Any]											=      self.char_tokenizer(__UpperCAmelCase    ,			return_tensors=__UpperCAmelCase    ,			**__UpperCAmelCase)

      if text is None:
        return inputs
      elif images is None:
        return encodings
      else:
        a						:		Any											=      encodings["input_ids"]
        return inputs






    def      __snake_case							( self    :   List[Any]    ,			__UpperCAmelCase    :   List[str]):
      a							,		a							,		a						:		Tuple											=      sequences
      a						:		Optional[int]											=      char_preds.size(0)

      a							,		a						:		Dict											=      self._decode_helper(__UpperCAmelCase    ,			"char")
      a							,		a						:		Dict											=      self._decode_helper(__UpperCAmelCase    ,			"bpe")
      a							,		a						:		Union[str, Any]											=      self._decode_helper(__UpperCAmelCase    ,			"wp")

      a						:		Any											=      []
      a						:		Union[str, Any]											=      []
      for i in range(__UpperCAmelCase):
        a						:		Any											=      [char_scores[i], bpe_scores[i], wp_scores[i]]
        a						:		Optional[Any]											=      [char_strs[i], bpe_strs[i], wp_strs[i]]
        a						:		List[str]											=      scores.index(max(__UpperCAmelCase))
        final_strs.append(strs[max_score_index])
        final_scores.append(scores[max_score_index])

      a						:		Dict											=      {}
      a						:		List[str]											=      final_strs
      a						:		str											=      final_scores
      a						:		int											=      char_strs
      a						:		int											=      bpe_strs
      a						:		Tuple											=      wp_strs
      return out






    def      __snake_case							( self    :   Any    ,			__UpperCAmelCase    :   List[str]    ,			__UpperCAmelCase    :   List[str]):
      if format == DecodeType.CHARACTER:
        a						:		int											=      self.char_decode
        a						:		int											=      1
        a						:		Dict											=      "[s]"
      elif format == DecodeType.BPE:
        a						:		List[str]											=      self.bpe_decode
        a						:		List[str]											=      2
        a						:		int											=      "#"
      elif format == DecodeType.WORDPIECE:
        a						:		Union[str, Any]											=      self.wp_decode
        a						:		List[str]											=      102
        a						:		int											=      "[SEP]"
      else:
        raise ValueError(f'''Format {format} is not supported.''')

      a							,		a						:		str											=      [], []
      a						:		Optional[int]											=      pred_logits.size(0)
      a						:		List[str]											=      pred_logits.size(1)
      a							,		a						:		Tuple											=      pred_logits.topk(1    ,			dim=-1    ,			largest=__UpperCAmelCase    ,			sorted=__UpperCAmelCase)
      a						:		List[str]											=      preds_index.view(-1    ,			__UpperCAmelCase)[:, 1:]
      a						:		Any											=      decoder(__UpperCAmelCase)
      a							,		a						:		Union[str, Any]											=      torch.nn.functional.softmax(__UpperCAmelCase    ,			dim=2).max(dim=2)
      a						:		Union[str, Any]											=      preds_max_prob[:, 1:]

      for index in range(__UpperCAmelCase):
        a						:		str											=      preds_str[index].find(__UpperCAmelCase)
        a						:		Optional[Any]											=      preds_str[index][:pred_eos]
        a						:		Optional[int]											=      preds_index[index].cpu().tolist()
        a						:		Optional[int]											=      pred_index.index(__UpperCAmelCase) if eos_token in pred_index else -1
        a						:		List[str]											=      preds_max_prob[index][: pred_eos_index + 1]
        a						:		int											=      pred_max_prob.cumprod(dim=0)[-1] if pred_max_prob.nelement() != 0 else 0.0
        dec_strs.append(__UpperCAmelCase)
        conf_scores.append(__UpperCAmelCase)

      return dec_strs, conf_scores






    def      __snake_case							( self    :   Optional[int]    ,			__UpperCAmelCase    :   Any):
      a						:		Dict											=      [seq.replace(" "    ,			"") for seq in self.char_tokenizer.batch_decode(__UpperCAmelCase)]
      return decode_strs






    def      __snake_case							( self    :   Optional[int]    ,			__UpperCAmelCase    :   List[str]):
      return self.bpe_tokenizer.batch_decode(__UpperCAmelCase)






    def      __snake_case							( self    :   Union[str, Any]    ,			__UpperCAmelCase    :   int):
      a						:		Any											=      [seq.replace(" "    ,			"") for seq in self.wp_tokenizer.batch_decode(__UpperCAmelCase)]
      return decode_strs






