
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_						     =					logging.get_logger(__name__)
lowerCamelCase_						     =					{'''vocab_file''': '''spiece.model'''}

lowerCamelCase_						     =					{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

lowerCamelCase_						     =					{
    '''albert-base-v1''': 5_12,
    '''albert-large-v1''': 5_12,
    '''albert-xlarge-v1''': 5_12,
    '''albert-xxlarge-v1''': 5_12,
    '''albert-base-v2''': 5_12,
    '''albert-large-v2''': 5_12,
    '''albert-xlarge-v2''': 5_12,
    '''albert-xxlarge-v2''': 5_12,
}

lowerCamelCase_						     =					'''▁'''




class      __A(    __lowerCamelCase			):

  """simple docstring"""
  SCREAMING_SNAKE_CASE__         =      VOCAB_FILES_NAMES
  SCREAMING_SNAKE_CASE__         =      PRETRAINED_VOCAB_FILES_MAP
  SCREAMING_SNAKE_CASE__         =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







  def __init__(self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_=True	,      SCREAMING_SNAKE_CASE_=True	,      SCREAMING_SNAKE_CASE_=False	,      SCREAMING_SNAKE_CASE_="[CLS]"	,      SCREAMING_SNAKE_CASE_="[SEP]"	,      SCREAMING_SNAKE_CASE_="<unk>"	,      SCREAMING_SNAKE_CASE_="[SEP]"	,      SCREAMING_SNAKE_CASE_="<pad>"	,      SCREAMING_SNAKE_CASE_="[CLS]"	,      SCREAMING_SNAKE_CASE_="[MASK]"	,      SCREAMING_SNAKE_CASE_ = None	,      **SCREAMING_SNAKE_CASE_	,      ):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.
       UpperCamelCase__				=    (
           AddedToken(SCREAMING_SNAKE_CASE_	,      lstrip=SCREAMING_SNAKE_CASE_	,      rstrip=SCREAMING_SNAKE_CASE_	,      normalized=SCREAMING_SNAKE_CASE_						)
           if isinstance(SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_						)
           else mask_token
       )

       UpperCamelCase__				=    {} if sp_model_kwargs is None else sp_model_kwargs

       super().__init__(
           do_lower_case=SCREAMING_SNAKE_CASE_	,      remove_space=SCREAMING_SNAKE_CASE_	,      keep_accents=SCREAMING_SNAKE_CASE_	,      bos_token=SCREAMING_SNAKE_CASE_	,      eos_token=SCREAMING_SNAKE_CASE_	,      unk_token=SCREAMING_SNAKE_CASE_	,      sep_token=SCREAMING_SNAKE_CASE_	,      pad_token=SCREAMING_SNAKE_CASE_	,      cls_token=SCREAMING_SNAKE_CASE_	,      mask_token=SCREAMING_SNAKE_CASE_	,      sp_model_kwargs=self.sp_model_kwargs	,      **SCREAMING_SNAKE_CASE_	,      )

       UpperCamelCase__				=    do_lower_case
       UpperCamelCase__				=    remove_space
       UpperCamelCase__				=    keep_accents
       UpperCamelCase__				=    vocab_file

       UpperCamelCase__				=    spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(SCREAMING_SNAKE_CASE_						)







  @property
  def 	UpperCAmelCase_ (self						):
       return len(self.sp_model						)







  def 	UpperCAmelCase_ (self						):
       UpperCamelCase__				=    {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab







  def __getstate__(self						):
       UpperCamelCase__				=    self.__dict__.copy()
       UpperCamelCase__				=    None
       return state







  def __setstate__(self	,      SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=    d

       # for backward compatibility
       if not hasattr(self	,      """sp_model_kwargs"""						):
            UpperCamelCase__				=    {}

       UpperCamelCase__				=    spm.SentencePieceProcessor(**self.sp_model_kwargs						)
       self.sp_model.Load(self.vocab_file						)







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_						):
       if self.remove_space:
            UpperCamelCase__				=    """ """.join(inputs.strip().split()						)
       else:
            UpperCamelCase__				=    inputs
       UpperCamelCase__				=    outputs.replace("""``"""	,      """\""""						).replace("""''"""	,      """\""""						)

       if not self.keep_accents:
            UpperCamelCase__				=    unicodedata.normalize("""NFKD"""	,      SCREAMING_SNAKE_CASE_						)
            UpperCamelCase__				=    """""".join([c for c in outputs if not unicodedata.combining(SCREAMING_SNAKE_CASE_						)]						)
       if self.do_lower_case:
            UpperCamelCase__				=    outputs.lower()

       return outputs







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=    self.preprocess_text(SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=    self.sp_model.encode(SCREAMING_SNAKE_CASE_	,      out_type=SCREAMING_SNAKE_CASE_						)
       UpperCamelCase__				=    []
       for piece in pieces:
            if len(SCREAMING_SNAKE_CASE_						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
                 UpperCamelCase__				=    self.sp_model.EncodeAsPieces(piece[:-1].replace(SCREAMING_SNAKE_CASE_	,      """"""						)						)
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                      if len(cur_pieces[0]						) == 1:
                           UpperCamelCase__				=    cur_pieces[1:]
                      else:
                           UpperCamelCase__				=    cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]						)
                 new_pieces.extend(SCREAMING_SNAKE_CASE_						)
            else:
                 new_pieces.append(SCREAMING_SNAKE_CASE_						)

       return new_pieces







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_						):
       return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_						):
       return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE_						)







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_						):
       UpperCamelCase__				=    []
       UpperCamelCase__				=    """"""
       UpperCamelCase__				=    False
       for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
                 if not prev_is_special:
                      out_string += " "
                 out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_						) + token
                 UpperCamelCase__				=    True
                 UpperCamelCase__				=    []
            else:
                 current_sub_tokens.append(SCREAMING_SNAKE_CASE_						)
                 UpperCamelCase__				=    False
       out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE_						)
       return out_string.strip()







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None						):
       UpperCamelCase__				=    [self.sep_token_id]
       UpperCamelCase__				=    [self.cls_token_id]
       if token_ids_a is None:
            return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None	,      SCREAMING_SNAKE_CASE_ = False						):

       if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=SCREAMING_SNAKE_CASE_	,      token_ids_a=SCREAMING_SNAKE_CASE_	,      already_has_special_tokens=SCREAMING_SNAKE_CASE_						)

       if token_ids_a is not None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1]
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE_						)) + [1]







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None						):
       UpperCamelCase__				=    [self.sep_token_id]
       UpperCamelCase__				=    [self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







  def 	UpperCAmelCase_ (self	,      SCREAMING_SNAKE_CASE_	,      SCREAMING_SNAKE_CASE_ = None						):
       if not os.path.isdir(SCREAMING_SNAKE_CASE_						):
            logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
            return
       UpperCamelCase__				=    os.path.join(
           SCREAMING_SNAKE_CASE_	,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE_						) and os.path.isfile(self.vocab_file						):
            copyfile(self.vocab_file	,      SCREAMING_SNAKE_CASE_						)
       elif not os.path.isfile(self.vocab_file						):
            with open(SCREAMING_SNAKE_CASE_	,      """wb"""						) as fi:
                 UpperCamelCase__				=    self.sp_model.serialized_model_proto()
                 fi.write(SCREAMING_SNAKE_CASE_						)

       return (out_vocab_file,)







import warnings

from ...utils import logging
from .image_processing_segformer import SegformerImageProcessor


lowerCamelCase_						     =					logging.get_logger(__name__)




class      __A(    __lowerCamelCase			):

  """simple docstring"""







  def __init__(self	,      *SCREAMING_SNAKE_CASE_	,      **SCREAMING_SNAKE_CASE_						):
       warnings.warn(
           """The class SegformerFeatureExtractor is deprecated and will be removed in version 5 of Transformers."""
           """ Please use SegformerImageProcessor instead."""	,      SCREAMING_SNAKE_CASE_	,      )
       super().__init__(*SCREAMING_SNAKE_CASE_	,      **SCREAMING_SNAKE_CASE_						)





