






'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
     import torch


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__     =       {'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE__     =       {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE__     =       {
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}






class     a_						(      __lowercase    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["input_ids", "attention_mask"]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE=False      ,		_SCREAMING_SNAKE_CASE=False      ,		_SCREAMING_SNAKE_CASE=False      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE = None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				None:



        """simple docstring"""






        UpperCamelCase      = {} if sp_model_kwargs is None else sp_model_kwargs

        UpperCamelCase      = kwargs.get("""name_or_path"""				)
        if name_or_path is None:
              logger.warning(
                  """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                  """ you are testing the model, this can safely be ignored"""				)
              UpperCamelCase      = '''None'''

        # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
        UpperCamelCase      = '''<|endoftext|>''' if eos_token is None else eos_token
        UpperCamelCase      = '''<unk>''' if unk_token is None else unk_token
        if "gpt-sw3-7b" in name_or_path:
              UpperCamelCase      = unk_token if pad_token is None else pad_token
              UpperCamelCase      = eos_token if bos_token is None else bos_token
        else:
              UpperCamelCase      = '''<pad>''' if pad_token is None else pad_token
              UpperCamelCase      = '''<s>''' if bos_token is None else bos_token

        super().__init__(
            do_lower_case=_a      ,		remove_space=_a      ,		keep_accents=_a      ,		bos_token=_a      ,		eos_token=_a      ,		unk_token=_a      ,		pad_token=_a      ,		sp_model_kwargs=self.sp_model_kwargs      ,		**_a      ,		)

        UpperCamelCase      = do_lower_case
        UpperCamelCase      = remove_space
        UpperCamelCase      = keep_accents
        UpperCamelCase      = vocab_file

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(_a				)

        # Used for whitespace normalization in input texts
        # fmt : off
        UpperCamelCase      = {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
        # fmt : on

        # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
        UpperCamelCase      = re.compile(
            F"[{''.join(map(_a      ,		list(range(0      ,		9				)				) + list(range(11      ,		32				)				) + list(range(127      ,		160				)				) + [160, 173, 8203]				)				)}]"				)

  def __getstate__(				self				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = self.__dict__.copy()
        UpperCamelCase      = None
        return state

  def __setstate__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Dict:



        """simple docstring"""






        UpperCamelCase      = d

        # for backward compatibility
        if not hasattr(self      ,		"""sp_model_kwargs"""				):
              UpperCamelCase      = {}

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.vocab_file				)

  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def   A__						(				self				)  ->				int:



        """simple docstring"""






        return len(self.sp_model				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        UpperCamelCase      = self.non_printing_characters_re.sub(""""""      ,		_a				)

        # Normalize whitespaces
        UpperCamelCase      = ''''''.join([char if char not in self.whitespaces else """ """ for char in text]				)

        # NFC Unicode normalization
        UpperCamelCase      = unicodedata.normalize("""NFC"""      ,		_a				)
        return text

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = self.preprocess_text(_a				)
        return self.sp_model.encode(_a      ,		out_type=_a				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        return self.sp_model.PieceToId(_a				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        return self.sp_model.IdToPiece(_a				)

  @staticmethod
  def   A__						(				_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        return out_string

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        UpperCamelCase      = []
        UpperCamelCase      = ''''''
        UpperCamelCase      = False
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                    if not prev_is_special:
                          out_string += " "

                    out_string += self.sp_model.decode(_a				) + token
                    UpperCamelCase      = True
                    UpperCamelCase      = []
              else:
                    current_sub_tokens.append(_a				)
                    UpperCamelCase      = False
        out_string += self.sp_model.decode(_a				)

        return out_string

  def   A__						(				self				)  ->				Dict[str, int]:



        """simple docstring"""






        UpperCamelCase      = {self.convert_ids_to_tokens(_a				): i for i in range(self.vocab_size				)}
        vocab.update(self.added_tokens_encoder				)
        return vocab

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



        """simple docstring"""






        if not os.path.isdir(_a				):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
              return
        UpperCamelCase      = os.path.join(
            _a      ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

        if os.path.abspath(self.vocab_file				) != os.path.abspath(_a				) and os.path.isfile(self.vocab_file				):
              copyfile(self.vocab_file      ,		_a				)
        elif not os.path.isfile(self.vocab_file				):
              with open(_a      ,		"""wb"""				) as fi:
                    UpperCamelCase      = self.sp_model.serialized_model_proto()
                    fi.write(_a				)

        return (out_vocab_file,)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = False				)  ->				Union[List[int], List[List[int]], "torch.Tensor"]:



        """simple docstring"""






        if isinstance(_a      ,		_a				):
              UpperCamelCase      = self.preprocess_text(_a				)
              UpperCamelCase      = self.sp_model.encode(_a				)
        else:
              UpperCamelCase      = [self.preprocess_text(_a				) for t in text]
              UpperCamelCase      = self.sp_model.encode(_a				)

        if return_tensors is True or return_tensors == "pt":
              UpperCamelCase      = torch.tensor(_a				)

        return token_ids

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        return self.sp_model.decode(_a				)


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[int]:



        """simple docstring"""






        UpperCamelCase      = [F"User: {text}" if is_user else F"Bot: {text}" for is_user, text in conversation.iter_texts()]
        UpperCamelCase      = (
            F"{self.eos_token}{self.bos_token}" + F"{self.bos_token}".join(_a				) + F"{self.bos_token}Bot:"
        )
        return self.encode(text=_a				)








'''simple docstring'''

import functools
import logging
import os
import sys
import threading
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

import huggingface_hub.utils as hf_hub_utils
from tqdm import auto as tqdm_lib


SCREAMING_SNAKE_CASE__     =       threading.Lock()
SCREAMING_SNAKE_CASE__     =       None

SCREAMING_SNAKE_CASE__     =       {
    'debug': logging.DEBUG,
    'info': logging.INFO,
    'warning': logging.WARNING,
    'error': logging.ERROR,
    'critical': logging.CRITICAL,
}

SCREAMING_SNAKE_CASE__     =       logging.WARNING

SCREAMING_SNAKE_CASE__     =       True







def    lowercase__			(				)->			Optional[int]:
      UpperCamelCase      = os.getenv("""TRANSFORMERS_VERBOSITY"""		,						__UpperCamelCase   )
      if env_level_str:
            if env_level_str in log_levels:
                  return log_levels[env_level_str]
            else:
                  logging.getLogger().warning(
                      F"Unknown option TRANSFORMERS_VERBOSITY={env_level_str}, "
                      F"has to be one of: { ', '.join(log_levels.keys()   ) }"   )
      return _default_log_level







def    lowercase__			(				)->			str:
      return __name__.split("""."""   )[0]







def    lowercase__			(				)->			logging.Logger:
      return logging.getLogger(_get_library_name()   )







def    lowercase__			(				)->			None:
      global _default_handler

      with _lock:
            if _default_handler:
                  # This library has already configured the library root logger.
                  return
            UpperCamelCase      = logging.StreamHandler()  # Set sys.stderr as stream.
            UpperCamelCase      = sys.stderr.flush

            # Apply our default configuration to the library root logger.
            UpperCamelCase      = _get_library_root_logger()
            library_root_logger.addHandler(_default_handler   )
            library_root_logger.setLevel(_get_default_logging_level()   )
            UpperCamelCase      = False







def    lowercase__			(				)->			None:
      global _default_handler

      with _lock:
            if not _default_handler:
                  return

            UpperCamelCase      = _get_library_root_logger()
            library_root_logger.removeHandler(_default_handler   )
            library_root_logger.setLevel(logging.NOTSET   )
            UpperCamelCase      = None







def    lowercase__			(				)->			Tuple:
      return log_levels







def    lowercase__			(				__UpperCamelCase = None   )->			logging.Logger:

      if name is None:
            UpperCamelCase      = _get_library_name()

      _configure_library_root_logger()
      return logging.getLogger(__UpperCamelCase   )







def    lowercase__			(				)->			int:
      _configure_library_root_logger()
      return _get_library_root_logger().getEffectiveLevel()







def    lowercase__			(				__UpperCamelCase   )->			None:
      _configure_library_root_logger()
      _get_library_root_logger().setLevel(__UpperCamelCase   )







def    lowercase__			(				)->			Tuple:
      return set_verbosity(__UpperCamelCase   )







def    lowercase__			(				)->			Union[str, Any]:
      return set_verbosity(__UpperCamelCase   )







def    lowercase__			(				)->			Optional[int]:
      return set_verbosity(__UpperCamelCase   )







def    lowercase__			(				)->			Tuple:
      return set_verbosity(__UpperCamelCase   )







def    lowercase__			(				)->			None:
      _configure_library_root_logger()

      assert _default_handler is not None
      _get_library_root_logger().removeHandler(_default_handler   )







def    lowercase__			(				)->			None:
      _configure_library_root_logger()

      assert _default_handler is not None
      _get_library_root_logger().addHandler(_default_handler   )







def    lowercase__			(				__UpperCamelCase   )->			None:
      _configure_library_root_logger()

      assert handler is not None
      _get_library_root_logger().addHandler(__UpperCamelCase   )







def    lowercase__			(				__UpperCamelCase   )->			None:
      _configure_library_root_logger()

      assert handler is not None and handler not in _get_library_root_logger().handlers
      _get_library_root_logger().removeHandler(__UpperCamelCase   )







def    lowercase__			(				)->			None:
      _configure_library_root_logger()
      UpperCamelCase      = False







def    lowercase__			(				)->			None:
      _configure_library_root_logger()
      UpperCamelCase      = True







def    lowercase__			(				)->			None:
      UpperCamelCase      = _get_library_root_logger().handlers

      for handler in handlers:
            UpperCamelCase      = logging.Formatter("""[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"""   )
            handler.setFormatter(__UpperCamelCase   )







def    lowercase__			(				)->			None:
      UpperCamelCase      = _get_library_root_logger().handlers

      for handler in handlers:
            handler.setFormatter(__UpperCamelCase   )







def    lowercase__			(				self		,						*__UpperCamelCase		,						**__UpperCamelCase   )->			Tuple:
      UpperCamelCase      = os.getenv("""TRANSFORMERS_NO_ADVISORY_WARNINGS"""		,						__UpperCamelCase   )
      if no_advisory_warnings:
            return
      self.warning(*__UpperCamelCase		,						**__UpperCamelCase   )


SCREAMING_SNAKE_CASE__     =       warning_advice







@functools.lru_cache(__UpperCamelCase   )
def    lowercase__			(				self		,						*__UpperCamelCase		,						**__UpperCamelCase   )->			Optional[Any]:
      self.warning(*__UpperCamelCase		,						**__UpperCamelCase   )


SCREAMING_SNAKE_CASE__     =       warning_once






class     a_						:

  def __init__(				self      ,		*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)  ->				Optional[int]:  # pylint: disable=unused-argument



        """simple docstring"""






        UpperCamelCase      = args[0] if args else None

  def __iter__(				self				)  ->				List[Any]:



        """simple docstring"""






        return iter(self._iterator				)

  def __getattr__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[Any]:



        """simple docstring"""






        def empty_fn(*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				):  # pylint: disable=unused-argument
              return

        return empty_fn

  def __enter__(				self				)  ->				Dict:



        """simple docstring"""






        return self


  def __exit__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)  ->				List[Any]:



        """simple docstring"""






        return






class     a_						:

  def __call__(				self      ,		*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)  ->				str:



        """simple docstring"""






        if _tqdm_active:
              return tqdm_lib.tqdm(*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)
        else:
              return EmptyTqdm(*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)

  def   A__						(				self      ,		*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)  ->				Optional[Any]:



        """simple docstring"""






        UpperCamelCase      = None
        if _tqdm_active:
              return tqdm_lib.tqdm.set_lock(*_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE				)


  def   A__						(				self				)  ->				Union[str, Any]:



        """simple docstring"""






        if _tqdm_active:
              return tqdm_lib.tqdm.get_lock()


SCREAMING_SNAKE_CASE__     =       _tqdm_cls()







def    lowercase__			(				)->			bool:
      global _tqdm_active
      return bool(_tqdm_active   )







def    lowercase__			(				)->			Optional[Any]:
      global _tqdm_active
      UpperCamelCase      = True
      hf_hub_utils.enable_progress_bars()







def    lowercase__			(				)->			str:
      global _tqdm_active
      UpperCamelCase      = False
      hf_hub_utils.disable_progress_bars()
