





'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {"""vocab_file""": """sentencepiece.bpe.model"""}

SCREAMING_SNAKE_CASE__     =       {
    """vocab_file""": {
        """camembert-base""": """https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model""",
    }
}

SCREAMING_SNAKE_CASE__     =       {
    """camembert-base""": 5_1_2,
}

SCREAMING_SNAKE_CASE__     =       """▁"""






class     a_						(      SCREAMING_SNAKE_CASE_    ):
  lowercase							=				VOCAB_FILES_NAMES
  lowercase							=				PRETRAINED_VOCAB_FILES_MAP
  lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowercase							=				["""input_ids""", """attention_mask"""]

  def __init__(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE="<mask>"      ,		_SCREAMING_SNAKE_CASE=["<s>NOTUSED", "</s>NOTUSED"]      ,		_SCREAMING_SNAKE_CASE = None      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				Any:



        """simple docstring"""






        UpperCamelCase      = AddedToken(snake_case__      ,		lstrip=snake_case__      ,		rstrip=snake_case__				) if isinstance(snake_case__      ,		snake_case__				) else mask_token

        UpperCamelCase      = {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=snake_case__      ,		eos_token=snake_case__      ,		unk_token=snake_case__      ,		sep_token=snake_case__      ,		cls_token=snake_case__      ,		pad_token=snake_case__      ,		mask_token=snake_case__      ,		additional_special_tokens=snake_case__      ,		sp_model_kwargs=self.sp_model_kwargs      ,		**snake_case__      ,		)
        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(str(snake_case__				)				)
        UpperCamelCase      = vocab_file
        # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
        # sentencepiece vocabulary (this is the case for <s> and </s>
        UpperCamelCase      = {'<s>NOTUSED': 0, '<pad>': 1, '</s>NOTUSED': 2, '<unk>': 3}
        UpperCamelCase      = len(self.fairseq_tokens_to_ids				)
        UpperCamelCase      = len(self.sp_model				) + len(self.fairseq_tokens_to_ids				)
        UpperCamelCase      = {v: k for k, v in self.fairseq_tokens_to_ids.items()}

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				List[Any]:



        """simple docstring"""






        if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        UpperCamelCase      = [self.cls_token_id]
        UpperCamelCase      = [self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[str]:



        """simple docstring"""






        if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=snake_case__      ,		token_ids_a=snake_case__      ,		already_has_special_tokens=snake_case__				)

        if token_ids_a is None:
              return [1] + ([0] * len(snake_case__				)) + [1]
        return [1] + ([0] * len(snake_case__				)) + [1, 1] + ([0] * len(snake_case__				)) + [1]

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				List[Any]:



        """simple docstring"""






        UpperCamelCase      = [self.sep_token_id]
        UpperCamelCase      = [self.cls_token_id]

        if token_ids_a is None:
              return len(cls + token_ids_a + sep				) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]

  @property
  def   A__						(				self				)  ->				Dict:



        """simple docstring"""






        return len(self.fairseq_tokens_to_ids				) + len(self.sp_model				)

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = {self.convert_ids_to_tokens(snake_case__				): i for i in range(self.vocab_size				)}
        vocab.update(self.added_tokens_encoder				)
        return vocab

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Optional[Any]:



        """simple docstring"""






        return self.sp_model.encode(snake_case__      ,		out_type=snake_case__				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Dict:



        """simple docstring"""






        if token in self.fairseq_tokens_to_ids:
              return self.fairseq_tokens_to_ids[token]
        elif self.sp_model.PieceToId(snake_case__				) == 0:
              # Convert sentence piece unk token to fairseq unk token index
              return self.unk_token_id
        return self.fairseq_offset + self.sp_model.PieceToId(snake_case__				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				List[str]:



        """simple docstring"""






        if index in self.fairseq_ids_to_tokens:
              return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset				)

  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				int:



        """simple docstring"""






        UpperCamelCase      = []
        UpperCamelCase      = ''
        UpperCamelCase      = False
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    if not prev_is_special:
                          out_string += " "
                    out_string += self.sp_model.decode(snake_case__				) + token
                    UpperCamelCase      = True
                    UpperCamelCase      = []
              else:
                    current_sub_tokens.append(snake_case__				)
                    UpperCamelCase      = False
        out_string += self.sp_model.decode(snake_case__				)
        return out_string.strip()

  def __getstate__(				self				)  ->				List[str]:



        """simple docstring"""






        UpperCamelCase      = self.__dict__.copy()
        UpperCamelCase      = None
        return state

  def __setstate__(				self      ,		_SCREAMING_SNAKE_CASE				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = d

        # for backward compatibility
        if not hasattr(self      ,		"""sp_model_kwargs"""				):
              UpperCamelCase      = {}

        UpperCamelCase      = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
        self.sp_model.Load(self.vocab_file				)


  def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				int:



        """simple docstring"""






        if not os.path.isdir(snake_case__				):
              logger.error(F"Vocabulary path ({save_directory}) should be a directory"				)
              return
        UpperCamelCase      = os.path.join(
            snake_case__      ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

        if os.path.abspath(self.vocab_file				) != os.path.abspath(snake_case__				) and os.path.isfile(self.vocab_file				):
              copyfile(self.vocab_file      ,		snake_case__				)
        elif not os.path.isfile(self.vocab_file				):
              with open(snake_case__      ,		"""wb"""				) as fi:
                    UpperCamelCase      = self.sp_model.serialized_model_proto()
                    fi.write(snake_case__				)

        return (out_vocab_file,)








'''simple docstring'''

import importlib.util
import json
import os
import warnings
from dataclasses import dataclass, field

import torch

from ..training_args import TrainingArguments
from ..utils import cached_property, is_sagemaker_dp_enabled, logging


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)







def    lowercase__			(				)->			Tuple:
      # Get the sagemaker specific mp parameters from smp_options variable.
      UpperCamelCase      = os.getenv("""SM_HP_MP_PARAMETERS"""		,						"""{}"""   )
      try:
            # Parse it and check the field "partitions" is included, it is required for model parallel.
            UpperCamelCase      = json.loads(__UpperCamelCase   )
            if "partitions" not in smp_options:
                  return False
      except json.JSONDecodeError:
            return False

      # Get the sagemaker specific framework parameters from mpi_options variable.
      UpperCamelCase      = os.getenv("""SM_FRAMEWORK_PARAMS"""		,						"""{}"""   )
      try:
            # Parse it and check the field "sagemaker_distributed_dataparallel_enabled".
            UpperCamelCase      = json.loads(__UpperCamelCase   )
            if not mpi_options.get("""sagemaker_mpi_enabled"""		,						__UpperCamelCase   ):
                  return False
      except json.JSONDecodeError:
            return False
      # Lastly, check if the `smdistributed` module is present.
      return importlib.util.find_spec("""smdistributed"""   ) is not None


if is_sagemaker_model_parallel_available():
     import smdistributed.modelparallel.torch as smp

     smp.init()






@dataclass
class     a_						(      lowerCamelCase    ):
  lowercase							=				field(
      default=""""""     ,	metadata={"""help""": """Used by the SageMaker launcher to send mp-specific args. Ignored in SageMakerTrainer"""}     ,	)

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        super().__post_init__()
        warnings.warn(
            """`SageMakerTrainingArguments` is deprecated and will be removed in v5 of Transformers. You can use """
            """`TrainingArguments` instead."""      ,		_SCREAMING_SNAKE_CASE      ,		)

  @cached_property
  def   A__						(				self				)  ->				"torch.device":



        """simple docstring"""






        logger.info("""PyTorch: setting up devices"""				)
        if torch.distributed.is_available() and torch.distributed.is_initialized() and self.local_rank == -1:
              logger.warning(
                  """torch.distributed process group is initialized, but local_rank == -1. """
                  """In order to use Torch DDP, launch your script with `python -m torch.distributed.launch"""				)
        if self.no_cuda:
              UpperCamelCase      = torch.device("""cpu"""				)
              UpperCamelCase      = 0
        elif is_sagemaker_model_parallel_available():
              UpperCamelCase      = smp.local_rank()
              UpperCamelCase      = torch.device("""cuda"""      ,		_SCREAMING_SNAKE_CASE				)
              UpperCamelCase      = 1
        elif is_sagemaker_dp_enabled():
              import smdistributed.dataparallel.torch.torch_smddp  # noqa: F401

              torch.distributed.init_process_group(backend="""smddp"""      ,		timeout=self.ddp_timeout_delta				)
              UpperCamelCase      = int(os.getenv("""SMDATAPARALLEL_LOCAL_RANK"""				)				)
              UpperCamelCase      = torch.device("""cuda"""      ,		self.local_rank				)
              UpperCamelCase      = 1
        elif self.local_rank == -1:
              # if n_gpu is > 1 we'll use nn.DataParallel.
              # If you only want to use a specific subset of GPUs use `CUDA_VISIBLE_DEVICES=0`
              # Explicitly set CUDA to the first (index 0) CUDA device, otherwise `set_device` will
              # trigger an error that a device index is missing. Index 0 takes into account the
              # GPUs available in the environment, so `CUDA_VISIBLE_DEVICES=1,2` with `cuda:0`
              # will use the first GPU in that env, i.e. GPU#1
              UpperCamelCase      = torch.device("""cuda:0""" if torch.cuda.is_available() else """cpu"""				)
              # Sometimes the line in the postinit has not been run before we end up here, so just checking we're not at
              # the default value.
              UpperCamelCase      = torch.cuda.device_count()
        else:
              # Here, we'll use torch.distributed.
              # Initializes the distributed backend which will take care of synchronizing nodes/GPUs
              if not torch.distributed.is_initialized():
                    torch.distributed.init_process_group(backend="""nccl"""      ,		timeout=self.ddp_timeout_delta				)
              UpperCamelCase      = torch.device("""cuda"""      ,		self.local_rank				)
              UpperCamelCase      = 1

        if device.type == "cuda":
              torch.cuda.set_device(_SCREAMING_SNAKE_CASE				)

        return device

  @property
  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        if is_sagemaker_model_parallel_available():
              return smp.dp_size()

        return super().world_size

  @property
  def   A__						(				self				)  ->				Optional[Any]:



        """simple docstring"""






        return not is_sagemaker_model_parallel_available()


  @property
  def   A__						(				self				)  ->				str:



        """simple docstring"""






        return False
