





'''simple docstring'''



import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE   =      logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE   =      {'''vocab_file''': '''sentencepiece.bpe.model'''}

_SCREAMING_SNAKE_CASE   =      {
    '''vocab_file''': {
        '''camembert-base''': '''https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model''',
    }
}

_SCREAMING_SNAKE_CASE   =      {
    '''camembert-base''': 5_1_2,
}

_SCREAMING_SNAKE_CASE   =      '''▁'''






class 							__lowercase    ( lowerCAmelCase__  ):







	'''simple docstring'''

	a      :			Optional[Any]     =    VOCAB_FILES_NAMES
	a      :			Tuple     =    PRETRAINED_VOCAB_FILES_MAP
	a      :			Optional[Any]     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a      :			Tuple     =    ["input_ids", "attention_mask"]


	def __init__(self       ,_lowerCamelCase       ,_lowerCamelCase="<s>"       ,_lowerCamelCase="</s>"       ,_lowerCamelCase="</s>"       ,_lowerCamelCase="<s>"       ,_lowerCamelCase="<unk>"       ,_lowerCamelCase="<pad>"       ,_lowerCamelCase="<mask>"       ,_lowerCamelCase=["<s>NOTUSED", "</s>NOTUSED"]       ,_lowerCamelCase = None       ,**_lowerCamelCase       ,)							->							None:







				'''simple docstring'''





				__lowercase			  =							AddedToken(_lowerCamelCase       ,lstrip=_lowerCamelCase       ,rstrip=_lowerCamelCase	) if isinstance(_lowerCamelCase       ,_lowerCamelCase	) else mask_token

				__lowercase			  =							{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=_lowerCamelCase       ,eos_token=_lowerCamelCase       ,unk_token=_lowerCamelCase       ,sep_token=_lowerCamelCase       ,cls_token=_lowerCamelCase       ,pad_token=_lowerCamelCase       ,mask_token=_lowerCamelCase       ,additional_special_tokens=_lowerCamelCase       ,sp_model_kwargs=self.sp_model_kwargs       ,**_lowerCamelCase       ,)
				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(str(_lowerCamelCase	)	)
				__lowercase			  =							vocab_file
				# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
				# sentencepiece vocabulary (this is the case for <s> and </s>
				__lowercase			  =							{'''<s>NOTUSED''': 0, '''<pad>''': 1, '''</s>NOTUSED''': 2, '''<unk>''': 3}
				__lowercase			  =							len(self.fairseq_tokens_to_ids	)
				__lowercase			  =							len(self.sp_model	) + len(self.fairseq_tokens_to_ids	)
				__lowercase			  =							{v: k for k, v in self.fairseq_tokens_to_ids.items()}


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							List[int]:







				'''simple docstring'''





				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				__lowercase			  =							[self.cls_token_id]
				__lowercase			  =							[self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None       ,_lowerCamelCase = False	)							->							List[int]:







				'''simple docstring'''





				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_lowerCamelCase       ,token_ids_a=_lowerCamelCase       ,already_has_special_tokens=_lowerCamelCase	)

				if token_ids_a is None:
							return [1] + ([0] * len(_lowerCamelCase	)) + [1]
				return [1] + ([0] * len(_lowerCamelCase	)) + [1, 1] + ([0] * len(_lowerCamelCase	)) + [1]


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							List[int]:







				'''simple docstring'''





				__lowercase			  =							[self.sep_token_id]
				__lowercase			  =							[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep	) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


	@property
	def   _UpperCAmelCase       (self	)							->							str:







				'''simple docstring'''





				return len(self.fairseq_tokens_to_ids	) + len(self.sp_model	)


	def   _UpperCAmelCase       (self	)							->							Dict:







				'''simple docstring'''





				__lowercase			  =							{self.convert_ids_to_tokens(_lowerCamelCase	): i for i in range(self.vocab_size	)}
				vocab.update(self.added_tokens_encoder	)
				return vocab


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							List[str]:







				'''simple docstring'''





				return self.sp_model.encode(_lowerCamelCase       ,out_type=_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				elif self.sp_model.PieceToId(_lowerCamelCase	) == 0:
							# Convert sentence piece unk token to fairseq unk token index
							return self.unk_token_id
				return self.fairseq_offset + self.sp_model.PieceToId(_lowerCamelCase	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Optional[int]:







				'''simple docstring'''





				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase	)							->							Tuple:







				'''simple docstring'''





				__lowercase			  =							[]
				__lowercase			  =							''''''
				__lowercase			  =							False
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										if not prev_is_special:
													out_string += " "
										out_string += self.sp_model.decode(_lowerCamelCase	) + token
										__lowercase			  =							True
										__lowercase			  =							[]
							else:
										current_sub_tokens.append(_lowerCamelCase	)
										__lowercase			  =							False
				out_string += self.sp_model.decode(_lowerCamelCase	)
				return out_string.strip()


	def __getstate__(self	)							->							str:







				'''simple docstring'''





				__lowercase			  =							self.__dict__.copy()
				__lowercase			  =							None
				return state


	def __setstate__(self       ,_lowerCamelCase	)							->							List[str]:







				'''simple docstring'''





				__lowercase			  =							d

				# for backward compatibility
				if not hasattr(self       ,'''sp_model_kwargs'''	):
							__lowercase			  =							{}

				__lowercase			  =							spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(self.vocab_file	)


	def   _UpperCAmelCase       (self       ,_lowerCamelCase       ,_lowerCamelCase = None	)							->							Tuple[str]:







				'''simple docstring'''





				if not os.path.isdir(_lowerCamelCase	):
							logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
							return
				__lowercase			  =							os.path.join(
				    _lowerCamelCase       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

				if os.path.abspath(self.vocab_file	) != os.path.abspath(_lowerCamelCase	) and os.path.isfile(self.vocab_file	):
							copyfile(self.vocab_file       ,_lowerCamelCase	)
				elif not os.path.isfile(self.vocab_file	):
							with open(_lowerCamelCase       ,'''wb'''	) as fi:
										__lowercase			  =							self.sp_model.serialized_model_proto()
										fi.write(_lowerCamelCase	)

				return (out_vocab_file,)












'''simple docstring'''



from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_SCREAMING_SNAKE_CASE   =      {'''configuration_focalnet''': ['''FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''FocalNetConfig''']}


try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_SCREAMING_SNAKE_CASE   =      [
	    '''FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''FocalNetForImageClassification''',
	    '''FocalNetForMaskedImageModeling''',
	    '''FocalNetBackbone''',
	    '''FocalNetModel''',
	    '''FocalNetPreTrainedModel''',
	]

if TYPE_CHECKING:
	from .configuration_focalnet import FOCALNET_PRETRAINED_CONFIG_ARCHIVE_MAP, FocalNetConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_focalnet import (
		    FOCALNET_PRETRAINED_MODEL_ARCHIVE_LIST,
		    FocalNetBackbone,
		    FocalNetForImageClassification,
		    FocalNetForMaskedImageModeling,
		    FocalNetModel,
		    FocalNetPreTrainedModel,
		)

else:
	import sys

	_SCREAMING_SNAKE_CASE   =      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





