




'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_snake_case				:				Tuple       							=		logging.get_logger(__name__)

_snake_case				:				List[Any]       							=		'▁'

_snake_case				:				List[str]       							=		{'vocab_file': 'sentencepiece.bpe.model'}

_snake_case				:				List[str]       							=		{
    'vocab_file': {
        'xlm-roberta-base': 'https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large': 'https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model',
        'xlm-roberta-large-finetuned-conll02-dutch': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll02-spanish': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-english': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'
        ),
        'xlm-roberta-large-finetuned-conll03-german': (
            'https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'
        ),
    }
}

_snake_case				:				List[str]       							=		{
    'xlm-roberta-base': 512,
    'xlm-roberta-large': 512,
    'xlm-roberta-large-finetuned-conll02-dutch': 512,
    'xlm-roberta-large-finetuned-conll02-spanish': 512,
    'xlm-roberta-large-finetuned-conll03-english': 512,
    'xlm-roberta-large-finetuned-conll03-german': 512,
}

class       A   (		_a   ):
							lowercase_          =    VOCAB_FILES_NAMES
							lowercase_          =    PRETRAINED_VOCAB_FILES_MAP
							lowercase_          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_          =    ['input_ids', 'attention_mask']







							def __init__(  self	:  List[Any]							,      lowerCAmelCase_	:  Optional[Any]							,      lowerCAmelCase_	:  Dict="<s>"							,      lowerCAmelCase_	:  List[str]="</s>"							,      lowerCAmelCase_	:  Union[str, Any]="</s>"							,      lowerCAmelCase_	:  Union[str, Any]="<s>"							,      lowerCAmelCase_	:  Dict="<unk>"							,      lowerCAmelCase_	:  Tuple="<pad>"							,      lowerCAmelCase_	:  Optional[Any]="<mask>"							,      lowerCAmelCase_	:  Optional[Dict[str, Any]] = None							,      **lowerCAmelCase_	:  List[Any]							,      )      ->							None:


														"""simple docstring"""


														_a     					=    AddedToken(lowerCAmelCase_							,      lstrip=lowerCAmelCase_							,      rstrip=lowerCAmelCase_ ) if isinstance(lowerCAmelCase_							,      lowerCAmelCase_ ) else mask_token

														_a     					=    {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=lowerCAmelCase_							,      eos_token=lowerCAmelCase_							,      unk_token=lowerCAmelCase_							,      sep_token=lowerCAmelCase_							,      cls_token=lowerCAmelCase_							,      pad_token=lowerCAmelCase_							,      mask_token=lowerCAmelCase_							,      sp_model_kwargs=self.sp_model_kwargs							,      **lowerCAmelCase_							,      )

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.Load(str(lowerCAmelCase_ ) )
														_a     					=    vocab_file

														# Original fairseq vocab and spm vocab must be "aligned":
														# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
														# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
														# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
														# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

														# Mimic fairseq token-to-id alignment for the first 4 token
														_a     					=    {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

														# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
														_a     					=    1

														_a     					=    len(self.sp_model ) + self.fairseq_offset
														_a     					=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}







							def __getstate__(  self	:  Optional[Any] )      ->							Dict:


														"""simple docstring"""


														_a     					=    self.__dict__.copy()
														_a     					=    None
														_a     					=    self.sp_model.serialized_model_proto()
														return state







							def __setstate__(  self	:  Union[str, Any]							,      lowerCAmelCase_	:  str )      ->							Tuple:


														"""simple docstring"""


														_a     					=    d

														# for backward compatibility
														if not hasattr(self							,      '''sp_model_kwargs''' ):
																					_a     					=    {}

														_a     					=    spm.SentencePieceProcessor(**self.sp_model_kwargs )
														self.sp_model.LoadFromSerializedProto(self.sp_model_proto )







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_a     					=    [self.cls_token_id]
														_a     					=    [self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep







							def        __lowerCAmelCase      (  self	:  Union[str, Any]							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None							,      lowerCAmelCase_	:  bool = False )      ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,      token_ids_a=lowerCAmelCase_							,      already_has_special_tokens=lowerCAmelCase_ )

														if token_ids_a is None:
																					return [1] + ([0] * len(lowerCAmelCase_ )) + [1]
														return [1] + ([0] * len(lowerCAmelCase_ )) + [1, 1] + ([0] * len(lowerCAmelCase_ )) + [1]







							def        __lowerCAmelCase      (  self	:  Any							,      lowerCAmelCase_	:  List[int]							,      lowerCAmelCase_	:  Optional[List[int]] = None )      ->							List[int]:


														"""simple docstring"""


														_a     					=    [self.sep_token_id]
														_a     					=    [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]







							@property
							def        __lowerCAmelCase      (  self	:  Optional[Any] )      ->							List[Any]:


														"""simple docstring"""


														return len(self.sp_model ) + self.fairseq_offset + 1  # Add the <mask> token







							def        __lowerCAmelCase      (  self	:  List[Any] )      ->							Dict:


														"""simple docstring"""


														_a     					=    {self.convert_ids_to_tokens(lowerCAmelCase_ ): i for i in range(self.vocab_size )}
														vocab.update(self.added_tokens_encoder )
														return vocab







							def        __lowerCAmelCase      (  self	:  str							,      lowerCAmelCase_	:  str )      ->							List[str]:


														"""simple docstring"""


														return self.sp_model.encode(lowerCAmelCase_							,      out_type=lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  List[str] )      ->							Optional[int]:


														"""simple docstring"""


														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														_a     					=    self.sp_model.PieceToId(lowerCAmelCase_ )

														# Need to return unknown token if the SP model returned 0
														return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







							def        __lowerCAmelCase      (  self	:  Tuple							,      lowerCAmelCase_	:  int )      ->							int:


														"""simple docstring"""


														if index in self.fairseq_ids_to_tokens:
																					return self.fairseq_ids_to_tokens[index]
														return self.sp_model.IdToPiece(index - self.fairseq_offset )







							def        __lowerCAmelCase      (  self	:  int							,      lowerCAmelCase_	:  Union[str, Any] )      ->							Union[str, Any]:


														"""simple docstring"""


														_a     					=    ''''''.join(lowerCAmelCase_ ).replace(lowerCAmelCase_							,      ''' ''' ).strip()
														return out_string







							def        __lowerCAmelCase      (  self	:  Optional[Any]							,      lowerCAmelCase_	:  str							,      lowerCAmelCase_	:  Optional[str] = None )      ->							Tuple[str]:


														"""simple docstring"""


														if not os.path.isdir(lowerCAmelCase_ ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory' )
																					return
														_a     					=    os.path.join(
														    lowerCAmelCase_							,      (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

														if os.path.abspath(self.vocab_file ) != os.path.abspath(lowerCAmelCase_ ) and os.path.isfile(self.vocab_file ):
																					copyfile(self.vocab_file							,      lowerCAmelCase_ )
														elif not os.path.isfile(self.vocab_file ):
																					with open(lowerCAmelCase_							,      '''wb''' ) as fi:
																												_a     					=    self.sp_model.serialized_model_proto()
																												fi.write(lowerCAmelCase_ )

														return (out_vocab_file,)







'''simple docstring'''


import unittest

from transformers import TrOCRConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers.models.trocr.modeling_trocr import TrOCRDecoder, TrOCRForCausalLM

@require_torch
class       A   :







							def __init__(  self	:  Union[str, Any]							,      lowerCAmelCase_	:  List[Any]							,      lowerCAmelCase_	:  Union[str, Any]=99							,      lowerCAmelCase_	:  Optional[int]=13							,      lowerCAmelCase_	:  Tuple=16							,      lowerCAmelCase_	:  Optional[int]=7							,      lowerCAmelCase_	:  Optional[Any]=True							,      lowerCAmelCase_	:  Union[str, Any]=True							,      lowerCAmelCase_	:  Any=True							,      lowerCAmelCase_	:  Union[str, Any]=False							,      lowerCAmelCase_	:  int=True							,      lowerCAmelCase_	:  Tuple=2							,      lowerCAmelCase_	:  Any=32							,      lowerCAmelCase_	:  str=4							,      lowerCAmelCase_	:  Optional[Any]=4							,      lowerCAmelCase_	:  Optional[Any]=30							,      lowerCAmelCase_	:  int=0							,      lowerCAmelCase_	:  Any=1							,      lowerCAmelCase_	:  str=2							,      lowerCAmelCase_	:  Union[str, Any]=None							,      )      ->							Any:


														"""simple docstring"""


														_a     					=    parent
														_a     					=    batch_size
														_a     					=    decoder_seq_length
														# For common tests
														_a     					=    self.decoder_seq_length
														_a     					=    is_training
														_a     					=    use_attention_mask
														_a     					=    use_labels

														_a     					=    vocab_size
														_a     					=    d_model
														_a     					=    d_model
														_a     					=    decoder_layers
														_a     					=    decoder_layers
														_a     					=    decoder_ffn_dim
														_a     					=    decoder_attention_heads
														_a     					=    decoder_attention_heads
														_a     					=    eos_token_id
														_a     					=    bos_token_id
														_a     					=    pad_token_id
														_a     					=    decoder_start_token_id
														_a     					=    use_cache
														_a     					=    max_position_embeddings

														_a     					=    None
														_a     					=    decoder_seq_length
														_a     					=    2
														_a     					=    1







							def        __lowerCAmelCase      (  self	:  str )      ->							str:


														"""simple docstring"""


														_a     					=    ids_tensor([self.batch_size, self.decoder_seq_length]							,      self.vocab_size )

														_a     					=    None
														if self.use_attention_mask:
																					_a     					=    ids_tensor([self.batch_size, self.decoder_seq_length]							,      vocab_size=2 )

														_a     					=    None
														if self.use_labels:
																					_a     					=    ids_tensor([self.batch_size, self.decoder_seq_length]							,      self.vocab_size )

														_a     					=    TrOCRConfig(
														    vocab_size=self.vocab_size							,      d_model=self.d_model							,      decoder_layers=self.decoder_layers							,      decoder_ffn_dim=self.decoder_ffn_dim							,      decoder_attention_heads=self.decoder_attention_heads							,      eos_token_id=self.eos_token_id							,      bos_token_id=self.bos_token_id							,      use_cache=self.use_cache							,      pad_token_id=self.pad_token_id							,      decoder_start_token_id=self.decoder_start_token_id							,      max_position_embeddings=self.max_position_embeddings							,      )

														return (config, input_ids, attention_mask, lm_labels)







							def        __lowerCAmelCase      (  self	:  List[str]							,      lowerCAmelCase_	:  Any							,      lowerCAmelCase_	:  Dict							,      lowerCAmelCase_	:  Optional[Any]							,      lowerCAmelCase_	:  int							,      )      ->							int:


														"""simple docstring"""


														_a     					=    True
														_a     					=    TrOCRDecoder(config=lowerCAmelCase_ ).to(lowerCAmelCase_ ).eval()
														_a     					=    input_ids[:2]

														input_ids[input_ids == 0] += 1
														# first forward pass
														_a     					=    model(lowerCAmelCase_							,      use_cache=lowerCAmelCase_ )
														_a     					=    model(lowerCAmelCase_ )
														_a     					=    model(lowerCAmelCase_							,      use_cache=lowerCAmelCase_ )

														self.parent.assertTrue(len(lowerCAmelCase_ ) == len(lowerCAmelCase_ ) )
														self.parent.assertTrue(len(lowerCAmelCase_ ) == len(lowerCAmelCase_ ) + 1 )

														_a     					=    outputs['''past_key_values''']

														# create hypothetical next token and extent to next_input_ids
														_a     					=    ids_tensor((2, 1)							,      config.vocab_size - 1 ) + 1

														# append to next input_ids and
														_a     					=    torch.cat([input_ids, next_tokens]							,      dim=-1 )

														_a     					=    model(lowerCAmelCase_ )['''last_hidden_state''']
														_a     					=    model(lowerCAmelCase_							,      past_key_values=lowerCAmelCase_ )['''last_hidden_state''']

														# select random slice
														_a     					=    ids_tensor((1,)							,      output_from_past.shape[-1] ).item()
														_a     					=    output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
														_a     					=    output_from_past[:, 0, random_slice_idx].detach()

														# test that outputs are equal for slice
														assert torch.allclose(lowerCAmelCase_							,      lowerCAmelCase_							,      atol=1e-3 )







							def        __lowerCAmelCase      (  self	:  int )      ->							List[str]:


														"""simple docstring"""


														_a     					=    self.prepare_config_and_inputs()
														_a					,							_a					,							_a					,							_a     					=    config_and_inputs

														_a     					=    {'''input_ids''': input_ids, '''attention_mask''': attention_mask}
														return config, inputs_dict



@require_torch
class       A   (		_a					,_a					,_a					,unittest.TestCase   ):
							lowercase_          =    (TrOCRDecoder, TrOCRForCausalLM) if is_torch_available() else ()
							lowercase_          =    (TrOCRForCausalLM,) if is_torch_available() else ()
							lowercase_          =    {'text-generation': TrOCRForCausalLM} if is_torch_available() else {}
							lowercase_          =    True
							lowercase_          =    False







							def        __lowerCAmelCase      (  self	:  List[str] )      ->							str:


														"""simple docstring"""


														_a     					=    TrOCRStandaloneDecoderModelTester(self							,      is_training=lowerCAmelCase_ )
														_a     					=    ConfigTester(self							,      config_class=lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  List[str] )      ->							Any:


														"""simple docstring"""


														pass







							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Dict:


														"""simple docstring"""


														pass







							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Optional[int]:


														"""simple docstring"""


														pass







							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Optional[int]:


														"""simple docstring"""


														self.config_tester.run_common_tests()







							def        __lowerCAmelCase      (  self	:  Tuple )      ->							Optional[int]:


														"""simple docstring"""


														_a     					=    self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_decoder_model_past(*lowerCAmelCase_ )







							def        __lowerCAmelCase      (  self	:  Optional[int] )      ->							str:


														"""simple docstring"""


														return







							@unittest.skip('''The model doesn\'t support left padding''' )  # and it's not used enough to be worth fixing :)
							def        __lowerCAmelCase      (  self	:  Union[str, Any] )      ->							Optional[int]:


														"""simple docstring"""


														pass

