






'''simple docstring'''

from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


SCREAMING_SNAKE_CASE__     =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     =       {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__     =       {
    "vocab_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json"
    },
    "merges_file": {
        "allegro/herbert-base-cased": "https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt"
    },
}

SCREAMING_SNAKE_CASE__     =       {"allegro/herbert-base-cased": 5_1_4}
SCREAMING_SNAKE_CASE__     =       {}






class     a_						(      _SCREAMING_SNAKE_CASE    ):
     lowercase							=				VOCAB_FILES_NAMES
     lowercase							=				PRETRAINED_VOCAB_FILES_MAP
     lowercase							=				PRETRAINED_INIT_CONFIGURATION
     lowercase							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase							=				HerbertTokenizer

     def __init__(				self      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE=None      ,		_SCREAMING_SNAKE_CASE="<s>"      ,		_SCREAMING_SNAKE_CASE="<unk>"      ,		_SCREAMING_SNAKE_CASE="<pad>"      ,		_SCREAMING_SNAKE_CASE="<mask>"      ,		_SCREAMING_SNAKE_CASE="</s>"      ,		**_SCREAMING_SNAKE_CASE      ,		)  ->				Optional[Any]:



           """simple docstring"""






           super().__init__(
               _SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		tokenizer_file=_SCREAMING_SNAKE_CASE      ,		cls_token=_SCREAMING_SNAKE_CASE      ,		unk_token=_SCREAMING_SNAKE_CASE      ,		pad_token=_SCREAMING_SNAKE_CASE      ,		mask_token=_SCREAMING_SNAKE_CASE      ,		sep_token=_SCREAMING_SNAKE_CASE      ,		**_SCREAMING_SNAKE_CASE      ,		)

     def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				List[int]:



           """simple docstring"""






           UpperCamelCase      = [self.cls_token_id]
           UpperCamelCase      = [self.sep_token_id]
           if token_ids_a is None:
                 return cls + token_ids_a + sep

           return cls + token_ids_a + sep + token_ids_a + sep

     def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None      ,		_SCREAMING_SNAKE_CASE = False				)  ->				List[int]:



           """simple docstring"""






           if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=_SCREAMING_SNAKE_CASE      ,		token_ids_a=_SCREAMING_SNAKE_CASE      ,		already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

           if token_ids_a is None:
                 return [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1]
           return [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1]

     def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				List[int]:



           """simple docstring"""






           UpperCamelCase      = [self.sep_token_id]
           UpperCamelCase      = [self.cls_token_id]

           if token_ids_a is None:
                 return len(cls + token_ids_a + sep				) * [0]
           return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]


     def   A__						(				self      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE = None				)  ->				Tuple[str]:



           """simple docstring"""






           UpperCamelCase      = self._tokenizer.model.save(_SCREAMING_SNAKE_CASE      ,		name=_SCREAMING_SNAKE_CASE				)
           return tuple(_SCREAMING_SNAKE_CASE				)








'''simple docstring'''

import unittest

from transformers.utils.backbone_utils import (
    BackboneMixin,
    get_aligned_output_features_output_indices,
    verify_out_features_out_indices,
)






class     a_						(      unittest.TestCase    ):

  def   A__						(				self				)  ->				Tuple:



        """simple docstring"""






        UpperCamelCase      = ["""a""", """b""", """c"""]

        # Defaults to last layer if both are None
        UpperCamelCase  ,UpperCamelCase      = get_aligned_output_features_output_indices(_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		["""c"""]				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		[2]				)

        # Out indices set to match out features
        UpperCamelCase  ,UpperCamelCase      = get_aligned_output_features_output_indices(["""a""", """c"""]      ,		_SCREAMING_SNAKE_CASE      ,		_SCREAMING_SNAKE_CASE				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		["""a""", """c"""]				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		[0, 2]				)

        # Out features set to match out indices
        UpperCamelCase  ,UpperCamelCase      = get_aligned_output_features_output_indices(_SCREAMING_SNAKE_CASE      ,		[0, 2]      ,		_SCREAMING_SNAKE_CASE				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		["""a""", """c"""]				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		[0, 2]				)

        # Out features selected from negative indices
        UpperCamelCase  ,UpperCamelCase      = get_aligned_output_features_output_indices(_SCREAMING_SNAKE_CASE      ,		[-3, -1]      ,		_SCREAMING_SNAKE_CASE				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		["""a""", """c"""]				)
        self.assertEqual(_SCREAMING_SNAKE_CASE      ,		[-3, -1]				)

  def   A__						(				self				)  ->				str:



        """simple docstring"""






        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(["""a""", """b"""]      ,		(0, 1)      ,		_SCREAMING_SNAKE_CASE				)

        # Out features must be a list
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(("""a""", """b""")      ,		(0, 1)      ,		["""a""", """b"""]				)

        # Out features must be a subset of stage names
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(["""a""", """b"""]      ,		(0, 1)      ,		["""a"""]				)

        # Out indices must be a list or tuple
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(_SCREAMING_SNAKE_CASE      ,		0      ,		["""a""", """b"""]				)

        # Out indices must be a subset of stage names
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(_SCREAMING_SNAKE_CASE      ,		(0, 1)      ,		["""a"""]				)

        # Out features and out indices must be the same length
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(["""a""", """b"""]      ,		(0,)      ,		["""a""", """b""", """c"""]				)

        # Out features should match out indices
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(["""a""", """b"""]      ,		(0, 2)      ,		["""a""", """b""", """c"""]				)

        # Out features and out indices should be in order
        with self.assertRaises(_SCREAMING_SNAKE_CASE				):
              verify_out_features_out_indices(["""b""", """a"""]      ,		(0, 1)      ,		["""a""", """b"""]				)

        # Check passes with valid inputs
        verify_out_features_out_indices(["""a""", """b""", """d"""]      ,		(0, 1, -1)      ,		["""a""", """b""", """c""", """d"""]				)


  def   A__						(				self				)  ->				Union[str, Any]:



        """simple docstring"""






        UpperCamelCase      = BackboneMixin()

        UpperCamelCase      = ["""a""", """b""", """c"""]
        UpperCamelCase      = ["""a""", """c"""]
        UpperCamelCase      = [0, 2]

        # Check that the output features and indices are set correctly
        self.assertEqual(backbone.out_features      ,		["""a""", """c"""]				)
        self.assertEqual(backbone.out_indices      ,		[0, 2]				)

        # Check out features and indices are updated correctly
        UpperCamelCase      = ["""a""", """b"""]
        self.assertEqual(backbone.out_features      ,		["""a""", """b"""]				)
        self.assertEqual(backbone.out_indices      ,		[0, 1]				)

        UpperCamelCase      = [-3, -1]
        self.assertEqual(backbone.out_features      ,		["""a""", """c"""]				)
        self.assertEqual(backbone.out_indices      ,		[-3, -1]				)
