
from abc import ABC, abstractmethod
from typing import List, Optional

class     _UpperCamelCase			(   __SCREAMING_SNAKE_CASE					):






   '''simple docstring'''

   def __init__(    self	:    Dict ):
       self.test()

   def  lowerCAmelCase__		(    self	:    Optional[Any] ):
       UpperCamelCase_:							List[Any]		=  0
       UpperCamelCase_:							Optional[int]		=  False
       while not completed:
           if counter == 1:
               self.reset()
           UpperCamelCase_:							str		=  self.advance()
           if not self.does_advance(__UpperCAmelCase ):
               raise Exception(
                   """Custom Constraint is not defined correctly. self.does_advance(self.advance()) must be true.""" )

           UpperCamelCase_:							str		=  self.update(__UpperCAmelCase )
           counter += 1

           if counter > 1_0000:
               raise Exception("""update() does not fulfill the constraint.""" )

       if self.remaining() != 0:
           raise Exception("""Custom Constraint is not defined correctly.""" )

   @abstractmethod
   def  lowerCAmelCase__		(    self	:    Dict ):
       raise NotImplementedError(
           f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.''' )

   @abstractmethod
   def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    int ):
       raise NotImplementedError(
           f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.''' )

   @abstractmethod
   def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    Tuple ):
       raise NotImplementedError(
           f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.''' )

   @abstractmethod
   def  lowerCAmelCase__		(    self	:    Any ):
       raise NotImplementedError(
           f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.''' )

   @abstractmethod
   def  lowerCAmelCase__		(    self	:    List[Any] ):
       raise NotImplementedError(
           f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.''' )




   @abstractmethod
   def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    str=False ):
       raise NotImplementedError(
           f'''{self.__class__} is an abstract class. Only classes inheriting this class can be called.''' )




class     _UpperCamelCase			(   __SCREAMING_SNAKE_CASE					):






   '''simple docstring'''

   def __init__(    self	:    Optional[Any]  ,		snake_case_	:    Dict ):
       super(__UpperCAmelCase  ,		self ).__init__()

       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ) or len(__UpperCAmelCase ) == 0:
           raise ValueError(f'''`token_ids` has to be a non-empty list, but is {token_ids}.''' )
       if any((not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ) or token_id < 0) for token_id in token_ids ):
           raise ValueError(f'''Each list in `token_ids` has to be a list of positive integers, but is {token_ids}.''' )

       UpperCamelCase_:							Tuple		=  token_ids

       UpperCamelCase_:							Dict		=  len(self.token_ids )
       UpperCamelCase_:							Dict		=  -1  # the index of the currently fulfilled step
       UpperCamelCase_:							Optional[int]		=  False

   def  lowerCAmelCase__		(    self	:    Optional[int] ):
       if self.completed:
           return None
       return self.token_ids[self.fulfilled_idx + 1]

   def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    List[Any] ):
       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
           raise ValueError(f'''`token_id` has to be an `int`, but is {token_id} of type {type(__UpperCAmelCase )}''' )

       if self.completed:
           return False

       return token_id == self.token_ids[self.fulfilled_idx + 1]

   def  lowerCAmelCase__		(    self	:    str  ,		snake_case_	:    Union[str, Any] ):
       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
           raise ValueError(f'''`token_id` has to be an `int`, but is {token_id} of type {type(__UpperCAmelCase )}''' )

       UpperCamelCase_:							str		=  False
       UpperCamelCase_:							Any		=  False
       UpperCamelCase_:							Optional[Any]		=  False

       if self.does_advance(__UpperCAmelCase ):
           self.fulfilled_idx += 1
           UpperCamelCase_:							Tuple		=  True
           if self.fulfilled_idx == (self.seqlen - 1):
               UpperCamelCase_:							Optional[int]		=  True
           UpperCamelCase_:							Tuple		=  completed
       else:
           # failed to make progress.
           UpperCamelCase_:							Union[str, Any]		=  True
           self.reset()
       return stepped, completed, reset

   def  lowerCAmelCase__		(    self	:    int ):
       UpperCamelCase_:							Union[str, Any]		=  False
       UpperCamelCase_:							int		=  0

   def  lowerCAmelCase__		(    self	:    int ):
       return self.seqlen - (self.fulfilled_idx + 1)




   def  lowerCAmelCase__		(    self	:    int  ,		snake_case_	:    Union[str, Any]=False ):
       UpperCamelCase_:							Tuple		=  PhrasalConstraint(self.token_ids )

       if stateful:
           UpperCamelCase_:							int		=  self.seqlen
           UpperCamelCase_:							List[Any]		=  self.fulfilled_idx
           UpperCamelCase_:							Dict		=  self.completed

       return new_constraint




class     _UpperCamelCase			:






   '''simple docstring'''

   def __init__(    self	:    Dict  ,		snake_case_	:    List[Any]  ,		snake_case_	:    Union[str, Any]=True ):
       UpperCamelCase_:							str		=  max([len(__UpperCAmelCase ) for one in nested_token_ids] )

       UpperCamelCase_:							Optional[int]		=  {}
       for token_ids in nested_token_ids:
           UpperCamelCase_:							str		=  root
           for tidx, token_id in enumerate(__UpperCAmelCase ):
               if token_id not in level:
                   UpperCamelCase_:							int		=  {}

               UpperCamelCase_:							List[Any]		=  level[token_id]

       if no_subsets and self.has_subsets(__UpperCAmelCase  ,		__UpperCAmelCase ):
           raise ValueError(
               """Each list in `nested_token_ids` can't be a complete subset of another list, but is"""
               f''' {nested_token_ids}.''' )

       UpperCamelCase_:							Optional[int]		=  root

   def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    Tuple ):
       UpperCamelCase_:							Dict		=  self.trie

       for current_token in current_seq:
           UpperCamelCase_:							int		=  start[current_token]

       UpperCamelCase_:							Dict		=  list(start.keys() )

       return next_tokens

   def  lowerCAmelCase__		(    self	:    Optional[int]  ,		snake_case_	:    Union[str, Any] ):
       UpperCamelCase_:							Optional[Any]		=  self.next_tokens(__UpperCAmelCase )

       return len(__UpperCAmelCase ) == 0

   def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    Any ):
       UpperCamelCase_:							Dict		=  list(root.values() )
       if len(__UpperCAmelCase ) == 0:
           return 1
       else:
           return sum([self.count_leaves(__UpperCAmelCase ) for nn in next_nodes] )




   def  lowerCAmelCase__		(    self	:    Dict  ,		snake_case_	:    Dict  ,		snake_case_	:    Optional[Any] ):
       UpperCamelCase_:							int		=  self.count_leaves(__UpperCAmelCase )
       return len(__UpperCAmelCase ) != leaf_count




class     _UpperCamelCase			(   __SCREAMING_SNAKE_CASE					):






   '''simple docstring'''

   def __init__(    self	:    str  ,		snake_case_	:    Dict ):
       super(__UpperCAmelCase  ,		self ).__init__()

       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ) or len(__UpperCAmelCase ) == 0:
           raise ValueError(f'''`nested_token_ids` has to be a non-empty list, but is {nested_token_ids}.''' )
       if any(not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ) for token_ids in nested_token_ids ):
           raise ValueError(f'''`nested_token_ids` has to be a list of lists, but is {nested_token_ids}.''' )
       if any(
           any((not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ) or token_id < 0) for token_id in token_ids )
           for token_ids in nested_token_ids ):
           raise ValueError(
               f'''Each list in `nested_token_ids` has to be a list of positive integers, but is {nested_token_ids}.''' )

       UpperCamelCase_:							int		=  DisjunctiveTrie(__UpperCAmelCase )
       UpperCamelCase_:							Tuple		=  nested_token_ids

       UpperCamelCase_:							Tuple		=  self.trie.max_height
       UpperCamelCase_:							List[Any]		=  []
       UpperCamelCase_:							Optional[Any]		=  False

   def  lowerCAmelCase__		(    self	:    Optional[Any] ):
       UpperCamelCase_:							List[str]		=  self.trie.next_tokens(self.current_seq )

       if len(__UpperCAmelCase ) == 0:
           return None
       else:
           return token_list

   def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    List[str] ):
       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
           raise ValueError(f'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(__UpperCAmelCase )}''' )

       UpperCamelCase_:							int		=  self.trie.next_tokens(self.current_seq )

       return token_id in next_tokens

   def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    Any ):
       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
           raise ValueError(f'''`token_id` is supposed to be type `int`, but is {token_id} of type {type(__UpperCAmelCase )}''' )

       UpperCamelCase_:							Dict		=  False
       UpperCamelCase_:							int		=  False
       UpperCamelCase_:							List[Any]		=  False

       if self.does_advance(__UpperCAmelCase ):
           self.current_seq.append(__UpperCAmelCase )
           UpperCamelCase_:							Optional[Any]		=  True
       else:
           UpperCamelCase_:							List[str]		=  True
           self.reset()

       UpperCamelCase_:							List[str]		=  self.trie.reached_leaf(self.current_seq )
       UpperCamelCase_:							int		=  completed

       return stepped, completed, reset

   def  lowerCAmelCase__		(    self	:    str ):
       UpperCamelCase_:							List[str]		=  False
       UpperCamelCase_:							Dict		=  []

   def  lowerCAmelCase__		(    self	:    Optional[Any] ):
       if self.completed:
           # since this can be completed without reaching max height
           return 0
       else:
           return self.seqlen - len(self.current_seq )




   def  lowerCAmelCase__		(    self	:    int  ,		snake_case_	:    str=False ):
       UpperCamelCase_:							Optional[int]		=  DisjunctiveConstraint(self.token_ids )

       if stateful:
           UpperCamelCase_:							Any		=  self.seqlen
           UpperCamelCase_:							Optional[Any]		=  self.current_seq
           UpperCamelCase_:							Tuple		=  self.completed

       return new_constraint




class     _UpperCamelCase			:






   '''simple docstring'''

   def __init__(    self	:    int  ,		snake_case_	:    List[Any] ):
       UpperCamelCase_:							List[str]		=  constraints

       # max # of steps required to fulfill a given constraint
       UpperCamelCase_:							int		=  max([c.seqlen for c in constraints] )
       UpperCamelCase_:							str		=  len(__UpperCAmelCase )
       UpperCamelCase_:							int		=  False

       self.init_state()

   def  lowerCAmelCase__		(    self	:    Optional[int] ):
       UpperCamelCase_:							Optional[Any]		=  []
       UpperCamelCase_:							int		=  None
       UpperCamelCase_:							Any		=  [constraint.copy(stateful=__UpperCAmelCase ) for constraint in self.constraints]

   def  lowerCAmelCase__		(    self	:    List[str] ):
       UpperCamelCase_:							Optional[Any]		=  0
       if self.inprogress_constraint:
           # extra points for having a constraint mid-fulfilled
           add += self.max_seqlen - self.inprogress_constraint.remaining()

       return (len(self.complete_constraints ) * self.max_seqlen) + add

   def  lowerCAmelCase__		(    self	:    Optional[Any] ):
       UpperCamelCase_:							List[str]		=  []
       if self.inprogress_constraint is None:
           for constraint in self.pending_constraints:  # "pending" == "unfulfilled yet"
               UpperCamelCase_:							Union[str, Any]		=  constraint.advance()
               if isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
                   token_list.append(__UpperCAmelCase )
               elif isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
                   token_list.extend(__UpperCAmelCase )
       else:
           UpperCamelCase_:							List[Any]		=  self.inprogress_constraint.advance()
           if isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
               token_list.append(__UpperCAmelCase )
           elif isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
               token_list.extend(__UpperCAmelCase )

       if len(__UpperCAmelCase ) == 0:
           return None
       else:
           return token_list

   def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    int ):
       self.init_state()

       if token_ids is not None:
           for token in token_ids:
               # completes or steps **one** constraint
               UpperCamelCase_:							Optional[int]		=  self.add(__UpperCAmelCase )

               # the entire list of constraints are fulfilled
               if self.completed:
                   break

   def  lowerCAmelCase__		(    self	:    List[Any]  ,		snake_case_	:    Any ):
       if not isinstance(__UpperCAmelCase  ,		__UpperCAmelCase ):
           raise ValueError(f'''`token_id` should be an `int`, but is `{token_id}`.''' )

       UpperCamelCase_:							Dict		=  False, False

       if self.completed:
           UpperCamelCase_:							Optional[int]		=  True
           UpperCamelCase_:							Union[str, Any]		=  False
           return complete, stepped

       if self.inprogress_constraint is not None:
           # In the middle of fulfilling a constraint. If the `token_id` *does* makes an incremental progress to current
           # job, simply update the state

           UpperCamelCase_:							List[str]		=  self.inprogress_constraint.update(__UpperCAmelCase )
           if reset:
               # 1. If the next token breaks the progress, then we must restart.
               #     e.g. constraint = "I love pies" and sequence so far is "I love" but `token_id` == "books".

               #     But that doesn't mean we self.init_state(), since we only reset the state for this particular
               #     constraint, not the full list of constraints.

               self.pending_constraints.append(self.inprogress_constraint.copy(stateful=__UpperCAmelCase ) )
               UpperCamelCase_:							int		=  None

           if complete:
               # 2. If the next token completes the constraint, move it to completed list, set
               #     inprogress to None. If there are no pending constraints either, then this full list of constraints
               #     is complete.

               self.complete_constraints.append(self.inprogress_constraint )
               UpperCamelCase_:							Dict		=  None

               if len(self.pending_constraints ) == 0:
                   # we're done!
                   UpperCamelCase_:							Tuple		=  True

       else:
           # Not in the middle of fulfilling a constraint. So does this `token_id` helps us step towards any of our list
           # of constraints?

           for cidx, pending_constraint in enumerate(self.pending_constraints ):
               if pending_constraint.does_advance(__UpperCAmelCase ):
                   UpperCamelCase_:							str		=  pending_constraint.update(__UpperCAmelCase )

                   if not stepped:
                       raise Exception(
                           """`constraint.update(token_id)` is not yielding incremental progress, """
                           """even though `constraint.does_advance(token_id)` is true.""" )

                   if complete:
                       self.complete_constraints.append(__UpperCAmelCase )
                       UpperCamelCase_:							Tuple		=  None

                   if not complete and stepped:
                       UpperCamelCase_:							Dict		=  pending_constraint

                   if complete or stepped:
                       # If we made any progress at all, then it's at least not a "pending constraint".

                       UpperCamelCase_:							Tuple		=  (
                           self.pending_constraints[:cidx] + self.pending_constraints[cidx + 1 :]
                       )

                       if len(self.pending_constraints ) == 0 and self.inprogress_constraint is None:
                           # If there's no longer any pending after this and no inprogress either, then we must be
                           # complete.

                           UpperCamelCase_:							Union[str, Any]		=  True

                       break  # prevent accidentally stepping through multiple constraints with just one token.

       return complete, stepped




   def  lowerCAmelCase__		(    self	:    Optional[Any]  ,		snake_case_	:    Tuple=True ):
       UpperCamelCase_:							Any		=  ConstraintListState(self.constraints )  # we actually never though self.constraints objects
       # throughout this process. So it's at initialization state.

       if stateful:
           UpperCamelCase_:							Dict		=  [
               constraint.copy(stateful=__UpperCAmelCase ) for constraint in self.complete_constraints
           ]
           if self.inprogress_constraint is not None:
               UpperCamelCase_:							Dict		=  self.inprogress_constraint.copy(stateful=__UpperCAmelCase )
           UpperCamelCase_:							Tuple		=  [constraint.copy() for constraint in self.pending_constraints]

       return new_state








import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mvp import MvpTokenizer


lowerCamelCase_  :	Any 				=					logging.get_logger(__name__)


lowerCamelCase_  :	Optional[int] 				=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all MVP models at https://huggingface.co/models?filter=mvp
lowerCamelCase_  :	str 				=					{
    """vocab_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/vocab.json""",
    },
    """added_tokens.json""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/added_tokens.json""",
    },
    """merges_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """RUCAIBox/mvp""": """https://huggingface.co/RUCAIBox/mvp/resolve/main/tokenizer.json""",
    },
}

lowerCamelCase_  :	Union[str, Any] 				=					{
    """RUCAIBox/mvp""": 10_24,
}

class     _UpperCamelCase			(   _A					):






			'''simple docstring'''

			__UpperCamelCase :  List[Any]    =	VOCAB_FILES_NAMES
			__UpperCamelCase :  int    =	PRETRAINED_VOCAB_FILES_MAP
			__UpperCamelCase :  Any    =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			__UpperCamelCase :  Any    =	["""input_ids""", """attention_mask"""]
			__UpperCamelCase :  int    =	MvpTokenizer

			def __init__(    self	:    Union[str, Any]  ,		snake_case_	:    Any=None  ,		snake_case_	:    Any=None  ,		snake_case_	:    Union[str, Any]=None  ,		snake_case_	:    Tuple="replace"  ,		snake_case_	:    Dict="<s>"  ,		snake_case_	:    Dict="</s>"  ,		snake_case_	:    Tuple="</s>"  ,		snake_case_	:    int="<s>"  ,		snake_case_	:    Tuple="<unk>"  ,		snake_case_	:    Optional[int]="<pad>"  ,		snake_case_	:    Any="<mask>"  ,		snake_case_	:    Union[str, Any]=False  ,		snake_case_	:    Optional[int]=True  ,		**snake_case_	:    Union[str, Any]  ,		):
							super().__init__(
							    snake_case_  ,		snake_case_  ,		tokenizer_file=snake_case_  ,		errors=snake_case_  ,		bos_token=snake_case_  ,		eos_token=snake_case_  ,		sep_token=snake_case_  ,		cls_token=snake_case_  ,		unk_token=snake_case_  ,		pad_token=snake_case_  ,		mask_token=snake_case_  ,		add_prefix_space=snake_case_  ,		trim_offsets=snake_case_  ,		**snake_case_  ,		)

							UpperCamelCase_:							Any		=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
							if pre_tok_state.get("""add_prefix_space"""  ,		snake_case_ ) != add_prefix_space:
											UpperCamelCase_:							str		=  getattr(snake_case_  ,		pre_tok_state.pop("""type""" ) )
											UpperCamelCase_:							int		=  add_prefix_space
											UpperCamelCase_:							int		=  pre_tok_class(**snake_case_ )

							UpperCamelCase_:							int		=  add_prefix_space

							# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
							UpperCamelCase_:							Tuple		=  """post_processor"""
							UpperCamelCase_:							Optional[int]		=  getattr(self.backend_tokenizer  ,		snake_case_  ,		snake_case_ )
							if tokenizer_component_instance:
											UpperCamelCase_:							Optional[int]		=  json.loads(tokenizer_component_instance.__getstate__() )

											# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
											if "sep" in state:
															UpperCamelCase_:							Union[str, Any]		=  tuple(state["""sep"""] )
											if "cls" in state:
															UpperCamelCase_:							List[Any]		=  tuple(state["""cls"""] )

											UpperCamelCase_:							List[str]		=  False

											if state.get("""add_prefix_space"""  ,		snake_case_ ) != add_prefix_space:
															UpperCamelCase_:							str		=  add_prefix_space
															UpperCamelCase_:							Union[str, Any]		=  True

											if state.get("""trim_offsets"""  ,		snake_case_ ) != trim_offsets:
															UpperCamelCase_:							Optional[int]		=  trim_offsets
															UpperCamelCase_:							Any		=  True

											if changes_to_apply:
															UpperCamelCase_:							Optional[int]		=  getattr(snake_case_  ,		state.pop("""type""" ) )
															UpperCamelCase_:							Union[str, Any]		=  component_class(**snake_case_ )
															setattr(self.backend_tokenizer  ,		snake_case_  ,		snake_case_ )

			@property
			def  lowerCAmelCase__		(    self	:    Union[str, Any] ):
							if self._mask_token is None:
											if self.verbose:
															logger.error("""Using mask_token, but it is not set yet.""" )
											return None
							return str(self._mask_token )

			@mask_token.setter
			def  lowerCAmelCase__		(    self	:    Union[str, Any]  ,		snake_case_	:    Dict ):
							UpperCamelCase_:							List[str]		=  AddedToken(snake_case_  ,		lstrip=snake_case_  ,		rstrip=snake_case_ ) if isinstance(snake_case_  ,		snake_case_ ) else value
							UpperCamelCase_:							Optional[int]		=  value

			def  lowerCAmelCase__		(    self	:    Optional[int]  ,		*snake_case_	:    Dict  ,		**snake_case_	:    Optional[int] ):
							UpperCamelCase_:							Optional[int]		=  kwargs.get("""is_split_into_words"""  ,		snake_case_ )

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    """to use it with pretokenized inputs.""" )

							return super()._batch_encode_plus(*snake_case_  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Union[str, Any]  ,		*snake_case_	:    Dict  ,		**snake_case_	:    Tuple ):
							UpperCamelCase_:							Any		=  kwargs.get("""is_split_into_words"""  ,		snake_case_ )

							if is_split_into_words and not self.add_prefix_space:
											raise ValueError(
											    f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    """to use it with pretokenized inputs.""" )

							return super()._encode_plus(*snake_case_  ,		**snake_case_ )

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    str  ,		snake_case_	:    Optional[str] = None ):
							UpperCamelCase_:							Optional[int]		=  self._tokenizer.model.save(snake_case_  ,		name=snake_case_ )
							return tuple(snake_case_ )

			def  lowerCAmelCase__		(    self	:    Tuple  ,		snake_case_	:    Union[str, Any]  ,		snake_case_	:    Any=None ):
							UpperCamelCase_:							str		=  [self.bos_token_id] + token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
											return output

							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




			def  lowerCAmelCase__		(    self	:    Any  ,		snake_case_	:    List[int]  ,		snake_case_	:    Optional[List[int]] = None ):
							UpperCamelCase_:							Tuple		=  [self.sep_token_id]
							UpperCamelCase_:							List[str]		=  [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






