




import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin


class     UpperCamelCase_   (			UpperCAmelCase__		, unittest.TestCase					):

    '''simple docstring'''






    UpperCAmelCase__							  =				CTRLTokenizer
    UpperCAmelCase__							  =				False
    UpperCAmelCase__							  =				False


    def 	SCREAMING_SNAKE_CASE							(      self				:     List[str])						->Union[str, Any]:





           '''simple docstring'''

           super().setUp()

           # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
           A__							=   ['''adapt''', '''re@@''', '''a@@''', '''apt''', '''c@@''', '''t''', '''<unk>''']
           A__							=   dict(zip(UpperCAmelCase__ ,			range(len(UpperCAmelCase__))))
           A__							=   ['''#version: 0.2''', '''a p''', '''ap t</w>''', '''r e''', '''a d''', '''ad apt</w>''', '''''']
           A__							=   {'''unk_token''': '''<unk>'''}

           A__							=   os.path.join(self.tmpdirname ,			VOCAB_FILES_NAMES['''vocab_file'''])
           A__							=   os.path.join(self.tmpdirname ,			VOCAB_FILES_NAMES['''merges_file'''])
           with open(self.vocab_file ,			'''w''' ,			encoding='''utf-8''') as fp:
                  fp.write(json.dumps(UpperCAmelCase__) + '''\n''')
           with open(self.merges_file ,			'''w''' ,			encoding='''utf-8''') as fp:
                  fp.write('''\n'''.join(UpperCAmelCase__))


    def 	SCREAMING_SNAKE_CASE							(      self				:     str ,			**UpperCAmelCase__				:     Union[str, Any])						->str:





           '''simple docstring'''

           kwargs.update(self.special_tokens_map)
           return CTRLTokenizer.from_pretrained(self.tmpdirname ,			**UpperCAmelCase__)


    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[int] ,			UpperCAmelCase__				:     Any)						->List[Any]:





           '''simple docstring'''

           A__							=   '''adapt react readapt apt'''
           A__							=   '''adapt react readapt apt'''
           return input_text, output_text





    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[Any])						->Union[str, Any]:





           '''simple docstring'''

           A__							=   CTRLTokenizer(self.vocab_file ,			self.merges_file ,			**self.special_tokens_map)
           A__							=   '''adapt react readapt apt'''
           A__							=   '''adapt re@@ a@@ c@@ t re@@ adapt apt'''.split()
           A__							=   tokenizer.tokenize(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__ ,			UpperCAmelCase__)

           A__							=   tokens + [tokenizer.unk_token]

           A__							=   [0, 1, 2, 4, 5, 1, 0, 3, 6]
           self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__) ,			UpperCAmelCase__)








import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
  import torch

  from transformers.generation import DisjunctiveConstraint


@require_torch
class     UpperCamelCase_   (			unittest.TestCase					):

    '''simple docstring'''






    def 	SCREAMING_SNAKE_CASE							(      self				:     Dict)						->Tuple:





           '''simple docstring'''

           A__							=   [[1, 2, 4], [1, 2, 3, 4]]
           A__							=   DisjunctiveConstraint(UpperCAmelCase__)
           self.assertTrue(isinstance(dc.token_ids ,			UpperCAmelCase__))

           with self.assertRaises(UpperCAmelCase__):
                  DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]))

           with self.assertRaises(UpperCAmelCase__):
                  DisjunctiveConstraint([torch.LongTensor([1, 2, 4]), torch.LongTensor([1, 2, 3, 4, 5])])


    def 	SCREAMING_SNAKE_CASE							(      self				:     int)						->str:





           '''simple docstring'''

           A__							=   [[1, 2], [1, 2, 3, 4]]

           with self.assertRaises(UpperCAmelCase__):
                  DisjunctiveConstraint(UpperCAmelCase__)  # fails here


    def 	SCREAMING_SNAKE_CASE							(      self				:     Any)						->str:





           '''simple docstring'''

           A__							=   [[1, 2, 3], [1, 2, 4]]

           A__							=   DisjunctiveConstraint(UpperCAmelCase__)

           A__			,							A__			,							A__							=   dc.update(1)
           A__							=   stepped is True and completed is False and reset is False
           self.assertTrue(UpperCAmelCase__)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.current_seq == [1])

           A__			,							A__			,							A__							=   dc.update(2)
           A__							=   stepped is True and completed is False and reset is False
           self.assertTrue(UpperCAmelCase__)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.current_seq == [1, 2])

           A__			,							A__			,							A__							=   dc.update(3)
           A__							=   stepped is True and completed is True and reset is False
           self.assertTrue(UpperCAmelCase__)
           self.assertTrue(dc.completed)  # Completed!
           self.assertTrue(dc.current_seq == [1, 2, 3])





    def 	SCREAMING_SNAKE_CASE							(      self				:     Optional[Any])						->Tuple:





           '''simple docstring'''

           A__							=   [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

           A__							=   DisjunctiveConstraint(UpperCAmelCase__)

           A__			,							A__			,							A__							=   dc.update(1)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.current_seq == [1])

           A__			,							A__			,							A__							=   dc.update(2)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.current_seq == [1, 2])

           A__			,							A__			,							A__							=   dc.update(4)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.current_seq == [1, 2, 4])

           A__			,							A__			,							A__							=   dc.update(5)
           self.assertTrue(dc.completed)  # Completed!
           self.assertTrue(dc.current_seq == [1, 2, 4, 5])

           dc.reset()

           A__			,							A__			,							A__							=   dc.update(1)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.remaining() == 3)
           self.assertTrue(dc.current_seq == [1])

           A__			,							A__			,							A__							=   dc.update(2)
           self.assertTrue(not dc.completed)
           self.assertTrue(dc.remaining() == 2)
           self.assertTrue(dc.current_seq == [1, 2])

           A__			,							A__			,							A__							=   dc.update(5)
           self.assertTrue(dc.completed)  # Completed!
           self.assertTrue(dc.remaining() == 0)
           self.assertTrue(dc.current_seq == [1, 2, 5])


