




import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class   A(						UpperCamelCase							,				unittest.TestCase       ):



 '''simple docstring'''


 UpperCamelCase						       =						None
 UpperCamelCase						       =						BloomTokenizerFast
 UpperCamelCase						       =						BloomTokenizerFast
 UpperCamelCase						       =						True
 UpperCamelCase						       =						False
 UpperCamelCase						       =						'''tokenizer_file'''
 UpperCamelCase						       =						{'''bos_token''': '''<s>''', '''eos_token''': '''</s>''', '''unk_token''': '''<unk>''', '''pad_token''': '''<pad>'''}

 def     a__			(    self :  Optional[Any]  )       ->     Union[str, Any]:

       """simple docstring"""






       super().setUp()
       lowerCamelCase_							=   BloomTokenizerFast.from_pretrained('bigscience/tokenizer'  )
       tokenizer.save_pretrained(self.tmpdirname  )

 def     a__			(    self :  List[str]						,							**A_ :  List[Any]  )       ->     Tuple:

       """simple docstring"""






       kwargs.update(self.special_tokens_map  )
       return BloomTokenizerFast.from_pretrained(self.tmpdirname						,							**A_  )

 def     a__			(    self :  str  )       ->     int:

       """simple docstring"""






       lowerCamelCase_							=   self.get_rust_tokenizer()

       lowerCamelCase_							=   ['The quick brown fox</s>', 'jumps over the lazy dog</s>']
       lowerCamelCase_							=   [[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

       lowerCamelCase_							=   tokenizer.batch_encode_plus(A_  )['input_ids']
       self.assertListEqual(A_						,							A_  )

       lowerCamelCase_							=   tokenizer.batch_decode(A_  )
       self.assertListEqual(A_						,							A_  )

 def     a__			(    self :  Union[str, Any]						,							A_ :  Dict=6  )       ->     List[str]:

       """simple docstring"""






       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
                   lowerCamelCase_							=   self.rust_tokenizer_class.from_pretrained(A_						,							**A_  )
                   # tokenizer_r.pad_token = None # Hotfixing padding = None
                   # Simple input
                   lowerCamelCase_							=   'This is a simple input'
                   lowerCamelCase_							=   ['This is a simple input 1', 'This is a simple input 2']
                   lowerCamelCase_							=   ('This is a simple input', 'This is a pair')
                   lowerCamelCase_							=   [
                       ('This is a simple input 1', 'This is a simple input 2'),
                       ('This is a simple pair 1', 'This is a simple pair 2'),
                   ]

                   # Simple input tests
                   try:
                         tokenizer_r.encode(A_						,							max_length=A_  )
                         tokenizer_r.encode_plus(A_						,							max_length=A_  )

                         tokenizer_r.batch_encode_plus(A_						,							max_length=A_  )
                         tokenizer_r.encode(A_						,							max_length=A_  )
                         tokenizer_r.batch_encode_plus(A_						,							max_length=A_  )
                   except ValueError:
                         self.fail('Bloom Tokenizer should be able to deal with padding'  )

                   lowerCamelCase_							=   None  # Hotfixing padding = None
                   self.assertRaises(A_						,							tokenizer_r.encode						,							A_						,							max_length=A_						,							padding='max_length'  )

                   # Simple input
                   self.assertRaises(A_						,							tokenizer_r.encode_plus						,							A_						,							max_length=A_						,							padding='max_length'  )

                   # Simple input
                   self.assertRaises(
                       A_						,							tokenizer_r.batch_encode_plus						,							A_						,							max_length=A_						,							padding='max_length'						,							)

                   # Pair input
                   self.assertRaises(A_						,							tokenizer_r.encode						,							A_						,							max_length=A_						,							padding='max_length'  )

                   # Pair input
                   self.assertRaises(A_						,							tokenizer_r.encode_plus						,							A_						,							max_length=A_						,							padding='max_length'  )

                   # Pair input
                   self.assertRaises(
                       A_						,							tokenizer_r.batch_encode_plus						,							A_						,							max_length=A_						,							padding='max_length'						,							)

 def     a__			(    self :  int  )       ->     Optional[int]:

       """simple docstring"""






       lowerCamelCase_							=   self.get_rust_tokenizer()
       lowerCamelCase_							=   load_dataset('xnli'						,							'all_languages'						,							split='test'						,							streaming=A_  )

       lowerCamelCase_							=   next(iter(A_  )  )['premise']  # pick up one data
       lowerCamelCase_							=   list(sample_data.values()  )

       lowerCamelCase_							=   list(map(tokenizer.encode						,							A_  )  )
       lowerCamelCase_							=   [tokenizer.decode(A_						,							clean_up_tokenization_spaces=A_  ) for x in output_tokens]
       self.assertListEqual(A_						,							A_  )





 def     a__			(    self :  Optional[Any]  )       ->     Dict:

       """simple docstring"""






       self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map  )						,							1  )
       self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()  )[0]  )						,							1  )









import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
lowerCamelCase    :		int			   =       logging.getLogger()

lowerCamelCase    :		Tuple			   =       logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)







class   A(						UpperCamelCase       ):



 '''simple docstring'''


 def     a__			(    self :  List[str]						,							A_ :  Optional[Any]  )       ->     int:

       """simple docstring"""






       os.makedirs(A_						,							exist_ok=A_  )
       lowerCamelCase_							=   {'source': 'What is love ?', 'target': 'life'}
       lowerCamelCase_							=   {'train': 12, 'val': 2, 'test': 2}
       for split in ["train", "test", "val"]:
             for field in ["source", "target"]:
                   lowerCamelCase_							=   '\n'.join([contents[field]] * n_lines[split]  )
                   with open(os.path.join(A_						,							f"""{split}.{field}"""  )						,							'w'  ) as f:
                         f.write(A_  )

 def     a__			(    self :  Optional[Any]						,							A_ :  int						,							A_ :  str = "pytorch"  )       ->     Any:

       """simple docstring"""






       lowerCamelCase_							=   self.get_auto_remove_tmp_dir()
       lowerCamelCase_							=   os.path.join(A_						,							'output'  )
       lowerCamelCase_							=   os.path.join(A_						,							'data'  )
       self._create_dummy_data(data_dir=A_  )

       lowerCamelCase_							=   f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

       if gpus > 0:
             testargs.append(f"""--gpus={gpus}"""  )
             if is_apex_available():
                   testargs.append('--fp16'  )
       else:
             testargs.append('--gpus=0'  )
             testargs.append('--distributed_backend=ddp_cpu'  )
             testargs.append('--num_processes=2'  )

       lowerCamelCase_							=   [sys.executable, str(Path(finetune_rag.__file__  ).resolve()  )] + testargs
       execute_subprocess_async(A_						,							env=self.get_env()  )

       lowerCamelCase_							=   os.path.join(A_						,							'metrics.json'  )
       with open(A_  ) as f:
             lowerCamelCase_							=   json.load(A_  )
       return result

 @require_torch_gpu
 def     a__			(    self :  List[Any]  )       ->     Any:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=1  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )

 @require_torch_multi_gpu
 def     a__			(    self :  List[Any]  )       ->     Union[str, Any]:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=2  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )

 @require_torch_gpu
 @require_ray
 def     a__			(    self :  Tuple  )       ->     int:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=1						,							distributed_retriever='ray'  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )





 @require_torch_multi_gpu
 @require_ray
 def     a__			(    self :  Dict  )       ->     Dict:

       """simple docstring"""






       lowerCamelCase_							=   self._run_finetune(gpus=1						,							distributed_retriever='ray'  )
       self.assertGreaterEqual(result['test'][0]['test_avg_em']						,							0.2  )



