


import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_	   =     get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
       from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowercase_	   =     5_0_0_0_3
lowercase_	   =     5_0_0_0_2


@require_sentencepiece
@require_tokenizers
class 	A_ (	__UpperCamelCase ,				unittest.TestCase   ):




      '''simple docstring'''


      __snake_case 	=       PLBartTokenizer
      __snake_case 	=       None
      __snake_case 	=       False




      def 					_snake_case     (    self:	Any    ):
       super().setUp()

       # We have a SentencePiece fixture for testing
       __lowerCamelCase		:							List[Any]		 =						PLBartTokenizer(a						,				language_codes='base'						,				keep_accents=a    )
       tokenizer.save_pretrained(self.tmpdirname    )




      def 					_snake_case     (    self:	Any    ):
       __lowerCamelCase		:							List[Any]		 =						PLBartTokenizer(a						,				language_codes='base'						,				keep_accents=a    )

       __lowerCamelCase		:							Optional[Any]		 =						tokenizer.tokenize('This is a test'    )
       self.assertListEqual(a						,				['▁This', '▁is', '▁a', '▁t', 'est']    )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(a    )						,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,				)

       __lowerCamelCase		:							str		 =						tokenizer.tokenize('I was born in 92000, and this is falsé.'    )
       self.assertListEqual(
           a						,				[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]						,				)
       __lowerCamelCase		:							List[str]		 =						tokenizer.convert_tokens_to_ids(a    )
       self.assertListEqual(
           a						,				[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]						,				)

       __lowerCamelCase		:							str		 =						tokenizer.convert_ids_to_tokens(a    )
       self.assertListEqual(
           a						,				[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '<unk>',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '<unk>',
               '.',
           ]						,				)

       __lowerCamelCase		:							Dict		 =						tokenizer.vocab_size
       __lowerCamelCase		:							List[str]		 =						[tokenizer.convert_ids_to_tokens(a    ) for x in range(end - 4						,				a    )]

       self.assertListEqual(a						,				['__java__', '__python__', '__en_XX__', '<mask>']    )

       __lowerCamelCase		:							List[str]		 =						'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
       __lowerCamelCase		:							Dict		 =						tokenizer(a    ).input_ids
       self.assertEqual(
           tokenizer.decode(a						,				skip_special_tokens=a						,				clean_up_tokenization_spaces=a    )						,				a						,				)




      def 					_snake_case     (    self:	Any    ):
       __lowerCamelCase		:							Union[str, Any]		 =						PLBartTokenizer(a						,				language_codes='multi'						,				keep_accents=a    )

       __lowerCamelCase		:							List[str]		 =						tokenizer.tokenize('This is a test'    )
       self.assertListEqual(a						,				['▁This', '▁is', '▁a', '▁t', 'est']    )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(a    )						,				[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]						,				)

       __lowerCamelCase		:							Any		 =						tokenizer.tokenize('I was born in 92000, and this is falsé.'    )
       self.assertListEqual(
           a						,				[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]						,				)
       __lowerCamelCase		:							Dict		 =						tokenizer.convert_tokens_to_ids(a    )
       self.assertListEqual(
           a						,				[
               value + tokenizer.fairseq_offset
               for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
           ]						,				)

       __lowerCamelCase		:							Dict		 =						tokenizer.convert_ids_to_tokens(a    )
       self.assertListEqual(
           a						,				[
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '<unk>',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '<unk>',
               '.',
           ]						,				)
       __lowerCamelCase		:							List[Any]		 =						tokenizer.vocab_size
       __lowerCamelCase		:							Optional[int]		 =						[tokenizer.convert_ids_to_tokens(a    ) for x in range(end - 7						,				a    )]

       self.assertListEqual(
           a						,				['__java__', '__python__', '__en_XX__', '__javascript__', '__php__', '__ruby__', '__go__']    )
       __lowerCamelCase		:							Tuple		 =						'java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'
       __lowerCamelCase		:							List[Any]		 =						tokenizer(a    ).input_ids
       self.assertEqual(
           tokenizer.decode(a						,				skip_special_tokens=a						,				clean_up_tokenization_spaces=a    )						,				a						,				)



@require_torch
@require_sentencepiece
@require_tokenizers
class 	A_ (	unittest.TestCase   ):




      '''simple docstring'''


      __snake_case 	=       """uclanlp/plbart-python-en_XX"""
      __snake_case 	=       [
          """def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])""",
          """def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""",
      ]
      __snake_case 	=       [
          """Returns the maximum value of a b c.""",
          """Sums the values of a b c.""",
      ]
      __snake_case 	=       [
          134,
          5452,
          33460,
          33441,
          33463,
          33465,
          33463,
          33449,
          988,
          20,
          33456,
          19,
          33456,
          771,
          39,
          4258,
          889,
          3318,
          33441,
          33463,
          33465,
          33463,
          33449,
          2471,
          2,
          PYTHON_CODE,
      ]




      @classmethod
      def 					_snake_case     (    cls:	str    ):
       __lowerCamelCase		:							PLBartTokenizer		 =						PLBartTokenizer.from_pretrained(
           cls.checkpoint_name						,				language_codes='base'						,				src_lang='python'						,				tgt_lang='en_XX'    )
       __lowerCamelCase		:							int		 =						1
       return cls




      def 					_snake_case     (    self:	Optional[int]    ):
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__java__']						,				5_0001    )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__python__']						,				5_0002    )
       self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['__en_XX__']						,				5_0003    )




      def 					_snake_case     (    self:	str    ):
       __lowerCamelCase		:							Union[str, Any]		 =						self.tokenizer.batch_encode_plus(self.src_text    ).input_ids[0]
       self.assertListEqual(self.expected_src_tokens						,				a    )




      def 					_snake_case     (    self:	int    ):
       self.assertIn(a						,				self.tokenizer.all_special_ids    )
       __lowerCamelCase		:							Dict		 =						[EN_CODE, 9037, 3_3442, 57, 752, 153, 14, 56, 18, 9, 2]
       __lowerCamelCase		:							Dict		 =						self.tokenizer.decode(a						,				skip_special_tokens=a    )
       __lowerCamelCase		:							List[str]		 =						self.tokenizer.decode(generated_ids[1:]						,				skip_special_tokens=a    )
       self.assertEqual(a						,				a    )
       self.assertNotIn(self.tokenizer.eos_token						,				a    )




      def 					_snake_case     (    self:	Optional[Any]    ):
       __lowerCamelCase		:							Union[str, Any]		 =						['def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])' * 20]
       self.assertIsInstance(src_text[0]						,				a    )
       __lowerCamelCase		:							str		 =						10
       __lowerCamelCase		:							Dict		 =						self.tokenizer(a						,				max_length=a						,				truncation=a    ).input_ids[0]
       self.assertEqual(ids[-2]						,				2    )
       self.assertEqual(ids[-1]						,				a    )
       self.assertEqual(len(a    )						,				a    )




      def 					_snake_case     (    self:	Tuple    ):
       self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', '__java__']    )						,				[5_0004, 5_0001]    )




      def 					_snake_case     (    self:	Dict    ):
       __lowerCamelCase		:							int		 =						tempfile.mkdtemp()
       __lowerCamelCase		:							Union[str, Any]		 =						self.tokenizer.fairseq_tokens_to_ids
       self.tokenizer.save_pretrained(a    )
       __lowerCamelCase		:							Union[str, Any]		 =						PLBartTokenizer.from_pretrained(a    )
       self.assertDictEqual(new_tok.fairseq_tokens_to_ids						,				a    )




      @require_torch
      def 					_snake_case     (    self:	Union[str, Any]    ):
       __lowerCamelCase		:							Dict		 =						self.tokenizer(self.src_text						,				text_target=self.tgt_text						,				padding=a						,				return_tensors='pt'    )
       __lowerCamelCase		:							str		 =						shift_tokens_right(batch['labels']						,				self.tokenizer.pad_token_id    )

       # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
       self.assertEqual(batch.input_ids[1][-2:].tolist()						,				[2, PYTHON_CODE]    )
       self.assertEqual(batch.decoder_input_ids[1][0]						,				a    )
       self.assertEqual(batch.decoder_input_ids[1][-1]						,				2    )
       self.assertEqual(batch.labels[1][-2:].tolist()						,				[2, EN_CODE]    )




      @require_torch
      def 					_snake_case     (    self:	Tuple    ):
       __lowerCamelCase		:							Union[str, Any]		 =						self.tokenizer(
           self.src_text						,				text_target=self.tgt_text						,				padding=a						,				truncation=a						,				max_length=len(self.expected_src_tokens    )						,				return_tensors='pt'						,				)
       __lowerCamelCase		:							Optional[int]		 =						shift_tokens_right(batch['labels']						,				self.tokenizer.pad_token_id    )

       self.assertIsInstance(a						,				a    )

       self.assertEqual((2, 26)						,				batch.input_ids.shape    )
       self.assertEqual((2, 26)						,				batch.attention_mask.shape    )
       __lowerCamelCase		:							List[Any]		 =						batch.input_ids.tolist()[0]
       self.assertListEqual(self.expected_src_tokens						,				a    )
       self.assertEqual(2						,				batch.decoder_input_ids[0, -1]    )  # EOS
       # Test that special tokens are reset
       self.assertEqual(self.tokenizer.prefix_tokens						,				[]    )
       self.assertEqual(self.tokenizer.suffix_tokens						,				[self.tokenizer.eos_token_id, PYTHON_CODE]    )




      def 					_snake_case     (    self:	Optional[int]    ):
       __lowerCamelCase		:							Optional[int]		 =						self.tokenizer(self.src_text						,				padding=a						,				truncation=a						,				max_length=3						,				return_tensors='pt'    )
       __lowerCamelCase		:							Tuple		 =						self.tokenizer(
           text_target=self.tgt_text						,				padding=a						,				truncation=a						,				max_length=10						,				return_tensors='pt'    )
       __lowerCamelCase		:							Any		 =						targets['input_ids']
       __lowerCamelCase		:							Dict		 =						shift_tokens_right(a						,				self.tokenizer.pad_token_id    )

       self.assertEqual(batch.input_ids.shape[1]						,				3    )
       self.assertEqual(batch.decoder_input_ids.shape[1]						,				10    )




      @require_torch
      def 					_snake_case     (    self:	Tuple    ):
       __lowerCamelCase		:							str		 =						self.tokenizer._build_translation_inputs(
           'A test'						,				return_tensors='pt'						,				src_lang='en_XX'						,				tgt_lang='java'    )

       self.assertEqual(
           nested_simplify(a    )						,				{
               # A, test, EOS, en_XX
               'input_ids': [[150, 242, 2, 5_0003]],
               'attention_mask': [[1, 1, 1, 1]],
               # java
               'forced_bos_token_id': 5_0001,
           }						,				)





import datasets


lowercase_	   =     '\\n@InProceedings{conneau2018xnli,\n  author = "Conneau, Alexis\n                 and Rinott, Ruty\n                 and Lample, Guillaume\n                 and Williams, Adina\n                 and Bowman, Samuel R.\n                 and Schwenk, Holger\n                 and Stoyanov, Veselin",\n  title = "XNLI: Evaluating Cross-lingual Sentence Representations",\n  booktitle = "Proceedings of the 2018 Conference on Empirical Methods\n               in Natural Language Processing",\n  year = "2018",\n  publisher = "Association for Computational Linguistics",\n  location = "Brussels, Belgium",\n}\n'

lowercase_	   =     '\\nXNLI is a subset of a few thousand examples from MNLI which has been translated\ninto a 14 different languages (some low-ish resource). As with MNLI, the goal is\nto predict textual entailment (does sentence A imply/contradict/neither sentence\nB) and is a classification task (given two sentences, predict one of three\nlabels).\n'

lowercase_	   =     '\nComputes XNLI score which is just simple accuracy.\nArgs:\n    predictions: Predicted labels.\n    references: Ground truth labels.\nReturns:\n    \'accuracy\': accuracy\nExamples:\n\n    >>> predictions = [0, 1]\n    >>> references = [0, 1]\n    >>> xnli_metric = datasets.load_metric("xnli")\n    >>> results = xnli_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n'
def 				UpperCamelCase__					(  SCREAMING_SNAKE_CASE__      ,	SCREAMING_SNAKE_CASE__						):
 return (preds == labels).mean()



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION ,				_KWARGS_DESCRIPTION   )
class 	A_ (	datasets.Metric   ):




      '''simple docstring'''


      def 					_snake_case     (    self:	Union[str, Any]    ):
       return datasets.MetricInfo(
           description=_DESCRIPTION						,				citation=_CITATION						,				inputs_description=_KWARGS_DESCRIPTION						,				features=datasets.Features(
               {
                   'predictions': datasets.Value('int64' if self.config_name != 'sts-b' else 'float32'    ),
                   'references': datasets.Value('int64' if self.config_name != 'sts-b' else 'float32'    ),
               }    )						,				codebase_urls=[]						,				reference_urls=[]						,				format='numpy'						,				)




      def 					_snake_case     (    self:	int						,				a:	Optional[Any]						,				a:	Optional[Any]    ):
       return {"accuracy": simple_accuracy(a						,				a    )}

