



"""simple docstring"""


import pickle
import shutil
import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, XLMRobertaTokenizer, XLMRobertaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_    					=	get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
@require_tokenizers
class 				A_      (					SCREAMING_SNAKE_CASE_     ,       unittest.TestCase    ):



						"""simple docstring"""
						__UpperCamelCase		    =							XLMRobertaTokenizer
						__UpperCamelCase		    =							XLMRobertaTokenizerFast
						__UpperCamelCase		    =							True
						__UpperCamelCase		    =							True






						def      UpperCAmelCase__					( self			:int )    ->      Optional[Any]:
												super().setUp()

												# We have a SentencePiece fixture for testing
												UpperCAmelCase					=			XLMRobertaTokenizer(lowercase_       ,  keep_accents=lowercase_ )
												tokenizer.save_pretrained(self.tmpdirname )






						def      UpperCAmelCase__					( self			:Any )    ->      Dict:
												UpperCAmelCase					=			'<pad>'
												UpperCAmelCase					=			1

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_ )       ,  lowercase_ )
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_ )       ,  lowercase_ )






						def      UpperCAmelCase__					( self			:List[Any] )    ->      Union[str, Any]:
												UpperCAmelCase					=			list(self.get_tokenizer().get_vocab().keys() )

												self.assertEqual(vocab_keys[0]       ,  '<s>' )
												self.assertEqual(vocab_keys[1]       ,  '<pad>' )
												self.assertEqual(vocab_keys[-1]       ,  '<mask>' )
												self.assertEqual(len(lowercase_ )       ,  10_02 )






						def      UpperCAmelCase__					( self			:Tuple )    ->      Optional[Any]:
												self.assertEqual(self.get_tokenizer().vocab_size       ,  10_02 )






						def      UpperCAmelCase__					( self			:int )    ->      Any:
												UpperCAmelCase					=			XLMRobertaTokenizer(lowercase_       ,  keep_accents=lowercase_ )

												UpperCAmelCase					=			tokenizer.tokenize('This is a test' )
												self.assertListEqual(lowercase_       ,  ['▁This', '▁is', '▁a', '▁t', 'est'] )

												self.assertListEqual(
												    tokenizer.convert_tokens_to_ids(lowercase_ )       ,  [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]       ,  )

												UpperCAmelCase					=			tokenizer.tokenize('I was born in 92000, and this is falsé.' )
												self.assertListEqual(
												    lowercase_       ,  [
												        SPIECE_UNDERLINE + 'I',
												        SPIECE_UNDERLINE + 'was',
												        SPIECE_UNDERLINE + 'b',
												        'or',
												        'n',
												        SPIECE_UNDERLINE + 'in',
												        SPIECE_UNDERLINE + '',
												        '9',
												        '2',
												        '0',
												        '0',
												        '0',
												        ',',
												        SPIECE_UNDERLINE + 'and',
												        SPIECE_UNDERLINE + 'this',
												        SPIECE_UNDERLINE + 'is',
												        SPIECE_UNDERLINE + 'f',
												        'al',
												        's',
												        'é',
												        '.',
												    ]       ,  )
												UpperCAmelCase					=			tokenizer.convert_tokens_to_ids(lowercase_ )
												self.assertListEqual(
												    lowercase_       ,  [
												        value + tokenizer.fairseq_offset
												        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
												        #                                       ^ unk: 2 + 1 = 3                  unk: 2 + 1 = 3 ^
												    ]       ,  )

												UpperCAmelCase					=			tokenizer.convert_ids_to_tokens(lowercase_ )
												self.assertListEqual(
												    lowercase_       ,  [
												        SPIECE_UNDERLINE + 'I',
												        SPIECE_UNDERLINE + 'was',
												        SPIECE_UNDERLINE + 'b',
												        'or',
												        'n',
												        SPIECE_UNDERLINE + 'in',
												        SPIECE_UNDERLINE + '',
												        '<unk>',
												        '2',
												        '0',
												        '0',
												        '0',
												        ',',
												        SPIECE_UNDERLINE + 'and',
												        SPIECE_UNDERLINE + 'this',
												        SPIECE_UNDERLINE + 'is',
												        SPIECE_UNDERLINE + 'f',
												        'al',
												        's',
												        '<unk>',
												        '.',
												    ]       ,  )






						def      UpperCAmelCase__					( self			:List[Any] )    ->      str:
												if not self.test_slow_tokenizer:
																		# as we don't have a slow version, we can't compare the outputs between slow and fast versions
																		return

												UpperCAmelCase					=			(self.rust_tokenizer_class, 'hf-internal-testing/tiny-xlm-roberta', {})
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
																								UpperCAmelCase					=			self.rust_tokenizer_class.from_pretrained(lowercase_       ,  **lowercase_ )
																								UpperCAmelCase					=			self.tokenizer_class.from_pretrained(lowercase_       ,  **lowercase_ )

																								UpperCAmelCase					=			tempfile.mkdtemp()

																								UpperCAmelCase					=			tokenizer_r.save_pretrained(lowercase_ )
																								UpperCAmelCase					=			tokenizer_p.save_pretrained(lowercase_ )

																								# Checks it save with the same files + the tokenizer.json file for the fast one
																								self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files ) )
																								UpperCAmelCase					=			tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f )
																								self.assertSequenceEqual(lowercase_       ,  lowercase_ )

																								# Checks everything loads correctly in the same way
																								UpperCAmelCase					=			tokenizer_r.from_pretrained(lowercase_ )
																								UpperCAmelCase					=			tokenizer_p.from_pretrained(lowercase_ )

																								# Check special tokens are set accordingly on Rust and Python
																								for key in tokenizer_pp.special_tokens_map:
																														self.assertTrue(hasattr(lowercase_       ,  lowercase_ ) )
																														# self.assertEqual(getattr(tokenizer_rp, key), getattr(tokenizer_pp, key))
																														# self.assertEqual(getattr(tokenizer_rp, key + "_id"), getattr(tokenizer_pp, key + "_id"))

																								shutil.rmtree(lowercase_ )

																								# Save tokenizer rust, legacy_format=True
																								UpperCAmelCase					=			tempfile.mkdtemp()

																								UpperCAmelCase					=			tokenizer_r.save_pretrained(lowercase_       ,  legacy_format=lowercase_ )
																								UpperCAmelCase					=			tokenizer_p.save_pretrained(lowercase_ )

																								# Checks it save with the same files
																								self.assertSequenceEqual(lowercase_       ,  lowercase_ )

																								# Checks everything loads correctly in the same way
																								UpperCAmelCase					=			tokenizer_r.from_pretrained(lowercase_ )
																								UpperCAmelCase					=			tokenizer_p.from_pretrained(lowercase_ )

																								# Check special tokens are set accordingly on Rust and Python
																								for key in tokenizer_pp.special_tokens_map:
																														self.assertTrue(hasattr(lowercase_       ,  lowercase_ ) )

																								shutil.rmtree(lowercase_ )

																								# Save tokenizer rust, legacy_format=False
																								UpperCAmelCase					=			tempfile.mkdtemp()

																								UpperCAmelCase					=			tokenizer_r.save_pretrained(lowercase_       ,  legacy_format=lowercase_ )
																								UpperCAmelCase					=			tokenizer_p.save_pretrained(lowercase_ )

																								# Checks it saved the tokenizer.json file
																								self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files ) )

																								# Checks everything loads correctly in the same way
																								UpperCAmelCase					=			tokenizer_r.from_pretrained(lowercase_ )
																								UpperCAmelCase					=			tokenizer_p.from_pretrained(lowercase_ )

																								# Check special tokens are set accordingly on Rust and Python
																								for key in tokenizer_pp.special_tokens_map:
																														self.assertTrue(hasattr(lowercase_       ,  lowercase_ ) )

																								shutil.rmtree(lowercase_ )






						@cached_property
						def      UpperCAmelCase__					( self			:Dict )    ->      Optional[int]:
												return XLMRobertaTokenizer.from_pretrained('xlm-roberta-base' )






						def      UpperCAmelCase__					( self			:List[Any] )    ->      List[str]:
												with tempfile.NamedTemporaryFile() as f:
																		shutil.copyfile(lowercase_       ,  f.name )
																		UpperCAmelCase					=			XLMRobertaTokenizer(f.name       ,  keep_accents=lowercase_ )
																		UpperCAmelCase					=			pickle.dumps(lowercase_ )
												pickle.loads(lowercase_ )






						def      UpperCAmelCase__					( self			:Optional[Any] )    ->      Optional[Any]:
												if not self.test_rust_tokenizer:
																		return

												UpperCAmelCase					=			self.get_tokenizer()
												UpperCAmelCase					=			self.get_rust_tokenizer()

												UpperCAmelCase					=			'I was born in 92000, and this is falsé.'

												UpperCAmelCase					=			tokenizer.tokenize(lowercase_ )
												UpperCAmelCase					=			rust_tokenizer.tokenize(lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )

												UpperCAmelCase					=			tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_ )
												UpperCAmelCase					=			rust_tokenizer.encode(lowercase_       ,  add_special_tokens=lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )

												UpperCAmelCase					=			self.get_rust_tokenizer()
												UpperCAmelCase					=			tokenizer.encode(lowercase_ )
												UpperCAmelCase					=			rust_tokenizer.encode(lowercase_ )
												self.assertListEqual(lowercase_       ,  lowercase_ )






						@slow
						def      UpperCAmelCase__					( self			:Dict )    ->      Dict:
												UpperCAmelCase					=			'Hello World!'
												UpperCAmelCase					=			[0, 3_53_78, 66_61, 38, 2]
												# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
												# xlmr.eval()
												# xlmr.encode(symbols)

												self.assertListEqual(lowercase_       ,  self.big_tokenizer.encode(lowercase_ ) )






						@slow
						def      UpperCAmelCase__					( self			:Union[str, Any] )    ->      int:
												UpperCAmelCase					=			(
												    'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
												    ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
												)
												UpperCAmelCase					=			[
												    0,
												    32_93,
												    83,
												    10,
												    45_52,
												    49_89,
												    79_86,
												    6_78,
												    10,
												    59_15,
												    1_11,
												    17_94_59,
												    12_48_50,
												    4,
												    60_44,
												    2_37,
												    12,
												    6,
												    5,
												    6,
												    4,
												    67_80,
												    7_05,
												    15,
												    13_88,
												    44,
												    3_78,
												    1_01_14,
												    7_11,
												    1_52,
												    20,
												    6,
												    5,
												    2_23_76,
												    6_42,
												    12_21,
												    1_51_90,
												    3_41_53,
												    4_50,
												    56_08,
												    9_59,
												    11_19,
												    5_77_02,
												    1_36,
												    1_86,
												    47,
												    10_98,
												    2_93_67,
												    47,
												    # 4426, # What fairseq tokenizes from "<unk>": "_<"
												    # 3678, # What fairseq tokenizes from "<unk>": "unk"
												    # 2740, # What fairseq tokenizes from "<unk>": ">"
												    3,  # What we tokenize from "<unk>": "<unk>"
												    6,  # Residue from the tokenization: an extra sentencepiece underline
												    4,
												    60_44,
												    2_37,
												    62_84,
												    5_09_01,
												    5_28,
												    31,
												    90,
												    34,
												    9_27,
												    2,
												]
												# xlmr = torch.hub.load('pytorch/fairseq', 'xlmr.base')  # xlmr.large has same tokenizer
												# xlmr.eval()
												# xlmr.encode(symbols)

												self.assertListEqual(lowercase_       ,  self.big_tokenizer.encode(lowercase_ ) )






						@slow
						def      UpperCAmelCase__					( self			:List[Any] )    ->      Any:
												# fmt: off
												UpperCAmelCase					=			{'input_ids': [[0, 1_10_62, 8_27_72, 7, 15, 8_27_72, 5_38, 5_15_29, 2_37, 1_71_98, 12_90, 2_06, 9, 21_51_75, 13_14, 1_36, 1_71_98, 12_90, 2_06, 9, 5_63_59, 42, 12_20_09, 9, 1_64_66, 16, 8_73_44, 45_37, 9, 47_17, 7_83_81, 6, 15_99_58, 7, 15, 2_44_80, 6_18, 4, 5_27, 2_26_93, 54_28, 4, 27_77, 2_44_80, 98_74, 4, 4_35_23, 5_94, 4, 8_03, 1_83_92, 3_31_89, 18, 4, 4_35_23, 2_44_47, 1_23_99, 1_00, 2_49_55, 8_36_58, 96_26, 14_40_57, 15, 8_39, 2_23_35, 16, 1_36, 2_49_55, 8_36_58, 8_34_79, 15, 3_91_02, 7_24, 16, 6_78, 6_45, 27_89, 13_28, 45_89, 42, 12_20_09, 11_57_74, 23, 8_05, 13_28, 4_68_76, 7, 1_36, 5_38_94, 19_40, 4_22_27, 4_11_59, 1_77_21, 8_23, 4_25, 4, 2_75_12, 9_87_22, 2_06, 1_36, 55_31, 49_70, 9_19, 1_73_36, 5, 2], [0, 2_00_80, 6_18, 83, 8_27_75, 47, 4_79, 9, 15_17, 73, 5_38_94, 3_33, 8_05_81, 11_01_17, 1_88_11, 52_56, 12_95, 51, 15_25_26, 2_97, 79_86, 3_90, 12_44_16, 5_38, 3_54_31, 2_14, 98, 1_50_44, 2_57_37, 1_36, 71_08, 4_37_01, 23, 7_56, 13_53_55, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 5_81, 6_37_73, 11_94_55, 6, 14_77_97, 8_82_03, 7, 6_45, 70, 21, 32_85, 1_02_69, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=lowercase_       ,  model_name='xlm-roberta-base'       ,  revision='d9d8a8ea5eb94b1c6654ae9249df7793cd2933d3'       ,  )










"""simple docstring"""


# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool







class 				A_      (					SCREAMING_SNAKE_CASE_    ):



						"""simple docstring"""
						__UpperCamelCase		    =							"""philschmid/bart-large-cnn-samsum"""
						__UpperCamelCase		    =							(
						    """This is a tool that summarizes an English text. It takes an input `text` containing the text to summarize, """
						    """and returns a summary of the text."""
						)
						__UpperCamelCase		    =							"""summarizer"""
						__UpperCamelCase		    =							AutoTokenizer
						__UpperCamelCase		    =							AutoModelForSeqaSeqLM

						__UpperCamelCase		    =							["""text"""]
						__UpperCamelCase		    =							["""text"""]






						def      UpperCAmelCase__					( self			:int       ,  lowercase_			:Tuple )    ->      int:
												return self.pre_processor(lowercase_       ,  return_tensors='pt'       ,  truncation=lowercase_ )






						def      UpperCAmelCase__					( self			:Dict       ,  lowercase_			:Dict )    ->      Optional[int]:
												return self.model.generate(**lowercase_ )[0]






						def      UpperCAmelCase__					( self			:Dict       ,  lowercase_			:Tuple )    ->      List[str]:
												return self.pre_processor.decode(lowercase_       ,  skip_special_tokens=lowercase_       ,  clean_up_tokenization_spaces=lowercase_ )





