





"""simple docstring"""






import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 			__lowercase     (     _UpperCamelCase				, unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase					=					GPTaTokenizer
					__lowerCAmelCase					=					GPTaTokenizerFast
					__lowerCAmelCase					=					True
					__lowerCAmelCase					=					{'''add_prefix_space''': True}
					__lowerCAmelCase					=					False







					def 	_lowerCamelCase						(  self       ):
									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									__a	:  List[str]         =					[
									    '''l''',
									    '''o''',
									    '''w''',
									    '''e''',
									    '''r''',
									    '''s''',
									    '''t''',
									    '''i''',
									    '''d''',
									    '''n''',
									    '''\u0120''',
									    '''\u0120l''',
									    '''\u0120n''',
									    '''\u0120lo''',
									    '''\u0120low''',
									    '''er''',
									    '''\u0120lowest''',
									    '''\u0120newer''',
									    '''\u0120wider''',
									    '''<unk>''',
									    '''<|endoftext|>''',
									]
									__a	:  Optional[int]         =					dict(zip(_UpperCAmelCase		,     range(len(_UpperCAmelCase       )       )       )       )
									__a	:  int         =					['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
									__a	:  Dict         =					{'''unk_token''': '''<unk>'''}

									__a	:  Optional[int]         =					os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''vocab_file''']       )
									__a	:  Any         =					os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES['''merges_file''']       )
									with open(self.vocab_file		,     '''w'''		,     encoding='''utf-8'''       ) as fp:
													fp.write(json.dumps(_UpperCAmelCase       ) + '''\n'''       )
									with open(self.merges_file		,     '''w'''		,     encoding='''utf-8'''       ) as fp:
													fp.write('''\n'''.join(_UpperCAmelCase       )       )







					def 	_lowerCamelCase						(  self		,     **_UpperCAmelCase       ):
									kwargs.update(self.special_tokens_map       )
									return GPTaTokenizer.from_pretrained(self.tmpdirname		,     **_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self		,     **_UpperCAmelCase       ):
									kwargs.update(self.special_tokens_map       )
									return GPTaTokenizerFast.from_pretrained(self.tmpdirname		,     **_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									__a	:  Optional[int]         =					'''lower newer'''
									__a	:  Tuple         =					'''lower newer'''
									return input_text, output_text







					def 	_lowerCamelCase						(  self       ):
									__a	:  str         =					GPTaTokenizer(self.vocab_file		,     self.merges_file		,     **self.special_tokens_map       )
									__a	:  Dict         =					'''lower newer'''
									__a	:  List[Any]         =					['''\u0120low''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
									__a	:  Tuple         =					tokenizer.tokenize(_UpperCAmelCase		,     add_prefix_space=_UpperCAmelCase       )
									self.assertListEqual(_UpperCAmelCase		,     _UpperCAmelCase       )

									__a	:  Optional[int]         =					tokens + [tokenizer.unk_token]
									__a	:  int         =					[14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )		,     _UpperCAmelCase       )







					def 	_lowerCamelCase						(  self       ):
									if not self.test_rust_tokenizer:
													return

									__a	:  List[str]         =					self.get_tokenizer()
									__a	:  Any         =					self.get_rust_tokenizer(add_prefix_space=_UpperCAmelCase       )

									__a	:  Union[str, Any]         =					'''lower newer'''

									# Testing tokenization
									__a	:  Optional[Any]         =					tokenizer.tokenize(_UpperCAmelCase		,     add_prefix_space=_UpperCAmelCase       )
									__a	:  List[Any]         =					rust_tokenizer.tokenize(_UpperCAmelCase       )
									self.assertListEqual(_UpperCAmelCase		,     _UpperCAmelCase       )

									# Testing conversion to ids without special tokens
									__a	:  Optional[int]         =					tokenizer.encode(_UpperCAmelCase		,     add_special_tokens=_UpperCAmelCase		,     add_prefix_space=_UpperCAmelCase       )
									__a	:  Union[str, Any]         =					rust_tokenizer.encode(_UpperCAmelCase		,     add_special_tokens=_UpperCAmelCase       )
									self.assertListEqual(_UpperCAmelCase		,     _UpperCAmelCase       )

									# Testing conversion to ids with special tokens
									__a	:  List[Any]         =					self.get_rust_tokenizer(add_prefix_space=_UpperCAmelCase       )
									__a	:  List[Any]         =					tokenizer.encode(_UpperCAmelCase		,     add_prefix_space=_UpperCAmelCase       )
									__a	:  Union[str, Any]         =					rust_tokenizer.encode(_UpperCAmelCase       )
									self.assertListEqual(_UpperCAmelCase		,     _UpperCAmelCase       )

									# Testing the unknown token
									__a	:  Dict         =					tokens + [rust_tokenizer.unk_token]
									__a	:  Tuple         =					[14, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(_UpperCAmelCase       )		,     _UpperCAmelCase       )







					def 	_lowerCamelCase						(  self		,     *_UpperCAmelCase		,     **_UpperCAmelCase       ):
									# It's very difficult to mix/test pretokenization with byte-level
									# And get both GPT2 and Roberta to work at the same time (mostly an issue of adding a space before the string)
									pass







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase=15       ):
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
													with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																	__a	:  List[Any]         =					self.rust_tokenizer_class.from_pretrained(_UpperCAmelCase		,     **_UpperCAmelCase       )

																	# Simple input
																	__a	:  Union[str, Any]         =					'''This is a simple input'''
																	__a	:  Optional[int]         =					['''This is a simple input 1''', '''This is a simple input 2''']
																	__a	:  Tuple         =					('''This is a simple input''', '''This is a pair''')
																	__a	:  Optional[Any]         =					[
																	    ('''This is a simple input 1''', '''This is a simple input 2'''),
																	    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																	]

																	# Simple input tests
																	self.assertRaises(_UpperCAmelCase		,     tokenizer_r.encode		,     _UpperCAmelCase		,     max_length=_UpperCAmelCase		,     padding='''max_length'''       )

																	# Simple input
																	self.assertRaises(_UpperCAmelCase		,     tokenizer_r.encode_plus		,     _UpperCAmelCase		,     max_length=_UpperCAmelCase		,     padding='''max_length'''       )

																	# Simple input
																	self.assertRaises(
																	    _UpperCAmelCase		,     tokenizer_r.batch_encode_plus		,     _UpperCAmelCase		,     max_length=_UpperCAmelCase		,     padding='''max_length'''		,     )

																	# Pair input
																	self.assertRaises(_UpperCAmelCase		,     tokenizer_r.encode		,     _UpperCAmelCase		,     max_length=_UpperCAmelCase		,     padding='''max_length'''       )

																	# Pair input
																	self.assertRaises(_UpperCAmelCase		,     tokenizer_r.encode_plus		,     _UpperCAmelCase		,     max_length=_UpperCAmelCase		,     padding='''max_length'''       )

																	# Pair input
																	self.assertRaises(
																	    _UpperCAmelCase		,     tokenizer_r.batch_encode_plus		,     _UpperCAmelCase		,     max_length=_UpperCAmelCase		,     padding='''max_length'''		,     )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Optional[int]         =					GPTaTokenizer.from_pretrained(self.tmpdirname		,     pad_token='''<pad>'''       )

									# Simple input
									__a	:  Dict         =					'''This is a simple input'''
									__a	:  List[Any]         =					['''This is a simple input looooooooong''', '''This is a simple input''']
									__a	:  Union[str, Any]         =					('''This is a simple input''', '''This is a pair''')
									__a	:  Optional[int]         =					[
									    ('''This is a simple input loooooong''', '''This is a simple input'''),
									    ('''This is a simple pair loooooong''', '''This is a simple pair'''),
									]

									__a	:  Dict         =					tokenizer.pad_token_id

									__a	:  List[str]         =					tokenizer(_UpperCAmelCase		,     padding='''max_length'''		,     max_length=30		,     return_tensors='''np'''       )
									__a	:  int         =					tokenizer(_UpperCAmelCase		,     padding=_UpperCAmelCase		,     truncate=_UpperCAmelCase		,     return_tensors='''np'''       )
									__a	:  Union[str, Any]         =					tokenizer(*_UpperCAmelCase		,     padding='''max_length'''		,     max_length=60		,     return_tensors='''np'''       )
									__a	:  Optional[int]         =					tokenizer(_UpperCAmelCase		,     padding=_UpperCAmelCase		,     truncate=_UpperCAmelCase		,     return_tensors='''np'''       )

									# s
									# test single string max_length padding
									self.assertEqual(out_s['''input_ids'''].shape[-1]		,     30       )
									self.assertTrue(pad_token_id in out_s['''input_ids''']       )
									self.assertTrue(0 in out_s['''attention_mask''']       )

									# s2
									# test automatic padding
									self.assertEqual(out_sa['''input_ids'''].shape[-1]		,     33       )
									# long slice doesn't have padding
									self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]       )
									self.assertFalse(0 in out_sa['''attention_mask'''][0]       )
									# short slice does have padding
									self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]       )
									self.assertTrue(0 in out_sa['''attention_mask'''][1]       )

									# p
									# test single pair max_length padding
									self.assertEqual(out_p['''input_ids'''].shape[-1]		,     60       )
									self.assertTrue(pad_token_id in out_p['''input_ids''']       )
									self.assertTrue(0 in out_p['''attention_mask''']       )

									# p2
									# test automatic padding pair
									self.assertEqual(out_pa['''input_ids'''].shape[-1]		,     52       )
									# long slice pair doesn't have padding
									self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]       )
									self.assertFalse(0 in out_pa['''attention_mask'''][0]       )
									# short slice pair does have padding
									self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]       )
									self.assertTrue(0 in out_pa['''attention_mask'''][1]       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  str         =					'''$$$'''
									__a	:  int         =					GPTaTokenizer.from_pretrained(self.tmpdirname		,     bos_token=_UpperCAmelCase		,     add_bos_token=_UpperCAmelCase       )

									__a	:  List[Any]         =					'''This is a simple input'''
									__a	:  Union[str, Any]         =					['''This is a simple input 1''', '''This is a simple input 2''']

									__a	:  Any         =					tokenizer.bos_token_id

									__a	:  Optional[int]         =					tokenizer(_UpperCAmelCase       )
									__a	:  Tuple         =					tokenizer(_UpperCAmelCase       )

									self.assertEqual(out_s.input_ids[0]		,     _UpperCAmelCase       )
									self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids       )       )

									__a	:  Optional[int]         =					tokenizer.decode(out_s.input_ids       )
									__a	:  Dict         =					tokenizer.batch_decode(out_sa.input_ids       )

									self.assertEqual(decode_s.split()[0]		,     _UpperCAmelCase       )
									self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa       )       )







					def 	_lowerCamelCase						(  self       ):
									pass







					def 	_lowerCamelCase						(  self       ):
									# TODO: change to self.get_tokenizers() when the fast version is implemented
									__a	:  Dict         =					[self.get_tokenizer(do_lower_case=_UpperCAmelCase		,     add_bos_token=_UpperCAmelCase       )]
									for tokenizer in tokenizers:
													with self.subTest(f"""{tokenizer.__class__.__name__}"""       ):
																	__a	:  Optional[int]         =					'''Encode this.'''
																	__a	:  Tuple         =					'''This one too please.'''
																	__a	:  List[Any]         =					tokenizer.encode(_UpperCAmelCase		,     add_special_tokens=_UpperCAmelCase       )
																	encoded_sequence += tokenizer.encode(_UpperCAmelCase		,     add_special_tokens=_UpperCAmelCase       )
																	__a	:  Dict         =					tokenizer.encode_plus(
																	    _UpperCAmelCase		,     _UpperCAmelCase		,     add_special_tokens=_UpperCAmelCase		,     return_special_tokens_mask=_UpperCAmelCase		,     )
																	__a	:  int         =					encoded_sequence_dict['''input_ids''']
																	__a	:  Dict         =					encoded_sequence_dict['''special_tokens_mask''']
																	self.assertEqual(len(_UpperCAmelCase       )		,     len(_UpperCAmelCase       )       )

																	__a	:  Optional[Any]         =					[
																	    (x if not special_tokens_mask[i] else None) for i, x in enumerate(_UpperCAmelCase       )
																	]
																	__a	:  Optional[int]         =					[x for x in filtered_sequence if x is not None]
																	self.assertEqual(_UpperCAmelCase		,     _UpperCAmelCase       )






@require_tokenizers
class 			__lowercase     (     unittest.TestCase							):







					'''simple docstring'''




					def 	_lowerCamelCase						(  self       ):
									# More context:
									# https://huggingface.co/wjmcat/opt-350m-paddle/discussions/1
									# https://huggingface.slack.com/archives/C01N44FJDHT/p1653511495183519
									# https://github.com/huggingface/transformers/pull/17088#discussion_r871246439

									__a	:  Optional[int]         =					AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,     from_slow=_UpperCAmelCase       )
									__a	:  List[Any]         =					'''A photo of a cat'''

									__a	:  List[Any]         =					tokenizer.encode(
									    _UpperCAmelCase		,     )
									self.assertEqual(_UpperCAmelCase		,     [2, 250, 1345, 9, 10, 4758]       )
									tokenizer.save_pretrained('''test_opt'''       )

									__a	:  List[Any]         =					AutoTokenizer.from_pretrained('''./test_opt'''       )
									__a	:  Any         =					tokenizer.encode(
									    _UpperCAmelCase		,     )
									self.assertEqual(_UpperCAmelCase		,     [2, 250, 1345, 9, 10, 4758]       )







					def 	_lowerCamelCase						(  self       ):
									__a	:  Optional[int]         =					AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,     use_slow=_UpperCAmelCase       )
									__a	:  Tuple         =					'''A photo of a cat'''

									__a	:  Any         =					tokenizer.encode(
									    _UpperCAmelCase		,     )
									# Same as above
									self.assertEqual(_UpperCAmelCase		,     [2, 250, 1345, 9, 10, 4758]       )







					@unittest.skip('''This test is failing because of a bug in the fast tokenizer'''       )
					def 	_lowerCamelCase						(  self       ):
									__a	:  Tuple         =					AutoTokenizer.from_pretrained('''facebook/opt-350m'''		,     from_slow=_UpperCAmelCase       )

									__a	:  Tuple         =					'''bos'''
									__a	:  Optional[int]         =					tokenizer.get_vocab()['''bos''']

									__a	:  Union[str, Any]         =					'''A photo of a cat'''
									__a	:  Any         =					tokenizer.encode(
									    _UpperCAmelCase		,     )
									# We changed the bos token
									self.assertEqual(_UpperCAmelCase		,     [31957, 250, 1345, 9, 10, 4758]       )
									tokenizer.save_pretrained('''./tok'''       )
									__a	:  Any         =					AutoTokenizer.from_pretrained('''./tok'''       )
									self.assertTrue(tokenizer.is_fast       )
									__a	:  List[str]         =					tokenizer.encode(
									    _UpperCAmelCase		,     )
									self.assertEqual(_UpperCAmelCase		,     [31957, 250, 1345, 9, 10, 4758]       )






"""simple docstring"""






# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool




class 			__lowercase     (     _UpperCamelCase							):







					'''simple docstring'''




					__lowerCAmelCase					=					'''openai/whisper-base'''
					__lowerCAmelCase					=					(
					    '''This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the '''
					    '''transcribed text.'''
					)
					__lowerCAmelCase					=					'''transcriber'''
					__lowerCAmelCase					=					WhisperProcessor
					__lowerCAmelCase					=					WhisperForConditionalGeneration

					__lowerCAmelCase					=					['''audio''']
					__lowerCAmelCase					=					['''text''']







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									return self.pre_processor(_UpperCAmelCase		,     return_tensors='''pt'''       ).input_features







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									return self.model.generate(inputs=_UpperCAmelCase       )







					def 	_lowerCamelCase						(  self		,     _UpperCAmelCase       ):
									return self.pre_processor.batch_decode(_UpperCAmelCase		,     skip_special_tokens=_UpperCAmelCase       )[0]