


'''simple docstring'''






import argparse
import hashlib
import os
import urllib
import warnings

import torch
from torch import nn
from tqdm import tqdm

from transformers import WhisperConfig, WhisperForConditionalGeneration


__UpperCAmelCase       =							{
    '''tiny.en''': '''https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt''',
    '''tiny''': '''https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt''',
    '''base.en''': '''https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt''',
    '''base''': '''https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt''',
    '''small.en''': '''https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt''',
    '''small''': '''https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt''',
    '''medium.en''': '''https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt''',
    '''medium''': '''https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt''',
    '''large''': '''https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large.pt''',
    '''large-v2''': '''https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt''',
}


def  _snake_case       (					A	)						->   str:
							lowerCAmelCase__	       =     ['''layers''', '''blocks''']
							for k in ignore_keys:
														state_dict.pop(A			,   A	)


__UpperCAmelCase       =							{
    '''blocks''': '''layers''',
    '''mlp.0''': '''fc1''',
    '''mlp.2''': '''fc2''',
    '''mlp_ln''': '''final_layer_norm''',
    '''.attn.query''': '''.self_attn.q_proj''',
    '''.attn.key''': '''.self_attn.k_proj''',
    '''.attn.value''': '''.self_attn.v_proj''',
    '''.attn_ln''': '''.self_attn_layer_norm''',
    '''.attn.out''': '''.self_attn.out_proj''',
    '''.cross_attn.query''': '''.encoder_attn.q_proj''',
    '''.cross_attn.key''': '''.encoder_attn.k_proj''',
    '''.cross_attn.value''': '''.encoder_attn.v_proj''',
    '''.cross_attn_ln''': '''.encoder_attn_layer_norm''',
    '''.cross_attn.out''': '''.encoder_attn.out_proj''',
    '''decoder.ln.''': '''decoder.layer_norm.''',
    '''encoder.ln.''': '''encoder.layer_norm.''',
    '''token_embedding''': '''embed_tokens''',
    '''encoder.positional_embedding''': '''encoder.embed_positions.weight''',
    '''decoder.positional_embedding''': '''decoder.embed_positions.weight''',
    '''ln_post''': '''layer_norm''',
}


def  _snake_case       (					A	)						->   Any:
							lowerCAmelCase__	       =     list(s_dict.keys()	)
							for key in keys:
														lowerCAmelCase__	       =     key
														for k, v in WHISPER_MAPPING.items():
																					if k in key:
																												lowerCAmelCase__	       =     new_key.replace(A			,   A	)

														print(F"""{key} -> {new_key}"""	)

														lowerCAmelCase__	       =     s_dict.pop(A	)
							return s_dict


def  _snake_case       (					A	)						->   Union[str, Any]:
							lowerCAmelCase__       , lowerCAmelCase__	       =     emb.weight.shape
							lowerCAmelCase__	       =     nn.Linear(A			,   A			,   bias=A	)
							lowerCAmelCase__	       =     emb.weight.data
							return lin_layer


def  _snake_case       (					A			,   A	)						->   bytes:
							os.makedirs(A			,   exist_ok=A	)
							lowerCAmelCase__	       =     os.path.basename(A	)

							lowerCAmelCase__	       =     url.split('''/'''	)[-2]
							lowerCAmelCase__	       =     os.path.join(A			,   A	)

							if os.path.exists(A	) and not os.path.isfile(A	):
														raise RuntimeError(F"""{download_target} exists and is not a regular file"""	)

							if os.path.isfile(A	):
														lowerCAmelCase__	       =     open(A			,   '''rb'''	).read()
														if hashlib.shaaaa(A	).hexdigest() == expected_shaaaa:
																					return model_bytes
														else:
																					warnings.warn(F"""{download_target} exists, but the SHA256 checksum does not match; re-downloading the file"""	)

							with urllib.request.urlopen(A	) as source, open(A			,   '''wb'''	) as output:
														with tqdm(
														    total=int(source.info().get('''Content-Length'''	)	)			,   ncols=80			,   unit='''iB'''			,   unit_scale=A			,   unit_divisor=1024	) as loop:
																					while True:
																												lowerCAmelCase__	       =     source.read(8192	)
																												if not buffer:
																																			break

																												output.write(A	)
																												loop.update(len(A	)	)

							lowerCAmelCase__	       =     open(A			,   '''rb'''	).read()
							if hashlib.shaaaa(A	).hexdigest() != expected_shaaaa:
														raise RuntimeError(
														    '''Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model.'''	)

							return model_bytes


def  _snake_case       (					A			,   A	)						->   str:
							if ".pt" not in checkpoint_path:
														lowerCAmelCase__	       =     _download(_MODELS[checkpoint_path]	)
							else:
														lowerCAmelCase__	       =     torch.load(A			,   map_location='''cpu'''	)
							lowerCAmelCase__	       =     original_checkpoint['''dims''']
							lowerCAmelCase__	       =     original_checkpoint['''model_state_dict''']
							lowerCAmelCase__	       =     state_dict['''decoder.token_embedding.weight''']
							remove_ignore_keys_(A	)
							rename_keys(A	)
							lowerCAmelCase__	       =     True
							lowerCAmelCase__	       =     state_dict['''decoder.layers.0.fc1.weight'''].shape[0]

							lowerCAmelCase__	       =     WhisperConfig(
							    vocab_size=dimensions['''n_vocab''']			,   encoder_ffn_dim=A			,   decoder_ffn_dim=A			,   num_mel_bins=dimensions['''n_mels''']			,   d_model=dimensions['''n_audio_state''']			,   max_target_positions=dimensions['''n_text_ctx''']			,   encoder_layers=dimensions['''n_audio_layer''']			,   encoder_attention_heads=dimensions['''n_audio_head''']			,   decoder_layers=dimensions['''n_text_layer''']			,   decoder_attention_heads=dimensions['''n_text_state''']			,   max_source_positions=dimensions['''n_audio_ctx''']			,   )

							lowerCAmelCase__	       =     WhisperForConditionalGeneration(A	)
							lowerCAmelCase__       , lowerCAmelCase__	       =     model.model.load_state_dict(A			,   strict=A	)
							if len(A	) > 0 and not set(A	) <= {
							    "encoder.embed_positions.weights",
							    "decoder.embed_positions.weights",
							}:
														raise ValueError(
														    '''Only `encoder.embed_positions.weights` and `decoder.embed_positions.weights`  are allowed to be missing,'''
														    F""" but all the following weights are missing {missing}"""	)

							if tie_embeds:
														lowerCAmelCase__	       =     make_linear_from_emb(model.model.decoder.embed_tokens	)
							else:
														lowerCAmelCase__	       =     proj_out_weights

							model.save_pretrained(A	)


if __name__ == "__main__":
		__UpperCAmelCase       =							argparse.ArgumentParser()
		# # Required parameters
		parser.add_argument('''--checkpoint_path''', type=str, help='''Patht to the downloaded checkpoints''')
		parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
		__UpperCAmelCase       =							parser.parse_args()

		convert_openai_whisper_to_tfms(args.checkpoint_path, args.pytorch_dump_folder_path)



'''simple docstring'''






import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 		a__    (							a__    ,    unittest.TestCase  ):





							'''simple docstring'''






							lowercase__      :       List[Any]     						=				CLIPTokenizer
							lowercase__      :       List[str]     						=				CLIPTokenizerFast
							lowercase__      :       Dict     						=				True
							lowercase__      :       Any     						=				{}
							lowercase__      :       Optional[int]     						=				False







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														super().setUp()

														# fmt: off
														lowerCAmelCase__	       =     ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
														# fmt: on
														lowerCAmelCase__	       =     dict(zip(lowerCamelCase_ ,   range(len(lowerCamelCase_       )       )       )       )
														lowerCAmelCase__	       =     ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
														lowerCAmelCase__	       =     {'''unk_token''': '''<unk>'''}

														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''vocab_file''']       )
														lowerCAmelCase__	       =     os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['''merges_file''']       )
														with open(self.vocab_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write(json.dumps(lowerCamelCase_       ) + '''\n'''       )
														with open(self.merges_file ,   '''w''' ,   encoding='''utf-8'''       ) as fp:
																					fp.write('''\n'''.join(lowerCamelCase_       )       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							Any:
														kwargs.update(self.special_tokens_map       )
														return CLIPTokenizer.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   **lowerCamelCase_       )	->							int:
														kwargs.update(self.special_tokens_map       )
														return CLIPTokenizerFast.from_pretrained(self.tmpdirname ,   **lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_       )	->							Any:
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     '''lower newer'''
														return input_text, output_text







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														lowerCAmelCase__	       =     CLIPTokenizer(self.vocab_file ,   self.merges_file ,   **self.special_tokens_map       )
														lowerCAmelCase__	       =     '''lower newer'''
														lowerCAmelCase__	       =     ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
														lowerCAmelCase__	       =     tokenizer.tokenize(lowerCamelCase_       )
														self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

														lowerCAmelCase__	       =     tokens + [tokenizer.unk_token]
														lowerCAmelCase__	       =     [10, 2, 16, 9, 3, 2, 16, 20]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_       ) ,   lowerCamelCase_       )







							@require_ftfy
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Tuple:
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     self.tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )
																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(lowerCamelCase_ ,   **lowerCamelCase_       )

																												lowerCAmelCase__	       =     '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
																												lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																												# with Tilde) encoded in 2 different ways
																												lowerCAmelCase__	       =     '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
																												lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																												lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																												self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on unicode of space type
																												lowerCAmelCase__	       =     [
																												    '''\u0009''',  # (horizontal tab, '\t')
																												    '''\u000B''',  # (vertical tab)
																												    '''\u000C''',  # (form feed)
																												    '''\u0020''',  # (space, ' ')
																												    '''\u200E''',  # (left-to-right mark):w
																												    '''\u200F''',  # (right-to-left mark)
																												]
																												for unicode_seq in spaces_unicodes:
																																			lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																																			lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																																			self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )

																												# Test that the tokenization is identical on unicode of line break type
																												lowerCAmelCase__	       =     [
																												    '''\u000A''',  # (line feed, '\n')
																												    '''\r\n''',  # (carriage return and line feed, '\r\n')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\r''',  # (carriage return, '\r')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\u2028''',  # (line separator)
																												    '''\u2029''',  # (paragraph separator)
																												    # "\u0085", # (next line)
																												]

																												# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																												# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																												# space (and thus into an empty list).

																												for unicode_seq in line_break_unicodes:
																																			lowerCAmelCase__	       =     tokenizer_s.tokenize(lowerCamelCase_       )
																																			lowerCAmelCase__	       =     tokenizer_r.tokenize(lowerCamelCase_       )

																																			self.assertListEqual(lowerCamelCase_ ,   lowerCamelCase_       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							str:
														# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space`
														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
																												lowerCAmelCase__	       =     '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																												lowerCAmelCase__	       =     F"""{text_of_1_token} {text_of_1_token}"""

																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(
																												    lowerCamelCase_ ,   use_fast=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     tokenizer_r(lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												self.assertEqual(encoding.offset_mapping[0] ,   (0, len(lowerCamelCase_       ))       )
																												self.assertEqual(
																												    encoding.offset_mapping[1] ,   (len(lowerCamelCase_       ) + 1, len(lowerCamelCase_       ) + 1 + len(lowerCamelCase_       )) ,   )

																												lowerCAmelCase__	       =     F""" {text}"""

																												lowerCAmelCase__	       =     self.rust_tokenizer_class.from_pretrained(
																												    lowerCamelCase_ ,   use_fast=lowerCamelCase_ ,   )
																												lowerCAmelCase__	       =     tokenizer_r(lowerCamelCase_ ,   return_offsets_mapping=lowerCamelCase_ ,   add_special_tokens=lowerCamelCase_       )
																												self.assertEqual(encoding.offset_mapping[0] ,   (1, 1 + len(lowerCamelCase_       ))       )
																												self.assertEqual(
																												    encoding.offset_mapping[1] ,   (1 + len(lowerCamelCase_       ) + 1, 1 + len(lowerCamelCase_       ) + 1 + len(lowerCamelCase_       )) ,   )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														# Test related to the breaking change introduced in transformers v4.17.0
														# We need to check that an error in raised when the user try to load a previous version of the tokenizer.
														with self.assertRaises(lowerCamelCase_       ) as context:
																					self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''       )

														self.assertTrue(
														    context.exception.args[0].startswith(
														        '''The `backend_tokenizer` provided does not match the expected format.'''       )       )







							@require_ftfy
							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														super().test_tokenization_python_rust_equals()







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Any:
														# CLIP always lower cases letters
														pass