





import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin

class 		SCREAMING_SNAKE_CASE__    (	_UpperCAmelCase  ):
      A_						:    Tuple							=   'MCTCTFeatureExtractor'
      A_						:    List[Any]							=   'AutoTokenizer'


      def __init__(self	:				Tuple    ,      a__	:				Tuple    ,      a__	:				Optional[int]				):





         """simple docstring"""



         super().__init__(a__    ,      a__				)
         __snake_case								=       self.feature_extractor
         __snake_case								=       False


      def __call__(self	:				Union[str, Any]    ,      *a__	:				List[str]    ,      **a__	:				str				):





         """simple docstring"""



         if self._in_target_context_manager:
            return self.current_processor(*a__    ,      **a__				)

         if "raw_speech" in kwargs:
            warnings.warn('''Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead.'''				)
            __snake_case								=       kwargs.pop('''raw_speech'''				)
         else:
            __snake_case								=       kwargs.pop('''audio'''    ,      a__				)
         __snake_case								=       kwargs.pop('''sampling_rate'''    ,      a__				)
         __snake_case								=       kwargs.pop('''text'''    ,      a__				)
         if len(a__				) > 0:
            __snake_case								=       args[0]
            __snake_case								=       args[1:]

         if audio is None and text is None:
            raise ValueError('''You need to specify either an `audio` or `text` input to process.'''				)

         if audio is not None:
            __snake_case								=       self.feature_extractor(a__    ,      *a__    ,      sampling_rate=a__    ,      **a__				)
         if text is not None:
            __snake_case								=       self.tokenizer(a__    ,      **a__				)

         if text is None:
            return inputs
         elif audio is None:
            return encodings
         else:
            __snake_case								=       encodings['''input_ids''']
            return inputs


      def        a			(self	:				Dict    ,      *a__	:				str    ,      **a__	:				Tuple				):





         """simple docstring"""



         return self.tokenizer.batch_decode(*a__    ,      **a__				)


      def        a			(self	:				List[Any]    ,      *a__	:				Tuple    ,      **a__	:				List[str]				):





         """simple docstring"""



         if self._in_target_context_manager:
            return self.current_processor.pad(*a__    ,      **a__				)

         __snake_case								=       kwargs.pop('''input_features'''    ,      a__				)
         __snake_case								=       kwargs.pop('''labels'''    ,      a__				)
         if len(a__				) > 0:
            __snake_case								=       args[0]
            __snake_case								=       args[1:]

         if input_features is not None:
            __snake_case								=       self.feature_extractor.pad(a__    ,      *a__    ,      **a__				)
         if labels is not None:
            __snake_case								=       self.tokenizer.pad(a__    ,      **a__				)

         if labels is None:
            return input_features
         elif input_features is None:
            return labels
         else:
            __snake_case								=       labels['''input_ids''']
            return input_features


      def        a			(self	:				int    ,      *a__	:				Any    ,      **a__	:				List[Any]				):





         """simple docstring"""



         return self.tokenizer.decode(*a__    ,      **a__				)


      @contextmanager
      def        a			(self	:				int				):





         """simple docstring"""



         warnings.warn(
             '''`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your '''
             '''labels by using the argument `text` of the regular `__call__` method (either in the same call as '''
             '''your audio inputs, or in a separate call.'''				)
         __snake_case								=       True
         __snake_case								=       self.tokenizer
         yield
         __snake_case								=       self.feature_extractor
         __snake_case								=       False








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_           =      data_utils.TransfoXLTokenizer
snake_case_           =      data_utils.TransfoXLCorpus
snake_case_           =      data_utils
snake_case_           =      data_utils






def lowerCamelCase__   (							snake_case_     :  Union[str, Any]	,				snake_case_     :  int	,				snake_case_     :  List[Any]	,				snake_case_     :  int	)				->    Dict:
   if transfo_xl_dataset_file:
      # Convert a pre-processed corpus (see original TensorFlow repo)
      with open(snake_case_	,				'''rb'''	) as fp:
         __snake_case								=       pickle.load(snake_case_	,				encoding='''latin1'''	)
      # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
      print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""	)
      __snake_case								=       corpus.vocab.__dict__
      torch.save(snake_case_	,				snake_case_	)

      __snake_case								=       corpus.__dict__
      corpus_dict_no_vocab.pop('''vocab'''	,				snake_case_	)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + CORPUS_NAME
      print(f"""Save dataset to {pytorch_dataset_dump_path}"""	)
      torch.save(snake_case_	,				snake_case_	)

   if tf_checkpoint_path:
      # Convert a pre-trained TensorFlow model
      __snake_case								=       os.path.abspath(snake_case_	)
      __snake_case								=       os.path.abspath(snake_case_	)

      print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""	)
      # Initialise PyTorch model
      if transfo_xl_config_file == "":
         __snake_case								=       TransfoXLConfig()
      else:
         __snake_case								=       TransfoXLConfig.from_json_file(snake_case_	)
      print(f"""Building PyTorch model from configuration: {config}"""	)
      __snake_case								=       TransfoXLLMHeadModel(snake_case_	)

      __snake_case								=       load_tf_weights_in_transfo_xl(snake_case_	,				snake_case_	,				snake_case_	)
      # Save pytorch-model
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      print(f"""Save PyTorch model to {os.path.abspath(snake_case_	)}"""	)
      torch.save(model.state_dict()	,				snake_case_	)
      print(f"""Save configuration file to {os.path.abspath(snake_case_	)}"""	)
      with open(snake_case_	,				'''w'''	,				encoding='''utf-8'''	) as f:
         f.write(config.to_json_string()	)


if __name__ == "__main__":
    snake_case_           =      argparse.ArgumentParser()
    parser.add_argument(
        '--pytorch_dump_folder_path',
        default=None,
        type=str,
        required=True,
        help='Path to the folder to store the PyTorch model or dataset/vocab.',
    )
    parser.add_argument(
        '--tf_checkpoint_path',
        default='',
        type=str,
        help='An optional path to a TensorFlow checkpoint path to be converted.',
    )
    parser.add_argument(
        '--transfo_xl_config_file',
        default='',
        type=str,
        help=(
            'An optional config json file corresponding to the pre-trained BERT model. \n'
            'This specifies the model architecture.'
        ),
    )
    parser.add_argument(
        '--transfo_xl_dataset_file',
        default='',
        type=str,
        help='An optional dataset file to be converted in a vocabulary.',
    )
    snake_case_           =      parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )

