





import importlib
import os

import fsspec
import pytest
from fsspec import register_implementation
from fsspec.registry import _registry as _fsspec_registry

from datasets.filesystems import COMPRESSION_FILESYSTEMS, HfFileSystem, extract_path_from_uri, is_remote_filesystem

from .utils import require_lza, require_zstandard






def lowerCamelCase__   (							snake_case_     :  Optional[Any]	)				->    int:
   assert "mock" in _fsspec_registry
   assert "bz2" in _fsspec_registry






def lowerCamelCase__   (							)				->    List[Any]:
   assert "mock" not in _fsspec_registry
   assert "bz2" in _fsspec_registry






def lowerCamelCase__   (							)				->    List[Any]:
   __snake_case								=       '''mock-s3-bucket'''
   __snake_case								=       f"""s3://{mock_bucket}"""
   __snake_case								=       extract_path_from_uri(snake_case_	)
   assert dataset_path.startswith('''s3://'''	) is False

   __snake_case								=       '''./local/path'''
   __snake_case								=       extract_path_from_uri(snake_case_	)
   assert dataset_path == new_dataset_path






def lowerCamelCase__   (							snake_case_     :  int	)				->    str:
   __snake_case								=       is_remote_filesystem(snake_case_	)
   assert is_remote is True

   __snake_case								=       fsspec.filesystem('''file'''	)

   __snake_case								=       is_remote_filesystem(snake_case_	)
   assert is_remote is False






@pytest.mark.parametrize('''compression_fs_class'''	,				snake_case_	)
def lowerCamelCase__   (							snake_case_     :  int	,				snake_case_     :  Dict	,				snake_case_     :  Any	,				snake_case_     :  Optional[int]	,				snake_case_     :  Optional[int]	,				snake_case_     :  Dict	,				snake_case_     :  Dict	)				->    str:
   __snake_case								=       {'''gzip''': gz_file, '''xz''': xz_file, '''zstd''': zstd_file, '''bz2''': bza_file, '''lz4''': lza_file}
   __snake_case								=       input_paths[compression_fs_class.protocol]
   if input_path is None:
      __snake_case								=       f"""for '{compression_fs_class.protocol}' compression protocol, """
      if compression_fs_class.protocol == "lz4":
         reason += require_lza.kwargs["reason"]
      elif compression_fs_class.protocol == "zstd":
         reason += require_zstandard.kwargs["reason"]
      pytest.skip(snake_case_	)
   __snake_case								=       fsspec.filesystem(compression_fs_class.protocol	,				fo=snake_case_	)
   assert isinstance(snake_case_	,				snake_case_	)
   __snake_case								=       os.path.basename(snake_case_	)
   __snake_case								=       expected_filename[: expected_filename.rindex('''.'''	)]
   assert fs.glob('''*'''	) == [expected_filename]
   with fs.open(snake_case_	,				'''r'''	,				encoding='''utf-8'''	) as f, open(snake_case_	,				encoding='''utf-8'''	) as expected_file:
      assert f.read() == expected_file.read()






@pytest.mark.parametrize('''protocol'''	,				['''zip''', '''gzip''']	)
def lowerCamelCase__   (							snake_case_     :  Any	,				snake_case_     :  Dict	,				snake_case_     :  str	)				->    Optional[Any]:
   __snake_case								=       {'''zip''': zip_jsonl_path, '''gzip''': jsonl_gz_path}
   __snake_case								=       compressed_file_paths[protocol]
   __snake_case								=       '''dataset.jsonl'''
   __snake_case								=       f"""{protocol}://{member_file_path}::{compressed_file_path}"""
   __snake_case	,     *__snake_case								=       fsspec.get_fs_token_paths(snake_case_	)
   assert fs.isfile(snake_case_	)
   assert not fs.isfile('''non_existing_''' + member_file_path	)






@pytest.mark.integration
def lowerCamelCase__   (							snake_case_     :  Optional[int]	,				snake_case_     :  Dict	,				snake_case_     :  Optional[Any]	,				snake_case_     :  Optional[Any]	)				->    Optional[Any]:
   __snake_case								=       hf_api.dataset_info(snake_case_	,				token=snake_case_	)
   __snake_case								=       HfFileSystem(repo_info=snake_case_	,				token=snake_case_	)
   assert sorted(hffs.glob('''*'''	)	) == [".gitattributes", "data"]
   assert hffs.isdir('''data'''	)
   assert hffs.isfile('''.gitattributes'''	) and hffs.isfile('''data/text_data.txt'''	)
   with open(snake_case_	) as f:
      assert hffs.open('''data/text_data.txt'''	,				'''r'''	).read() == f.read()







def lowerCamelCase__   (							)				->    int:
   __snake_case								=       '''bz2'''

   # Import module
   import datasets.filesystems

   # Overwrite protocol and reload
   register_implementation(snake_case_	,				snake_case_	,				clobber=snake_case_	)
   with pytest.warns(snake_case_	) as warning_info:
      importlib.reload(datasets.filesystems	)

   assert len(snake_case_	) == 1
   assert (
       str(warning_info[0].message	)
       == f"""A filesystem protocol was already set for {protocol} and will be overwritten."""
   )








import argparse
import os
import pickle
import sys

import torch

from transformers import TransfoXLConfig, TransfoXLLMHeadModel, load_tf_weights_in_transfo_xl
from transformers.models.transfo_xl import tokenization_transfo_xl as data_utils
from transformers.models.transfo_xl.tokenization_transfo_xl import CORPUS_NAME, VOCAB_FILES_NAMES
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


logging.set_verbosity_info()

# We do this to be able to load python 2 datasets pickles
# See e.g. https://stackoverflow.com/questions/2121874/python-pickling-after-changing-a-modules-directory/2121918#2121918
snake_case_           =      data_utils.TransfoXLTokenizer
snake_case_           =      data_utils.TransfoXLCorpus
snake_case_           =      data_utils
snake_case_           =      data_utils






def lowerCamelCase__   (							snake_case_     :  Union[str, Any]	,				snake_case_     :  int	,				snake_case_     :  List[Any]	,				snake_case_     :  int	)				->    Dict:
   if transfo_xl_dataset_file:
      # Convert a pre-processed corpus (see original TensorFlow repo)
      with open(snake_case_	,				'''rb'''	) as fp:
         __snake_case								=       pickle.load(snake_case_	,				encoding='''latin1'''	)
      # Save vocabulary and dataset cache as Dictionaries (should be better than pickles for the long-term)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + VOCAB_FILES_NAMES['''pretrained_vocab_file''']
      print(f"""Save vocabulary to {pytorch_vocab_dump_path}"""	)
      __snake_case								=       corpus.vocab.__dict__
      torch.save(snake_case_	,				snake_case_	)

      __snake_case								=       corpus.__dict__
      corpus_dict_no_vocab.pop('''vocab'''	,				snake_case_	)
      __snake_case								=       pytorch_dump_folder_path + '''/''' + CORPUS_NAME
      print(f"""Save dataset to {pytorch_dataset_dump_path}"""	)
      torch.save(snake_case_	,				snake_case_	)

   if tf_checkpoint_path:
      # Convert a pre-trained TensorFlow model
      __snake_case								=       os.path.abspath(snake_case_	)
      __snake_case								=       os.path.abspath(snake_case_	)

      print(f"""Converting Transformer XL checkpoint from {tf_path} with config at {config_path}."""	)
      # Initialise PyTorch model
      if transfo_xl_config_file == "":
         __snake_case								=       TransfoXLConfig()
      else:
         __snake_case								=       TransfoXLConfig.from_json_file(snake_case_	)
      print(f"""Building PyTorch model from configuration: {config}"""	)
      __snake_case								=       TransfoXLLMHeadModel(snake_case_	)

      __snake_case								=       load_tf_weights_in_transfo_xl(snake_case_	,				snake_case_	,				snake_case_	)
      # Save pytorch-model
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      __snake_case								=       os.path.join(snake_case_	,				snake_case_	)
      print(f"""Save PyTorch model to {os.path.abspath(snake_case_	)}"""	)
      torch.save(model.state_dict()	,				snake_case_	)
      print(f"""Save configuration file to {os.path.abspath(snake_case_	)}"""	)
      with open(snake_case_	,				'''w'''	,				encoding='''utf-8'''	) as f:
         f.write(config.to_json_string()	)


if __name__ == "__main__":
    snake_case_           =      argparse.ArgumentParser()
    parser.add_argument(
        '--pytorch_dump_folder_path',
        default=None,
        type=str,
        required=True,
        help='Path to the folder to store the PyTorch model or dataset/vocab.',
    )
    parser.add_argument(
        '--tf_checkpoint_path',
        default='',
        type=str,
        help='An optional path to a TensorFlow checkpoint path to be converted.',
    )
    parser.add_argument(
        '--transfo_xl_config_file',
        default='',
        type=str,
        help=(
            'An optional config json file corresponding to the pre-trained BERT model. \n'
            'This specifies the model architecture.'
        ),
    )
    parser.add_argument(
        '--transfo_xl_dataset_file',
        default='',
        type=str,
        help='An optional dataset file to be converted in a vocabulary.',
    )
    snake_case_           =      parser.parse_args()
    convert_transfo_xl_checkpoint_to_pytorch(
        args.tf_checkpoint_path,
        args.transfo_xl_config_file,
        args.pytorch_dump_folder_path,
        args.transfo_xl_dataset_file,
    )

