


"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCamelCase_       :       List[str]      						=  logging.get_logger(__name__)

lowerCamelCase_       :       List[Any]      						=  {}






class 				__A    (	_SCREAMING_SNAKE_CASE      ):






 """simple docstring"""




 __lowerCAmelCase			 =  "llama"
 __lowerCAmelCase			 =  ["past_key_values"]


 def __init__(		self			,   __A=3_2000			,   __A=4096			,   __A=1_1008			,   __A=32			,   __A=32			,   __A=None			,   __A="silu"			,   __A=2048			,   __A=0.02			,   __A=1E-6			,   __A=True			,   __A=0			,   __A=1			,   __A=2			,   __A=1			,   __A=False			,   __A=None			,   **__A			,   ) ->					str:
     a					=vocab_size
     a					=max_position_embeddings
     a					=hidden_size
     a					=intermediate_size
     a					=num_hidden_layers
     a					=num_attention_heads

     # for backward compatibility
     if num_key_value_heads is None:
         a					=num_attention_heads

     a					=num_key_value_heads
     a					=hidden_act
     a					=initializer_range
     a					=rms_norm_eps
     a					=pretraining_tp
     a					=use_cache
     a					=rope_scaling
     self._rope_scaling_validation()

     super().__init__(
         pad_token_id=__A			,   bos_token_id=__A			,   eos_token_id=__A			,   tie_word_embeddings=__A			,   **__A			,   )





 def 				SCREAMING_SNAKE_CASE							(		self						) ->					int:
     if self.rope_scaling is None:
         return

     if not isinstance(self.rope_scaling			,   __A						) or len(self.rope_scaling						) != 2:
         raise ValueError(
             '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
             f'''got {self.rope_scaling}'''						)
     a					=self.rope_scaling.get('''type'''			,   __A						)
     a					=self.rope_scaling.get('''factor'''			,   __A						)
     if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
         raise ValueError(
             f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''						)
     if rope_scaling_factor is None or not isinstance(__A			,   __A						) or rope_scaling_factor <= 1.0:
         raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''						)



"""simple docstring"""







import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser
def _A						(						lowercase		):







    """simple docstring"""
    a					={}
    a					=tokenizer(example['''content''']	,   truncation=lowercase		)['''input_ids''']
    a					=len(example['''content''']		) / len(output['''input_ids''']		)
    return output


lowerCamelCase_       :       Optional[int]      						=  HfArgumentParser(PretokenizationArguments)
lowerCamelCase_       :       Optional[Any]      						=  parser.parse_args()
if args.num_workers is None:
       lowerCamelCase_       :       Tuple      						=  multiprocessing.cpu_count()
lowerCamelCase_       :       Any      						=  AutoTokenizer.from_pretrained(args.tokenizer_dir)

lowerCamelCase_       :       Any      						=  time.time()
lowerCamelCase_       :       int      						=  load_dataset(args.dataset_name, split="""train""")
print(F'Dataset loaded in {time.time()-t_start:.2f}s')

lowerCamelCase_       :       List[str]      						=  time.time()
lowerCamelCase_       :       str      						=  ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(F'Dataset tokenized in {time.time()-t_start:.2f}s')

lowerCamelCase_       :       Union[str, Any]      						=  time.time()
ds.push_to_hub(args.tokenized_data_repo)





print(F'Data pushed to the hub in {time.time()-t_start:.2f}s')