




from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
     import PIL


__a          =			logging.get_logger(__name__)







def lowerCamelCase__       (      _lowercase				):

     '''simple docstring'''


     if isinstance(_lowercase			,						(list, tuple)				) and isinstance(videos[0]			,						(list, tuple)				) and is_valid_image(videos[0][0]				):
          return videos

     elif isinstance(_lowercase			,						(list, tuple)				) and is_valid_image(videos[0]				):
          return [videos]

     elif is_valid_image(_lowercase				):
          return [[videos]]

     raise ValueError(f'''Could not make batched video from {videos}'''				)



class   __a(					_a       ):






    """simple docstring"""





    lowerCAmelCase        =				['''pixel_values''']





    def __init__(				self       ,_SCREAMING_SNAKE_CASE = True       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = PILImageResampling.BILINEAR       ,_SCREAMING_SNAKE_CASE = True       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = True       ,_SCREAMING_SNAKE_CASE = 1 / 255       ,_SCREAMING_SNAKE_CASE = True       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,**_SCREAMING_SNAKE_CASE       ,)			->		None:
         super().__init__(**_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							int       =      size if size is not None else {'''shortest_edge''': 224}
         UpperCAmelCase_      :							Any       =      get_size_dict(_SCREAMING_SNAKE_CASE       ,default_to_square=_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							Any       =      crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
         UpperCAmelCase_      :							List[str]       =      get_size_dict(_SCREAMING_SNAKE_CASE       ,param_name='''crop_size'''  )

         UpperCAmelCase_      :							str       =      do_resize
         UpperCAmelCase_      :							Union[str, Any]       =      size
         UpperCAmelCase_      :							int       =      do_center_crop
         UpperCAmelCase_      :							List[str]       =      crop_size
         UpperCAmelCase_      :							Optional[int]       =      resample
         UpperCAmelCase_      :							List[Any]       =      do_rescale
         UpperCAmelCase_      :							Tuple       =      rescale_factor
         UpperCAmelCase_      :							Optional[Any]       =      do_normalize
         UpperCAmelCase_      :							int       =      image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
         UpperCAmelCase_      :							Optional[int]       =      image_std if image_std is not None else IMAGENET_STANDARD_STD





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE = PILImageResampling.BILINEAR       ,_SCREAMING_SNAKE_CASE = None       ,**_SCREAMING_SNAKE_CASE       ,)			->		np.ndarray:
         UpperCAmelCase_      :							Optional[int]       =      get_size_dict(_SCREAMING_SNAKE_CASE       ,default_to_square=_SCREAMING_SNAKE_CASE  )
         if "shortest_edge" in size:
              UpperCAmelCase_      :							Dict       =      get_resize_output_image_size(_SCREAMING_SNAKE_CASE       ,size['''shortest_edge''']       ,default_to_square=_SCREAMING_SNAKE_CASE  )
         elif "height" in size and "width" in size:
              UpperCAmelCase_      :							Tuple       =      (size['''height'''], size['''width'''])
         else:
              raise ValueError(f'''Size must have \'height\' and \'width\' or \'shortest_edge\' as keys. Got {size.keys()}'''  )
         return resize(_SCREAMING_SNAKE_CASE       ,size=_SCREAMING_SNAKE_CASE       ,resample=_SCREAMING_SNAKE_CASE       ,data_format=_SCREAMING_SNAKE_CASE       ,**_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE = None       ,**_SCREAMING_SNAKE_CASE       ,)			->		np.ndarray:
         UpperCAmelCase_      :							str       =      get_size_dict(_SCREAMING_SNAKE_CASE  )
         if "height" not in size or "width" not in size:
              raise ValueError(f'''Size must have \'height\' and \'width\' as keys. Got {size.keys()}'''  )
         return center_crop(_SCREAMING_SNAKE_CASE       ,size=(size['''height'''], size['''width'''])       ,data_format=_SCREAMING_SNAKE_CASE       ,**_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE = None       ,**_SCREAMING_SNAKE_CASE       ,)			->		Dict:
         return rescale(_SCREAMING_SNAKE_CASE       ,scale=_SCREAMING_SNAKE_CASE       ,data_format=_SCREAMING_SNAKE_CASE       ,**_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE = None       ,**_SCREAMING_SNAKE_CASE       ,)			->		np.ndarray:
         return normalize(_SCREAMING_SNAKE_CASE       ,mean=_SCREAMING_SNAKE_CASE       ,std=_SCREAMING_SNAKE_CASE       ,data_format=_SCREAMING_SNAKE_CASE       ,**_SCREAMING_SNAKE_CASE  )





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = ChannelDimension.FIRST       ,)			->		np.ndarray:
         if do_resize and size is None or resample is None:
              raise ValueError('''Size and resample must be specified if do_resize is True.'''  )

         if do_center_crop and crop_size is None:
              raise ValueError('''Crop size must be specified if do_center_crop is True.'''  )

         if do_rescale and rescale_factor is None:
              raise ValueError('''Rescale factor must be specified if do_rescale is True.'''  )

         if do_normalize and (image_mean is None or image_std is None):
              raise ValueError('''Image mean and std must be specified if do_normalize is True.'''  )

         # All transformations expect numpy arrays.
         UpperCAmelCase_      :							Any       =      to_numpy_array(_SCREAMING_SNAKE_CASE  )

         if do_resize:
              UpperCAmelCase_      :							Union[str, Any]       =      self.resize(image=_SCREAMING_SNAKE_CASE       ,size=_SCREAMING_SNAKE_CASE       ,resample=_SCREAMING_SNAKE_CASE  )

         if do_center_crop:
              UpperCAmelCase_      :							Optional[int]       =      self.center_crop(_SCREAMING_SNAKE_CASE       ,size=_SCREAMING_SNAKE_CASE  )

         if do_rescale:
              UpperCAmelCase_      :							str       =      self.rescale(image=_SCREAMING_SNAKE_CASE       ,scale=_SCREAMING_SNAKE_CASE  )

         if do_normalize:
              UpperCAmelCase_      :							List[Any]       =      self.normalize(image=_SCREAMING_SNAKE_CASE       ,mean=_SCREAMING_SNAKE_CASE       ,std=_SCREAMING_SNAKE_CASE  )

         UpperCAmelCase_      :							Optional[int]       =      to_channel_dimension_format(_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE  )
         return image





    def 							a__				(				self       ,_SCREAMING_SNAKE_CASE       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = None       ,_SCREAMING_SNAKE_CASE = ChannelDimension.FIRST       ,**_SCREAMING_SNAKE_CASE       ,)			->		PIL.Image.Image:
         UpperCAmelCase_      :							Dict       =      do_resize if do_resize is not None else self.do_resize
         UpperCAmelCase_      :							int       =      resample if resample is not None else self.resample
         UpperCAmelCase_      :							List[str]       =      do_center_crop if do_center_crop is not None else self.do_center_crop
         UpperCAmelCase_      :							Optional[Any]       =      do_rescale if do_rescale is not None else self.do_rescale
         UpperCAmelCase_      :							List[Any]       =      rescale_factor if rescale_factor is not None else self.rescale_factor
         UpperCAmelCase_      :							Tuple       =      do_normalize if do_normalize is not None else self.do_normalize
         UpperCAmelCase_      :							Optional[int]       =      image_mean if image_mean is not None else self.image_mean
         UpperCAmelCase_      :							Optional[int]       =      image_std if image_std is not None else self.image_std

         UpperCAmelCase_      :							List[str]       =      size if size is not None else self.size
         UpperCAmelCase_      :							Optional[int]       =      get_size_dict(_SCREAMING_SNAKE_CASE       ,default_to_square=_SCREAMING_SNAKE_CASE  )
         UpperCAmelCase_      :							List[str]       =      crop_size if crop_size is not None else self.crop_size
         UpperCAmelCase_      :							Any       =      get_size_dict(_SCREAMING_SNAKE_CASE       ,param_name='''crop_size'''  )

         if not valid_images(_SCREAMING_SNAKE_CASE  ):
              raise ValueError(
                  '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                  '''torch.Tensor, tf.Tensor or jax.ndarray.'''  )

         UpperCAmelCase_      :							List[Any]       =      make_batched(_SCREAMING_SNAKE_CASE  )

         UpperCAmelCase_      :							str       =      [
             [
                 self._preprocess_image(
                     image=_SCREAMING_SNAKE_CASE       ,do_resize=_SCREAMING_SNAKE_CASE       ,size=_SCREAMING_SNAKE_CASE       ,resample=_SCREAMING_SNAKE_CASE       ,do_center_crop=_SCREAMING_SNAKE_CASE       ,crop_size=_SCREAMING_SNAKE_CASE       ,do_rescale=_SCREAMING_SNAKE_CASE       ,rescale_factor=_SCREAMING_SNAKE_CASE       ,do_normalize=_SCREAMING_SNAKE_CASE       ,image_mean=_SCREAMING_SNAKE_CASE       ,image_std=_SCREAMING_SNAKE_CASE       ,data_format=_SCREAMING_SNAKE_CASE       ,)
                 for img in video
             ]
             for video in videos
         ]

         UpperCAmelCase_      :							Any       =      {'''pixel_values''': videos}
         return BatchFeature(data=_SCREAMING_SNAKE_CASE       ,tensor_type=_SCREAMING_SNAKE_CASE  )





from dataclasses import dataclass, field
from typing import Optional

@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Model name or path of model to be trained.'''}       )
    lowerCAmelCase        =				field(
        default='''./'''    ,						metadata={'''help''': '''Save dir where model repo is cloned and models updates are saved to.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-train'''    ,						metadata={'''help''': '''Name or path of training dataset.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-valid'''    ,						metadata={'''help''': '''Name or path of validation dataset.'''}       )
    lowerCAmelCase        =				field(default=2    ,						metadata={'''help''': '''Batch size for training.'''}       )
    lowerCAmelCase        =				field(default=2    ,						metadata={'''help''': '''Batch size for evaluation.'''}       )
    lowerCAmelCase        =				field(default=0.1    ,						metadata={'''help''': '''Value of weight decay.'''}       )
    lowerCAmelCase        =				field(
        default=1_0000    ,						metadata={'''help''': '''Size of buffer used to shuffle streaming dataset.'''}       )
    lowerCAmelCase        =				field(default=2E-4    ,						metadata={'''help''': '''Learning rate fo training.'''}       )
    lowerCAmelCase        =				field(default='''cosine'''    ,						metadata={'''help''': '''Learning rate.'''}       )
    lowerCAmelCase        =				field(
        default=750    ,						metadata={'''help''': '''Number of warmup steps in the learning rate schedule.'''}       )
    lowerCAmelCase        =				field(
        default=16    ,						metadata={'''help''': '''Number of gradient accumulation steps.'''}       )
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''Use gradient checkpointing to reduce memory footprint.'''}       )
    lowerCAmelCase        =				field(default=5_0000    ,						metadata={'''help''': '''Maximum number of training steps.'''}       )
    lowerCAmelCase        =				field(
        default=-1    ,						metadata={'''help''': '''Maximum number of evaluation steps. If -1 the full dataset is evaluated.'''}       )
    lowerCAmelCase        =				field(default=1024    ,						metadata={'''help''': '''Sequence lengths used for training.'''}       )
    lowerCAmelCase        =				field(default=1    ,						metadata={'''help''': '''Training seed.'''}       )
    lowerCAmelCase        =				field(
        default=1024    ,						metadata={'''help''': '''Interval to save checkpoints. Measured as number of forward passes not training steps.'''}    ,						)
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''States path if the training should continue from a checkpoint folder.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''If True the data is pretokenized.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Model name or path of model to be evaluated.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-valid'''    ,						metadata={'''help''': '''Name or path of validation dataset.'''}       )
    lowerCAmelCase        =				field(default=2    ,						metadata={'''help''': '''Batch size used for evaluation.'''}       )
    lowerCAmelCase        =				field(
        default=-1    ,						metadata={'''help''': '''Maximum number of evaluation steps. If -1 the full dataset is evaluated.'''}       )
    lowerCAmelCase        =				field(default=1024    ,						metadata={'''help''': '''Length of sequences to be evaluated.'''}       )
    lowerCAmelCase        =				field(default=1    ,						metadata={'''help''': '''Random seed used for evaluation.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Model name or path of model to be evaluated.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Number of workers used for code evaluation.'''}       )
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''The number of human-eval tasks to run. If not included all tasks are evaluated.'''}    ,						)
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''Sample from the language model\'s output distribution.'''}       )
    lowerCAmelCase        =				field(default=0.2    ,						metadata={'''help''': '''Sampling temperature used for generation.'''}       )
    lowerCAmelCase        =				field(default=256    ,						metadata={'''help''': '''Maximum number of newly generated tokens.'''}       )
    lowerCAmelCase        =				field(default=0    ,						metadata={'''help''': '''Top-k parameter used for generation.'''}       )
    lowerCAmelCase        =				field(default=0.95    ,						metadata={'''help''': '''Top-p parameter used for nucleus sampling.'''}       )
    lowerCAmelCase        =				field(default=10    ,						metadata={'''help''': '''Number of generations to run in parallel.'''}       )
    lowerCAmelCase        =				field(
        default=200    ,						metadata={'''help''': '''Number of completions to generate for each sample.'''}       )
    lowerCAmelCase        =				field(default=1    ,						metadata={'''help''': '''Random seed used for evaluation.'''}       )
    lowerCAmelCase        =				field(
        default='''eval_results.json'''    ,						metadata={'''help''': '''Random seed used for evaluation.'''}       )
    lowerCAmelCase        =				field(
        default='''0'''    ,						metadata={'''help''': '''Allow `code_eval` to execute Python code on machine'''}       )
    lowerCAmelCase        =				field(
        default=-1    ,						metadata={
            '''help''': (
                '''Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive'''
                ''' number corresponds to which GPU device id to run on.'''
            )
        }    ,						)


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default=_a    ,						metadata={
            '''help''': '''The number of CPU cores to use for parallel preprocessing. Default uses the maximum available.'''
        }    ,						)
    lowerCAmelCase        =				field(
        default='''transformersbook/codeparrot'''    ,						metadata={'''help''': '''Folder or name of dataset to process.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot-clean'''    ,						metadata={'''help''': '''Folder to save processed processed dataset.'''}       )
    lowerCAmelCase        =				field(
        default=10_0000    ,						metadata={'''help''': '''Number of files to save per JSON output file.'''}       )
    lowerCAmelCase        =				field(default='''content'''    ,						metadata={'''help''': '''Column containing text data to process.'''}       )
    lowerCAmelCase        =				field(
        default=1000    ,						metadata={'''help''': '''Maximum line length in file, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=100    ,						metadata={'''help''': '''Maximum mean line length in file, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=0.25    ,						metadata={'''help''': '''Maximum fraction of non-alphanumeric characters, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=1.5    ,						metadata={'''help''': '''Minimum character token ratio for the file, otherwise file is filtered.'''}       )
    lowerCAmelCase        =				field(
        default=0.7    ,						metadata={'''help''': '''Probability for filtering config, test and uncommon files.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Name or path to the tokenizer.'''}    ,						)
    lowerCAmelCase        =				field(
        default=_a    ,						metadata={'''help''': '''If True, near-duplicate samples are removed.'''}       )
    lowerCAmelCase        =				field(
        default=0.85    ,						metadata={'''help''': '''Jaccard threshold for near-duplicate samples.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''gpt2'''    ,						metadata={'''help''': '''Base tokenizer to build new tokenizer from.'''}       )
    lowerCAmelCase        =				field(
        default='''transformersbook/codeparrot-train'''    ,						metadata={'''help''': '''Dataset to train tokenizer on.'''}       )
    lowerCAmelCase        =				field(default='''content'''    ,						metadata={'''help''': '''Column containing text data to process.'''}       )
    lowerCAmelCase        =				field(default=20_0000    ,						metadata={'''help''': '''Number of examples to train tokenizer on.'''}       )
    lowerCAmelCase        =				field(
        default=3_2768    ,						metadata={'''help''': '''Number of examples to train the tokenizer on.'''}       )
    lowerCAmelCase        =				field(default='''codeparrot'''    ,						metadata={'''help''': '''Name of new tokenizer.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Push saved tokenizer to the hub.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Name or path to the tokenizer.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot-clean-train'''    ,						metadata={'''help''': '''Name or path to the dataset to pretokenize.'''}       )
    lowerCAmelCase        =				field(
        default='''tokenized-codeparrot-train'''    ,						metadata={'''help''': '''Repo name of the pretokenized data.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Number of workers used for code evaluation.'''}       )


@dataclass
class   __a:






    """simple docstring"""





    lowerCAmelCase        =				field(
        default='''gpt2-large'''    ,						metadata={'''help''': '''Configuration to use for model initialization.'''}       )
    lowerCAmelCase        =				field(
        default='''codeparrot/codeparrot'''    ,						metadata={'''help''': '''Tokenizer attached to model.'''}       )
    lowerCAmelCase        =				field(default='''codeparrot'''    ,						metadata={'''help''': '''Name of the created model.'''}       )
    lowerCAmelCase        =				field(default=_a    ,						metadata={'''help''': '''Push saved tokenizer to the hub.'''}       )