



import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger






class 							lowercase__     (							lowercase       ):





	def __init__(			self :    int			,lowerCamelCase__ :    Optional[int]			,lowerCamelCase__ :    Optional[int]			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       params

							_UpperCamelCase :		Optional[Any]			    =       np.array(lowerCamelCase__			)
							_UpperCamelCase :		Tuple			    =       np.array([len(lowerCamelCase__			) for t in data]			)

							self.check()
							self.remove_long_sequences()
							self.remove_empty_sequences()
							self.remove_unknown_sequences()
							self.check()
							self.print_statistics()





	def __getitem__(			self :    Dict			,lowerCamelCase__ :    Tuple			):







							'''simple docstring'''
							return (self.token_ids[index], self.lengths[index])





	def __len__(			self :    str			):







							'''simple docstring'''
							return len(self.lengths			)





	def      UpperCamelCase_  (			self :    Tuple			):







							'''simple docstring'''
							assert len(self.token_ids			) == len(self.lengths			)
							assert all(self.lengths[i] == len(self.token_ids[i]			) for i in range(len(self.lengths			)			)			)





	def      UpperCamelCase_  (			self :    int			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       self.params.max_model_input_size
							_UpperCamelCase :		Any			    =       self.lengths > max_len
							logger.info(F'Splitting {sum(lowerCamelCase__			)} too long sequences.'			)

							def divide_chunks(lowerCamelCase__ :    Dict			,lowerCamelCase__ :    Optional[int]			):
													return [l[i : i + n] for i in range(0			,len(lowerCamelCase__			)			,lowerCamelCase__			)]

							_UpperCamelCase :		Optional[Any]			    =       []
							_UpperCamelCase :		List[Any]			    =       []
							if self.params.mlm:
													_UpperCamelCase :		int			    =       self.params.special_tok_ids['cls_token'], self.params.special_tok_ids['sep_token']
							else:
													_UpperCamelCase :		Optional[Any]			    =       self.params.special_tok_ids['bos_token'], self.params.special_tok_ids['eos_token']

							for seq_, len_ in zip(self.token_ids			,self.lengths			):
													assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
													if len_ <= max_len:
																			new_tok_ids.append(seq_			)
																			new_lengths.append(len_			)
													else:
																			_UpperCamelCase :		List[str]			    =       []
																			for sub_s in divide_chunks(seq_			,max_len - 2			):
																									if sub_s[0] != cls_id:
																															_UpperCamelCase :		Tuple			    =       np.insert(lowerCamelCase__			,0			,lowerCamelCase__			)
																									if sub_s[-1] != sep_id:
																															_UpperCamelCase :		Optional[int]			    =       np.insert(lowerCamelCase__			,len(lowerCamelCase__			)			,lowerCamelCase__			)
																									assert len(lowerCamelCase__			) <= max_len
																									assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
																									sub_seqs.append(lowerCamelCase__			)

																			new_tok_ids.extend(lowerCamelCase__			)
																			new_lengths.extend([len(lowerCamelCase__			) for l in sub_seqs]			)

							_UpperCamelCase :		str			    =       np.array(lowerCamelCase__			)
							_UpperCamelCase :		Any			    =       np.array(lowerCamelCase__			)





	def      UpperCamelCase_  (			self :    Union[str, Any]			):







							'''simple docstring'''
							_UpperCamelCase :		Dict			    =       len(self			)
							_UpperCamelCase :		Any			    =       self.lengths > 11
							_UpperCamelCase :		Optional[int]			    =       self.token_ids[indices]
							_UpperCamelCase :		Optional[Any]			    =       self.lengths[indices]
							_UpperCamelCase :		Optional[Any]			    =       len(self			)
							logger.info(F'Remove {init_size - new_size} too short (<=11 tokens) sequences.'			)





	def      UpperCamelCase_  (			self :    str			):







							'''simple docstring'''
							if "unk_token" not in self.params.special_tok_ids:
													return
							else:
													_UpperCamelCase :		Tuple			    =       self.params.special_tok_ids['unk_token']
							_UpperCamelCase :		Optional[int]			    =       len(self			)
							_UpperCamelCase :		str			    =       np.array([np.count_nonzero(a == unk_token_id			) for a in self.token_ids]			)
							_UpperCamelCase :		Union[str, Any]			    =       (unk_occs / self.lengths) < 0.5
							_UpperCamelCase :		Optional[Any]			    =       self.token_ids[indices]
							_UpperCamelCase :		int			    =       self.lengths[indices]
							_UpperCamelCase :		Optional[Any]			    =       len(self			)
							logger.info(F'Remove {init_size - new_size} sequences with a high level of unknown tokens (50%).'			)





	def      UpperCamelCase_  (			self :    Tuple			):







							'''simple docstring'''
							if not self.params.is_master:
													return
							logger.info(F'{len(self			)} sequences'			)
							# data_len = sum(self.lengths)
							# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
							# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

							# unk_idx = self.params.special_tok_ids['unk_token']
							# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
							# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')





	def      UpperCamelCase_  (			self :    str			,lowerCamelCase__ :    int			):







							'''simple docstring'''
							_UpperCamelCase :		int			    =       [t[0] for t in batch]
							_UpperCamelCase :		Any			    =       [t[1] for t in batch]
							assert len(lowerCamelCase__			) == len(lowerCamelCase__			)

							# Max for paddings
							_UpperCamelCase :		Dict			    =       max(lowerCamelCase__			)

							# Pad token ids
							if self.params.mlm:
													_UpperCamelCase :		str			    =       self.params.special_tok_ids['pad_token']
							else:
													_UpperCamelCase :		Optional[Any]			    =       self.params.special_tok_ids['unk_token']
							_UpperCamelCase :		Optional[Any]			    =       [list(t.astype(lowerCamelCase__			)			) + [pad_idx] * (max_seq_len_ - len(lowerCamelCase__			)) for t in token_ids]
							assert len(tk_			) == len(lowerCamelCase__			)
							assert all(len(lowerCamelCase__			) == max_seq_len_ for t in tk_			)

							_UpperCamelCase :		List[Any]			    =       torch.tensor(tk_			)  # (bs, max_seq_len_)
							_UpperCamelCase :		Union[str, Any]			    =       torch.tensor(lowerCamelCase__			)  # (bs)
							return tk_t, lg_t







'''simple docstring'''
from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
    is_vision_available,
)


snake_case_			:     Optional[Any]  	=  {
    'configuration_clip': [
        'CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'CLIPConfig',
        'CLIPOnnxConfig',
        'CLIPTextConfig',
        'CLIPVisionConfig',
    ],
    'processing_clip': ['CLIPProcessor'],
    'tokenization_clip': ['CLIPTokenizer'],
}

try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_			:     Optional[int]  	=  ['CLIPTokenizerFast']

try:
					if not is_vision_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_			:     Optional[int]  	=  ['CLIPFeatureExtractor']
					snake_case_			:     Dict  	=  ['CLIPImageProcessor']

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_			:     Tuple  	=  [
					    'CLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'CLIPModel',
					    'CLIPPreTrainedModel',
					    'CLIPTextModel',
					    'CLIPTextModelWithProjection',
					    'CLIPVisionModel',
					    'CLIPVisionModelWithProjection',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_			:     List[Any]  	=  [
					    'TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
					    'TFCLIPModel',
					    'TFCLIPPreTrainedModel',
					    'TFCLIPTextModel',
					    'TFCLIPVisionModel',
					]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_			:     Optional[int]  	=  [
					    'FlaxCLIPModel',
					    'FlaxCLIPPreTrainedModel',
					    'FlaxCLIPTextModel',
					    'FlaxCLIPTextPreTrainedModel',
					    'FlaxCLIPVisionModel',
					    'FlaxCLIPVisionPreTrainedModel',
					]


if TYPE_CHECKING:
					from .configuration_clip import (
					    CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
					    CLIPConfig,
					    CLIPOnnxConfig,
					    CLIPTextConfig,
					    CLIPVisionConfig,
					)
					from .processing_clip import CLIPProcessor
					from .tokenization_clip import CLIPTokenizer

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_clip_fast import CLIPTokenizerFast

					try:
										if not is_vision_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .feature_extraction_clip import CLIPFeatureExtractor
										from .image_processing_clip import CLIPImageProcessor

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_clip import (
										    CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
										    CLIPModel,
										    CLIPPreTrainedModel,
										    CLIPTextModel,
										    CLIPTextModelWithProjection,
										    CLIPVisionModel,
										    CLIPVisionModelWithProjection,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_clip import (
										    TF_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFCLIPModel,
										    TFCLIPPreTrainedModel,
										    TFCLIPTextModel,
										    TFCLIPVisionModel,
										)

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_flax_clip import (
										    FlaxCLIPModel,
										    FlaxCLIPPreTrainedModel,
										    FlaxCLIPTextModel,
										    FlaxCLIPTextPreTrainedModel,
										    FlaxCLIPVisionModel,
										    FlaxCLIPVisionPreTrainedModel,
										)


else:
					import sys

					snake_case_			:     str  	=  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


