


import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


__UpperCAmelCase       =							pytest.mark.integration




@require_faiss
class 		a__    (							__lowercase  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														lowerCAmelCase__	       =     Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(UpperCAmelCase__       ) for x in np.arange(30       ).tolist()]}       )
														return dset







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														import faiss

														lowerCAmelCase__	       =     self._create_dummy_dataset()
														lowerCAmelCase__	       =     dset.map(
														    lambda lowerCamelCase_ ,   lowerCamelCase_       : {"vecs": i * np.ones(5 ,   dtype=np.floataa       )} ,   with_indices=UpperCAmelCase__ ,   keep_in_memory=UpperCAmelCase__       )
														lowerCAmelCase__	       =     dset.add_faiss_index('''vecs''' ,   batch_size=1_00 ,   metric_type=faiss.METRIC_INNER_PRODUCT       )
														lowerCAmelCase__       , lowerCAmelCase__	       =     dset.get_nearest_examples('''vecs''' ,   np.ones(5 ,   dtype=np.floataa       )       )
														self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''       )
														dset.drop_index('''vecs'''       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														import faiss

														lowerCAmelCase__	       =     self._create_dummy_dataset()
														dset.add_faiss_index_from_external_arrays(
														    external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1 ,   1       ) ,   index_name='''vecs''' ,   batch_size=1_00 ,   metric_type=faiss.METRIC_INNER_PRODUCT ,   )
														lowerCAmelCase__       , lowerCAmelCase__	       =     dset.get_nearest_examples('''vecs''' ,   np.ones(5 ,   dtype=np.floataa       )       )
														self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														import faiss

														lowerCAmelCase__	       =     self._create_dummy_dataset()
														dset.add_faiss_index_from_external_arrays(
														    external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1 ,   1       ) ,   index_name='''vecs''' ,   metric_type=faiss.METRIC_INNER_PRODUCT ,   )

														# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
														# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
														# see https://bugs.python.org/issue14243 and
														# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
														with tempfile.NamedTemporaryFile(delete=UpperCAmelCase__       ) as tmp_file:
																					dset.save_faiss_index('''vecs''' ,   tmp_file.name       )
																					dset.load_faiss_index('''vecs2''' ,   tmp_file.name       )
														os.unlink(tmp_file.name       )

														lowerCAmelCase__       , lowerCAmelCase__	       =     dset.get_nearest_examples('''vecs2''' ,   np.ones(5 ,   dtype=np.floataa       )       )
														self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														lowerCAmelCase__	       =     self._create_dummy_dataset()
														dset.add_faiss_index_from_external_arrays(
														    external_arrays=np.ones((30, 5)       ) * np.arange(30       ).reshape(-1 ,   1       ) ,   index_name='''vecs'''       )
														dset.drop_index('''vecs'''       )
														self.assertRaises(UpperCAmelCase__ ,   partial(dset.get_nearest_examples ,   '''vecs2''' ,   np.ones(5 ,   dtype=np.floataa       )       )       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Optional[Any]:
														from elasticsearch import Elasticsearch

														lowerCAmelCase__	       =     self._create_dummy_dataset()
														with patch('''elasticsearch.Elasticsearch.search'''       ) as mocked_search, patch(
														    '''elasticsearch.client.IndicesClient.create'''       ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''       ) as mocked_bulk:
																					lowerCAmelCase__	       =     {'''acknowledged''': True}
																					mocked_bulk.return_value([(True, None)] * 30       )
																					lowerCAmelCase__	       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 29}]}}
																					lowerCAmelCase__	       =     Elasticsearch()

																					dset.add_elasticsearch_index('''filename''' ,   es_client=UpperCAmelCase__       )
																					lowerCAmelCase__       , lowerCAmelCase__	       =     dset.get_nearest_examples('''filename''' ,   '''my_name-train_29'''       )
																					self.assertEqual(examples['''filename'''][0] ,   '''my_name-train_29'''       )




@require_faiss
class 		a__    (							__lowercase  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[Any]:
														import faiss

														lowerCAmelCase__	       =     FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )

														# add vectors
														index.add_vectors(np.eye(5 ,   dtype=np.floataa       )       )
														self.assertIsNotNone(index.faiss_index       )
														self.assertEqual(index.faiss_index.ntotal ,   5       )
														index.add_vectors(np.zeros((5, 5) ,   dtype=np.floataa       )       )
														self.assertEqual(index.faiss_index.ntotal ,   10       )

														# single query
														lowerCAmelCase__	       =     np.zeros(5 ,   dtype=np.floataa       )
														lowerCAmelCase__	       =     1
														lowerCAmelCase__       , lowerCAmelCase__	       =     index.search(UpperCAmelCase__       )
														self.assertRaises(UpperCAmelCase__ ,   index.search ,   query.reshape(-1 ,   1       )       )
														self.assertGreater(scores[0] ,   0       )
														self.assertEqual(indices[0] ,   1       )

														# batched queries
														lowerCAmelCase__	       =     np.eye(5 ,   dtype=np.floataa       )[::-1]
														lowerCAmelCase__       , lowerCAmelCase__	       =     index.search_batch(UpperCAmelCase__       )
														self.assertRaises(UpperCAmelCase__ ,   index.search_batch ,   queries[0]       )
														lowerCAmelCase__	       =     [scores[0] for scores in total_scores]
														lowerCAmelCase__	       =     [indices[0] for indices in total_indices]
														self.assertGreater(np.min(UpperCAmelCase__       ) ,   0       )
														self.assertListEqual([4, 3, 2, 1, 0] ,   UpperCAmelCase__       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														import faiss

														lowerCAmelCase__	       =     FaissIndex(string_factory='''Flat'''       )
														index.add_vectors(np.eye(5 ,   dtype=np.floataa       )       )
														self.assertIsInstance(index.faiss_index ,   faiss.IndexFlat       )
														lowerCAmelCase__	       =     FaissIndex(string_factory='''LSH'''       )
														index.add_vectors(np.eye(5 ,   dtype=np.floataa       )       )
														self.assertIsInstance(index.faiss_index ,   faiss.IndexLSH       )
														with self.assertRaises(UpperCAmelCase__       ):
																					lowerCAmelCase__	       =     FaissIndex(string_factory='''Flat''' ,   custom_index=faiss.IndexFlat(5       )       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Dict:
														import faiss

														lowerCAmelCase__	       =     faiss.IndexFlat(5       )
														lowerCAmelCase__	       =     FaissIndex(custom_index=UpperCAmelCase__       )
														index.add_vectors(np.eye(5 ,   dtype=np.floataa       )       )
														self.assertIsInstance(index.faiss_index ,   faiss.IndexFlat       )







							def   __SCREAMING_SNAKE_CASE							(		self       )	->							List[str]:
														import faiss

														lowerCAmelCase__	       =     FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT       )
														index.add_vectors(np.eye(5 ,   dtype=np.floataa       )       )

														# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
														# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
														# see https://bugs.python.org/issue14243 and
														# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
														with tempfile.NamedTemporaryFile(delete=UpperCAmelCase__       ) as tmp_file:
																					index.save(tmp_file.name       )
																					lowerCAmelCase__	       =     FaissIndex.load(tmp_file.name       )
														os.unlink(tmp_file.name       )

														lowerCAmelCase__	       =     np.zeros(5 ,   dtype=np.floataa       )
														lowerCAmelCase__	       =     1
														lowerCAmelCase__       , lowerCAmelCase__	       =     index.search(UpperCAmelCase__       )
														self.assertGreater(scores[0] ,   0       )
														self.assertEqual(indices[0] ,   1       )


@require_faiss
def  _snake_case       (					A	)						->   Dict:
							import faiss

							lowerCAmelCase__	       =     FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT	)
							index.add_vectors(np.eye(5			,   dtype=np.floataa	)	)

							lowerCAmelCase__	       =     '''index.faiss'''
							lowerCAmelCase__	       =     F"""mock://{index_name}"""
							index.save(A			,   storage_options=mockfs.storage_options	)
							lowerCAmelCase__	       =     FaissIndex.load(A			,   storage_options=mockfs.storage_options	)

							lowerCAmelCase__	       =     np.zeros(5			,   dtype=np.floataa	)
							lowerCAmelCase__	       =     1
							lowerCAmelCase__       , lowerCAmelCase__	       =     index.search(A	)
							assert scores[0] > 0
							assert indices[0] == 1






@require_elasticsearch
class 		a__    (							__lowercase  ):





							'''simple docstring'''






							def   __SCREAMING_SNAKE_CASE							(		self       )	->							Union[str, Any]:
														from elasticsearch import Elasticsearch

														with patch('''elasticsearch.Elasticsearch.search'''       ) as mocked_search, patch(
														    '''elasticsearch.client.IndicesClient.create'''       ) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''       ) as mocked_bulk:
																					lowerCAmelCase__	       =     Elasticsearch()
																					lowerCAmelCase__	       =     {'''acknowledged''': True}
																					lowerCAmelCase__	       =     ElasticSearchIndex(es_client=UpperCAmelCase__       )
																					mocked_bulk.return_value([(True, None)] * 3       )
																					index.add_documents(['''foo''', '''bar''', '''foobar''']       )

																					# single query
																					lowerCAmelCase__	       =     '''foo'''
																					lowerCAmelCase__	       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
																					lowerCAmelCase__       , lowerCAmelCase__	       =     index.search(UpperCAmelCase__       )
																					self.assertEqual(scores[0] ,   1       )
																					self.assertEqual(indices[0] ,   0       )

																					# single query with timeout
																					lowerCAmelCase__	       =     '''foo'''
																					lowerCAmelCase__	       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
																					lowerCAmelCase__       , lowerCAmelCase__	       =     index.search(UpperCAmelCase__ ,   request_timeout=30       )
																					self.assertEqual(scores[0] ,   1       )
																					self.assertEqual(indices[0] ,   0       )

																					# batched queries
																					lowerCAmelCase__	       =     ['''foo''', '''bar''', '''foobar''']
																					lowerCAmelCase__	       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
																					lowerCAmelCase__       , lowerCAmelCase__	       =     index.search_batch(UpperCAmelCase__       )
																					lowerCAmelCase__	       =     [scores[0] for scores in total_scores]
																					lowerCAmelCase__	       =     [indices[0] for indices in total_indices]
																					self.assertGreater(np.min(UpperCAmelCase__       ) ,   0       )
																					self.assertListEqual([1, 1, 1] ,   UpperCAmelCase__       )

																					# batched queries with timeout
																					lowerCAmelCase__	       =     ['''foo''', '''bar''', '''foobar''']
																					lowerCAmelCase__	       =     {'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
																					lowerCAmelCase__       , lowerCAmelCase__	       =     index.search_batch(UpperCAmelCase__ ,   request_timeout=30       )
																					lowerCAmelCase__	       =     [scores[0] for scores in total_scores]
																					lowerCAmelCase__	       =     [indices[0] for indices in total_indices]
																					self.assertGreater(np.min(UpperCAmelCase__       ) ,   0       )
																					self.assertListEqual([1, 1, 1] ,   UpperCAmelCase__       )



'''simple docstring'''






from dataclasses import dataclass
from typing import Optional, Tuple, Union

import torch
import torch.nn as nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin
from .unet_ad_blocks import get_down_block, get_mid_block, get_out_block, get_up_block




@dataclass
class 		a__    (							a__  ):





							'''simple docstring'''






							lowercase__      :       torch.FloatTensor




class 		a__    (							a__    ,    a__  ):





							'''simple docstring'''






							@register_to_config
							def __init__(		self ,   lowerCamelCase_ = 6_55_36 ,   lowerCamelCase_ = None ,   lowerCamelCase_ = 2 ,   lowerCamelCase_ = 2 ,   lowerCamelCase_ = 0 ,   lowerCamelCase_ = "fourier" ,   lowerCamelCase_ = True ,   lowerCamelCase_ = False ,   lowerCamelCase_ = 0.0 ,   lowerCamelCase_ = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D") ,   lowerCamelCase_ = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip") ,   lowerCamelCase_ = "UNetMidBlock1D" ,   lowerCamelCase_ = None ,   lowerCamelCase_ = (32, 32, 64) ,   lowerCamelCase_ = None ,   lowerCamelCase_ = 8 ,   lowerCamelCase_ = 1 ,   lowerCamelCase_ = False ,   )	->							Optional[int]:
														super().__init__()
														lowerCAmelCase__	       =     sample_size

														# time
														if time_embedding_type == "fourier":
																					lowerCAmelCase__	       =     GaussianFourierProjection(
																					    embedding_size=8 ,   set_W_to_weight=lowerCamelCase_ ,   log=lowerCamelCase_ ,   flip_sin_to_cos=lowerCamelCase_       )
																					lowerCAmelCase__	       =     2 * block_out_channels[0]
														elif time_embedding_type == "positional":
																					lowerCAmelCase__	       =     Timesteps(
																					    block_out_channels[0] ,   flip_sin_to_cos=lowerCamelCase_ ,   downscale_freq_shift=lowerCamelCase_       )
																					lowerCAmelCase__	       =     block_out_channels[0]

														if use_timestep_embedding:
																					lowerCAmelCase__	       =     block_out_channels[0] * 4
																					lowerCAmelCase__	       =     TimestepEmbedding(
																					    in_channels=lowerCamelCase_ ,   time_embed_dim=lowerCamelCase_ ,   act_fn=lowerCamelCase_ ,   out_dim=block_out_channels[0] ,   )

														lowerCAmelCase__	       =     nn.ModuleList([]       )
														lowerCAmelCase__	       =     None
														lowerCAmelCase__	       =     nn.ModuleList([]       )
														lowerCAmelCase__	       =     None

														# down
														lowerCAmelCase__	       =     in_channels
														for i, down_block_type in enumerate(lowerCamelCase_       ):
																					lowerCAmelCase__	       =     output_channel
																					lowerCAmelCase__	       =     block_out_channels[i]

																					if i == 0:
																												input_channel += extra_in_channels

																					lowerCAmelCase__	       =     i == len(lowerCamelCase_       ) - 1

																					lowerCAmelCase__	       =     get_down_block(
																					    lowerCamelCase_ ,   num_layers=lowerCamelCase_ ,   in_channels=lowerCamelCase_ ,   out_channels=lowerCamelCase_ ,   temb_channels=block_out_channels[0] ,   add_downsample=not is_final_block or downsample_each_block ,   )
																					self.down_blocks.append(lowerCamelCase_       )

														# mid
														lowerCAmelCase__	       =     get_mid_block(
														    lowerCamelCase_ ,   in_channels=block_out_channels[-1] ,   mid_channels=block_out_channels[-1] ,   out_channels=block_out_channels[-1] ,   embed_dim=block_out_channels[0] ,   num_layers=lowerCamelCase_ ,   add_downsample=lowerCamelCase_ ,   )

														# up
														lowerCAmelCase__	       =     list(reversed(lowerCamelCase_       )       )
														lowerCAmelCase__	       =     reversed_block_out_channels[0]
														if out_block_type is None:
																					lowerCAmelCase__	       =     out_channels
														else:
																					lowerCAmelCase__	       =     block_out_channels[0]

														for i, up_block_type in enumerate(lowerCamelCase_       ):
																					lowerCAmelCase__	       =     output_channel
																					lowerCAmelCase__	       =     (
																					    reversed_block_out_channels[i + 1] if i < len(lowerCamelCase_       ) - 1 else final_upsample_channels
																					)

																					lowerCAmelCase__	       =     i == len(lowerCamelCase_       ) - 1

																					lowerCAmelCase__	       =     get_up_block(
																					    lowerCamelCase_ ,   num_layers=lowerCamelCase_ ,   in_channels=lowerCamelCase_ ,   out_channels=lowerCamelCase_ ,   temb_channels=block_out_channels[0] ,   add_upsample=not is_final_block ,   )
																					self.up_blocks.append(lowerCamelCase_       )
																					lowerCAmelCase__	       =     output_channel

														# out
														lowerCAmelCase__	       =     norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4 ,   32       )
														lowerCAmelCase__	       =     get_out_block(
														    out_block_type=lowerCamelCase_ ,   num_groups_out=lowerCamelCase_ ,   embed_dim=block_out_channels[0] ,   out_channels=lowerCamelCase_ ,   act_fn=lowerCamelCase_ ,   fc_dim=block_out_channels[-1] // 4 ,   )







							def   __SCREAMING_SNAKE_CASE							(		self ,   lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ = True ,   )	->							Union[UNetaDOutput, Tuple]:
														lowerCAmelCase__	       =     timestep
														if not torch.is_tensor(lowerCamelCase_       ):
																					lowerCAmelCase__	       =     torch.tensor([timesteps] ,   dtype=torch.long ,   device=sample.device       )
														elif torch.is_tensor(lowerCamelCase_       ) and len(timesteps.shape       ) == 0:
																					lowerCAmelCase__	       =     timesteps[None].to(sample.device       )

														lowerCAmelCase__	       =     self.time_proj(lowerCamelCase_       )
														if self.config.use_timestep_embedding:
																					lowerCAmelCase__	       =     self.time_mlp(lowerCamelCase_       )
														else:
																					lowerCAmelCase__	       =     timestep_embed[..., None]
																					lowerCAmelCase__	       =     timestep_embed.repeat([1, 1, sample.shape[2]]       ).to(sample.dtype       )
																					lowerCAmelCase__	       =     timestep_embed.broadcast_to((sample.shape[:1] + timestep_embed.shape[1:])       )

														# 2. down
														lowerCAmelCase__	       =     ()
														for downsample_block in self.down_blocks:
																					lowerCAmelCase__       , lowerCAmelCase__	       =     downsample_block(hidden_states=lowerCamelCase_ ,   temb=lowerCamelCase_       )
																					down_block_res_samples += res_samples

														# 3. mid
														if self.mid_block:
																					lowerCAmelCase__	       =     self.mid_block(lowerCamelCase_ ,   lowerCamelCase_       )

														# 4. up
														for i, upsample_block in enumerate(self.up_blocks       ):
																					lowerCAmelCase__	       =     down_block_res_samples[-1:]
																					lowerCAmelCase__	       =     down_block_res_samples[:-1]
																					lowerCAmelCase__	       =     upsample_block(lowerCamelCase_ ,   res_hidden_states_tuple=lowerCamelCase_ ,   temb=lowerCamelCase_       )

														# 5. post-process
														if self.out_block:
																					lowerCAmelCase__	       =     self.out_block(lowerCamelCase_ ,   lowerCamelCase_       )

														if not return_dict:
																					return (sample,)

														return UNetaDOutput(sample=lowerCamelCase_       )