




from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


_lowerCamelCase      			=						logging.get_logger(__name__)

_lowerCamelCase      			=						{
    'microsoft/focalnet-tiny': 'https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json',
}



class       a					(       __SCREAMING_SNAKE_CASE     ,       __SCREAMING_SNAKE_CASE      ):

     '''simple docstring'''





     lowerCAmelCase    :	int       		=  "focalnet"
     def __init__(     self       :      List[str] ,   __snake_case       :      int=2_24 ,   __snake_case       :      Union[str, Any]=4 ,   __snake_case       :      int=3 ,   __snake_case       :      Any=96 ,   __snake_case       :      List[str]=False ,   __snake_case       :      Union[str, Any]=[1_92, 3_84, 7_68, 7_68] ,   __snake_case       :      Any=[2, 2, 6, 2] ,   __snake_case       :      Optional[Any]=[2, 2, 2, 2] ,   __snake_case       :      Dict=[3, 3, 3, 3] ,   __snake_case       :      Union[str, Any]="gelu" ,   __snake_case       :      Dict=4.0 ,   __snake_case       :      Optional[Any]=0.0 ,   __snake_case       :      Tuple=0.1 ,   __snake_case       :      List[Any]=False ,   __snake_case       :      Union[str, Any]=1E-4 ,   __snake_case       :      Any=False ,   __snake_case       :      Dict=False ,   __snake_case       :      str=False ,   __snake_case       :      Union[str, Any]=0.02 ,   __snake_case       :      int=1E-5 ,   __snake_case       :      Tuple=32 ,   __snake_case       :      Optional[Any]=None ,   __snake_case       :      Dict=None ,   **__snake_case       :      str ,   ):
         super().__init__(**__UpperCAmelCase    )

         UpperCAmelCase_										=					image_size
         UpperCAmelCase_										=					patch_size
         UpperCAmelCase_										=					num_channels
         UpperCAmelCase_										=					embed_dim
         UpperCAmelCase_										=					use_conv_embed
         UpperCAmelCase_										=					hidden_sizes
         UpperCAmelCase_										=					depths
         UpperCAmelCase_										=					focal_levels
         UpperCAmelCase_										=					focal_windows
         UpperCAmelCase_										=					hidden_act
         UpperCAmelCase_										=					mlp_ratio
         UpperCAmelCase_										=					hidden_dropout_prob
         UpperCAmelCase_										=					drop_path_rate
         UpperCAmelCase_										=					use_layerscale
         UpperCAmelCase_										=					layerscale_value
         UpperCAmelCase_										=					use_post_layernorm
         UpperCAmelCase_										=					use_post_layernorm_in_modulation
         UpperCAmelCase_										=					normalize_modulator
         UpperCAmelCase_										=					initializer_range
         UpperCAmelCase_										=					layer_norm_eps
         UpperCAmelCase_										=					encoder_stride
         UpperCAmelCase_										=					["""stem"""] + [F'stage{idx}' for idx in range(1 ,   len(self.depths    ) + 1    )]
         UpperCAmelCase_										=					get_aligned_output_features_output_indices(
             out_features=__UpperCAmelCase ,   out_indices=__UpperCAmelCase ,   stage_names=self.stage_names    )











import logging
import math
from functools import partial
from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union

import torch

from .tensor_utils import tensor_tree_map, tree_map




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						Union[dict, list, tuple, torch.Tensor]				)  ->							List[Tuple[int, ...]]:
    UpperCAmelCase_										=					[]
    if isinstance(__UpperCamelCase	,					__UpperCamelCase				):
        for v in tree.values():
            shapes.extend(_fetch_dims(__UpperCamelCase				)				)
    elif isinstance(__UpperCamelCase	,					(list, tuple)				):
        for t in tree:
            shapes.extend(_fetch_dims(__UpperCamelCase				)				)
    elif isinstance(__UpperCamelCase	,					torch.Tensor				):
        shapes.append(tree.shape				)
    else:
        raise ValueError('''Not supported'''				)

    return shapes




@torch.jit.ignore
def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						int	,					__UpperCamelCase      :						Tuple[int, ...]				)  ->							Tuple[int, ...]:
    UpperCAmelCase_										=					[]
    for d in reversed(__UpperCamelCase				):
        idx.append(flat_idx % d				)
        UpperCAmelCase_										=					flat_idx // d

    return tuple(reversed(__UpperCamelCase				)				)




@torch.jit.ignore
def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						Sequence[int]	,					__UpperCamelCase      :						Sequence[int]	,					__UpperCamelCase      :						Sequence[int]	,					__UpperCamelCase      :						Optional[Sequence[bool]] = None	,					__UpperCamelCase      :						Optional[Sequence[bool]] = None	,					)  ->							List[Tuple[slice, ...]]:

    # start_edges and end_edges both indicate whether, starting from any given
    # dimension, the start/end index is at the top/bottom edge of the
    # corresponding tensor, modeled as a tree
    def reduce_edge_list(__UpperCamelCase      :						List[bool]				) -> None:
        UpperCAmelCase_										=					True
        for i in range(len(__UpperCamelCase				)				):
            UpperCAmelCase_										=					-1 * (i + 1)
            l[reversed_idx] &= tally
            UpperCAmelCase_										=					l[reversed_idx]

    if start_edges is None:
        UpperCAmelCase_										=					[s == 0 for s in start]
        reduce_edge_list(__UpperCamelCase				)
    if end_edges is None:
        UpperCAmelCase_										=					[e == (d - 1) for e, d in zip(__UpperCamelCase	,					__UpperCamelCase				)]
        reduce_edge_list(__UpperCamelCase				)

    # Base cases. Either start/end are empty and we're done, or the final,
    # one-dimensional tensor can be simply sliced
    if len(__UpperCamelCase				) == 0:
        return [()]
    elif len(__UpperCamelCase				) == 1:
        return [(slice(start[0]	,					end[0] + 1				),)]

    UpperCAmelCase_										=					[]
    UpperCAmelCase_										=					[]

    # Dimensions common to start and end can be selected directly
    for s, e in zip(__UpperCamelCase	,					__UpperCamelCase				):
        if s == e:
            path_list.append(slice(__UpperCamelCase	,					s + 1				)				)
        else:
            break

    UpperCAmelCase_										=					tuple(__UpperCamelCase				)
    UpperCAmelCase_										=					len(__UpperCamelCase				)

    # start == end, and we're done
    if divergence_idx == len(__UpperCamelCase				):
        return [path]

    def upper() -> Tuple[Tuple[slice, ...], ...]:
        assert start_edges is not None
        assert end_edges is not None

        UpperCAmelCase_										=					start[divergence_idx]
        return tuple(
            path + (slice(__UpperCamelCase	,					sdi + 1				),) + s
            for s in _get_minimal_slice_set(
                start[divergence_idx + 1 :]	,					[d - 1 for d in dims[divergence_idx + 1 :]]	,					dims[divergence_idx + 1 :]	,					start_edges=start_edges[divergence_idx + 1 :]	,					end_edges=[True for _ in end_edges[divergence_idx + 1 :]]	,					)				)

    def lower() -> Tuple[Tuple[slice, ...], ...]:
        assert start_edges is not None
        assert end_edges is not None

        UpperCAmelCase_										=					end[divergence_idx]
        return tuple(
            path + (slice(__UpperCamelCase	,					edi + 1				),) + s
            for s in _get_minimal_slice_set(
                [0 for _ in start[divergence_idx + 1 :]]	,					end[divergence_idx + 1 :]	,					dims[divergence_idx + 1 :]	,					start_edges=[True for _ in start_edges[divergence_idx + 1 :]]	,					end_edges=end_edges[divergence_idx + 1 :]	,					)				)

    # If both start and end are at the edges of the subtree rooted at
    # divergence_idx, we can just select the whole subtree at once
    if start_edges[divergence_idx] and end_edges[divergence_idx]:
        slices.append(path + (slice(start[divergence_idx]	,					end[divergence_idx] + 1				),)				)
    # If just start is at the edge, we can grab almost all of the subtree,
    # treating only the ragged bottom edge as an edge case
    elif start_edges[divergence_idx]:
        slices.append(path + (slice(start[divergence_idx]	,					end[divergence_idx]				),)				)
        slices.extend(lower()				)
    # Analogous to the previous case, but the top is ragged this time
    elif end_edges[divergence_idx]:
        slices.extend(upper()				)
        slices.append(path + (slice(start[divergence_idx] + 1	,					end[divergence_idx] + 1				),)				)
    # If both sides of the range are ragged, we need to handle both sides
    # separately. If there's contiguous meat in between them, we can index it
    # in one big chunk
    else:
        slices.extend(upper()				)
        UpperCAmelCase_										=					end[divergence_idx] - start[divergence_idx]
        if middle_ground > 1:
            slices.append(path + (slice(start[divergence_idx] + 1	,					end[divergence_idx]				),)				)
        slices.extend(lower()				)

    return slices




@torch.jit.ignore
def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						torch.Tensor	,					__UpperCamelCase      :						int	,					__UpperCamelCase      :						int	,					__UpperCamelCase      :						int				)  ->							torch.Tensor:
    UpperCAmelCase_										=					t.shape[:no_batch_dims]
    UpperCAmelCase_										=					list(_flat_idx_to_idx(__UpperCamelCase	,					__UpperCamelCase				)				)
    # _get_minimal_slice_set is inclusive
    UpperCAmelCase_										=					list(_flat_idx_to_idx(flat_end - 1	,					__UpperCamelCase				)				)

    # Get an ordered list of slices to perform
    UpperCAmelCase_										=					_get_minimal_slice_set(
        __UpperCamelCase	,					__UpperCamelCase	,					__UpperCamelCase	,					)

    UpperCAmelCase_										=					[t[s] for s in slices]

    return torch.cat([s.view((-1,) + t.shape[no_batch_dims:]				) for s in sliced_tensors]				)




def  SCREAMING_SNAKE_CASE     (	__UpperCamelCase      :						Callable	,					__UpperCamelCase      :						Dict[str, Any]	,					__UpperCamelCase      :						int	,					__UpperCamelCase      :						int	,					__UpperCamelCase      :						bool = False	,					__UpperCamelCase      :						Any = None	,					__UpperCamelCase      :						bool = False	,					)  ->							Any:
    if not (len(__UpperCamelCase				) > 0):
        raise ValueError('''Must provide at least one input'''				)

    UpperCAmelCase_										=					[shape[:no_batch_dims] for shape in _fetch_dims(__UpperCamelCase				)]
    UpperCAmelCase_										=					tuple([max(__UpperCamelCase				) for s in zip(*__UpperCamelCase				)]				)

    def _prep_inputs(__UpperCamelCase      :						torch.Tensor				) -> torch.Tensor:
        if not low_mem:
            if not sum(t.shape[:no_batch_dims]				) == no_batch_dims:
                UpperCAmelCase_										=					t.expand(orig_batch_dims + t.shape[no_batch_dims:]				)
            UpperCAmelCase_										=					t.reshape(-1	,					*t.shape[no_batch_dims:]				)
        else:
            UpperCAmelCase_										=					t.expand(orig_batch_dims + t.shape[no_batch_dims:]				)
        return t

    UpperCAmelCase_										=					tensor_tree_map(_prep_inputs	,					__UpperCamelCase				)
    UpperCAmelCase_										=					None
    if _out is not None:
        UpperCAmelCase_										=					tensor_tree_map(lambda __UpperCamelCase				: t.view([-1] + list(t.shape[no_batch_dims:]				)				)	,					_out				)

    UpperCAmelCase_										=					1
    for d in orig_batch_dims:
        flat_batch_dim *= d

    UpperCAmelCase_										=					flat_batch_dim // chunk_size + (flat_batch_dim % chunk_size != 0)

    def _select_chunk(__UpperCamelCase      :						torch.Tensor				) -> torch.Tensor:
        return t[i : i + chunk_size] if t.shape[0] != 1 else t

    UpperCAmelCase_										=					0
    UpperCAmelCase_										=					prepped_outputs
    for _ in range(__UpperCamelCase				):
        # Chunk the input
        if not low_mem:
            UpperCAmelCase_										=					_select_chunk
        else:
            UpperCAmelCase_										=					partial(
                _chunk_slice	,					flat_start=__UpperCamelCase	,					flat_end=min(__UpperCamelCase	,					i + chunk_size				)	,					no_batch_dims=len(__UpperCamelCase				)	,					)

        UpperCAmelCase_										=					tensor_tree_map(__UpperCamelCase	,					__UpperCamelCase				)

        # Run the layer on the chunk
        UpperCAmelCase_										=					layer(**__UpperCamelCase				)

        # Allocate space for the output
        if out is None:
            UpperCAmelCase_										=					tensor_tree_map(lambda __UpperCamelCase				: t.new_zeros((flat_batch_dim,) + t.shape[1:]				)	,					__UpperCamelCase				)

        # Put the chunk in its pre-allocated space
        if isinstance(__UpperCamelCase	,					__UpperCamelCase				):

            def assign(__UpperCamelCase      :						dict	,					__UpperCamelCase      :						dict				) -> None:
                for k, v in da.items():
                    if isinstance(__UpperCamelCase	,					__UpperCamelCase				):
                        assign(__UpperCamelCase	,					da[k]				)
                    else:
                        if _add_into_out:
                            v[i : i + chunk_size] += da[k]
                        else:
                            UpperCAmelCase_										=					da[k]

            assign(__UpperCamelCase	,					__UpperCamelCase				)
        elif isinstance(__UpperCamelCase	,					__UpperCamelCase				):
            for xa, xa in zip(__UpperCamelCase	,					__UpperCamelCase				):
                if _add_into_out:
                    xa[i : i + chunk_size] += xa
                else:
                    UpperCAmelCase_										=					xa
        elif isinstance(__UpperCamelCase	,					torch.Tensor				):
            if _add_into_out:
                out[i : i + chunk_size] += output_chunk
            else:
                UpperCAmelCase_										=					output_chunk
        else:
            raise ValueError('''Not supported'''				)

        i += chunk_size

    UpperCAmelCase_										=					tensor_tree_map(lambda __UpperCamelCase				: t.view(orig_batch_dims + t.shape[1:]				)	,					__UpperCamelCase				)

    return out



class       a					:

     '''simple docstring'''





     def __init__(     self       :      List[Any] ,   __snake_case       :      int = 5_12 ,   ):
         UpperCAmelCase_										=					max_chunk_size
         UpperCAmelCase_										=					None
         UpperCAmelCase_										=					None
     def   lowerCamelCase_       (     self       :      List[Any] ,   __snake_case       :      Callable ,   __snake_case       :      tuple ,   __snake_case       :      int    ):
         logging.info('''Tuning chunk size...'''    )

         if min_chunk_size >= self.max_chunk_size:
             return min_chunk_size

         UpperCAmelCase_										=					[2**l for l in range(int(math.log(self.max_chunk_size ,   2    )    ) + 1    )]
         UpperCAmelCase_										=					[c for c in candidates if c > min_chunk_size]
         UpperCAmelCase_										=					[min_chunk_size] + candidates
         candidates[-1] += 4

         def test_chunk_size(__snake_case       :      int    ) -> bool:
             try:
                 with torch.no_grad():
                     fn(*__snake_case ,   chunk_size=__snake_case    )
                 return True
             except RuntimeError:
                 return False

         UpperCAmelCase_										=					0
         UpperCAmelCase_										=					len(__snake_case    ) - 1
         while i > min_viable_chunk_size_index:
             UpperCAmelCase_										=					test_chunk_size(candidates[i]    )
             if not viable:
                 UpperCAmelCase_										=					(min_viable_chunk_size_index + i) // 2
             else:
                 UpperCAmelCase_										=					i
                 UpperCAmelCase_										=					(i + len(__snake_case    ) - 1) // 2

         return candidates[min_viable_chunk_size_index]
     def   lowerCamelCase_       (     self       :      int ,   __snake_case       :      Iterable ,   __snake_case       :      Iterable    ):
         UpperCAmelCase_										=					True
         for aa, aa in zip(__snake_case ,   __snake_case    ):
             assert type(__snake_case    ) == type(__snake_case    )
             if isinstance(__snake_case ,   (list, tuple)    ):
                 consistent &= self._compare_arg_caches(__snake_case ,   __snake_case    )
             elif isinstance(__snake_case ,   __snake_case    ):
                 UpperCAmelCase_										=					[v for _, v in sorted(aa.items() ,   key=lambda __snake_case    : x[0]    )]
                 UpperCAmelCase_										=					[v for _, v in sorted(aa.items() ,   key=lambda __snake_case    : x[0]    )]
                 consistent &= self._compare_arg_caches(__snake_case ,   __snake_case    )
             else:
                 consistent &= aa == aa

         return consistent




     def   lowerCamelCase_       (     self       :      str ,   __snake_case       :      Callable ,   __snake_case       :      tuple ,   __snake_case       :      int ,   ):
         UpperCAmelCase_										=					True
         UpperCAmelCase_										=					tree_map(lambda __snake_case    : a.shape if isinstance(__snake_case ,   torch.Tensor    ) else a ,   __snake_case ,   __snake_case    )
         if self.cached_arg_data is not None:
             # If args have changed shape/value, we need to re-tune
             assert len(self.cached_arg_data    ) == len(__snake_case    )
             UpperCAmelCase_										=					self._compare_arg_caches(self.cached_arg_data ,   __snake_case    )
         else:
             # Otherwise, we can reuse the precomputed value
             UpperCAmelCase_										=					False

         if not consistent:
             UpperCAmelCase_										=					self._determine_favorable_chunk_size(
                 __snake_case ,   __snake_case ,   __snake_case ,   )
             UpperCAmelCase_										=					arg_data

         assert self.cached_chunk_size is not None

         return self.cached_chunk_size





