


'''simple docstring'''






import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
 import bitsandbytes as bnb
 import torch
 import torch.nn as nn

 from ..pytorch_utils import ConvaD

if is_accelerate_available():
 from accelerate import init_empty_weights
 from accelerate.utils import find_tied_parameters

lowercase							=logging.get_logger(__name__)





def lowerCamelCase__      (						__lowerCamelCase	:							Any	,   __lowerCamelCase	:							Optional[int]	,   __lowerCamelCase	:							Tuple	,   __lowerCamelCase	:							Dict=None	,   __lowerCamelCase	:							Tuple=None							):







   '''simple docstring'''




   if "." in tensor_name:
      _UpperCAmelCase       :       List[str]							   =tensor_name.split('.'							)
      for split in splits[:-1]:
         _UpperCAmelCase       :       int							   =getattr(SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_							)
         if new_module is None:
            raise ValueError(f"{module} has no attribute {split}."							)
         _UpperCAmelCase       :       Union[str, Any]							   =new_module
      _UpperCAmelCase       :       Tuple							   =splits[-1]

   if tensor_name not in module._parameters and tensor_name not in module._buffers:
      raise ValueError(f"{module} does not have a parameter or a buffer named {tensor_name}."							)
   _UpperCAmelCase       :       List[Any]							   =tensor_name in module._buffers
   _UpperCAmelCase       :       Union[str, Any]							   =getattr(SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_							)

   if old_value.device == torch.device('meta'							) and device not in ["meta", torch.device('meta'							)] and value is None:
      raise ValueError(f"{tensor_name} is on the meta device, we need a `value` to put in on {device}."							)

   _UpperCAmelCase       :       str							   =False
   _UpperCAmelCase       :       Optional[int]							   =False
   if is_buffer or not is_bitsandbytes_available():
      _UpperCAmelCase       :       int							   =False
      _UpperCAmelCase       :       List[str]							   =False
   else:
      _UpperCAmelCase       :       Optional[Any]							   =hasattr(bnb.nn	,   'Params4bit'							) and isinstance(module._parameters[tensor_name]	,   bnb.nn.Paramsabit							)
      _UpperCAmelCase       :       Optional[int]							   =isinstance(module._parameters[tensor_name]	,   bnb.nn.IntaParams							)

   if is_abit or is_abit:
      _UpperCAmelCase       :       int							   =module._parameters[tensor_name]
      if param.device.type != "cuda":
         if value is None:
            _UpperCAmelCase       :       Union[str, Any]							   =old_value.to(SCREAMING_SNAKE_CASE_							)
         elif isinstance(SCREAMING_SNAKE_CASE_	,   torch.Tensor							):
            _UpperCAmelCase       :       Dict							   =value.to('cpu'							)
            if value.dtype == torch.inta:
               _UpperCAmelCase       :       Any							   =version.parse(importlib.metadata.version('bitsandbytes'							)							) > version.parse(
                   '0.37.2'							)
               if not is_abit_serializable:
                  raise ValueError(
                      'Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. '
                      'Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`.'							)
         else:
            _UpperCAmelCase       :       Optional[Any]							   =torch.tensor(SCREAMING_SNAKE_CASE_	,   device='cpu'							)

         # Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
         # Since weights are saved in the correct "orientation", we skip transposing when loading.
         if issubclass(module.source_cls	,   SCREAMING_SNAKE_CASE_							) and fpaa_statistics is None:
            _UpperCAmelCase       :       Tuple							   =new_value.T

         _UpperCAmelCase       :       Tuple							   =old_value.__dict__
         if is_abit:
            _UpperCAmelCase       :       List[str]							   =bnb.nn.IntaParams(SCREAMING_SNAKE_CASE_	,   requires_grad=SCREAMING_SNAKE_CASE_	,   **SCREAMING_SNAKE_CASE_							).to(SCREAMING_SNAKE_CASE_							)
         elif is_abit:
            _UpperCAmelCase       :       List[str]							   =bnb.nn.Paramsabit(SCREAMING_SNAKE_CASE_	,   requires_grad=SCREAMING_SNAKE_CASE_	,   **SCREAMING_SNAKE_CASE_							).to(SCREAMING_SNAKE_CASE_							)

         _UpperCAmelCase       :       List[Any]							   =new_value
         if fpaa_statistics is not None:
            setattr(module.weight	,   'SCB'	,   fpaa_statistics.to(SCREAMING_SNAKE_CASE_							)							)

   else:
      if value is None:
         _UpperCAmelCase       :       Union[str, Any]							   =old_value.to(SCREAMING_SNAKE_CASE_							)
      elif isinstance(SCREAMING_SNAKE_CASE_	,   torch.Tensor							):
         _UpperCAmelCase       :       Union[str, Any]							   =value.to(SCREAMING_SNAKE_CASE_							)
      else:
         _UpperCAmelCase       :       List[str]							   =torch.tensor(SCREAMING_SNAKE_CASE_	,   device=SCREAMING_SNAKE_CASE_							)

      if is_buffer:
         _UpperCAmelCase       :       Dict							   =new_value
      else:
         _UpperCAmelCase       :       Optional[Any]							   =nn.Parameter(SCREAMING_SNAKE_CASE_	,   requires_grad=old_value.requires_grad							)
         _UpperCAmelCase       :       Optional[int]							   =new_value





def lowerCamelCase__      (						__lowerCamelCase	:							Optional[int]	,   __lowerCamelCase	:							Optional[Any]=None	,   __lowerCamelCase	:							Tuple=None	,   __lowerCamelCase	:							Optional[Any]=None	,   __lowerCamelCase	:							int=False							):







   '''simple docstring'''




   for name, module in model.named_children():
      if current_key_name is None:
         _UpperCAmelCase       :       Tuple							   =[]
      current_key_name.append(SCREAMING_SNAKE_CASE_							)

      if (isinstance(SCREAMING_SNAKE_CASE_	,   nn.Linear							) or isinstance(SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_							)) and name not in modules_to_not_convert:
         # Check if the current key is not in the `modules_to_not_convert`
         if not any(key in '.'.join(SCREAMING_SNAKE_CASE_							) for key in modules_to_not_convert							):
            with init_empty_weights():
               if isinstance(SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_							):
                  _UpperCAmelCase   ,	_UpperCAmelCase       :       str							   =module.weight.shape
               else:
                  _UpperCAmelCase       :       List[Any]							   =module.in_features
                  _UpperCAmelCase       :       Tuple							   =module.out_features

               if quantization_config.quantization_method() == "llm_int8":
                  _UpperCAmelCase       :       List[Any]							   =bnb.nn.LinearabitLt(
                      SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   module.bias is not None	,   has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight	,   threshold=quantization_config.llm_inta_threshold	,   )
                  _UpperCAmelCase       :       Any							   =True
               else:
                  if (
                      quantization_config.llm_inta_skip_modules is not None
                      and name in quantization_config.llm_inta_skip_modules
                  ):
                     pass
                  else:
                     _UpperCAmelCase       :       List[Any]							   =bnb.nn.Linearabit(
                         SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   module.bias is not None	,   quantization_config.bnb_abit_compute_dtype	,   compress_statistics=quantization_config.bnb_abit_use_double_quant	,   quant_type=quantization_config.bnb_abit_quant_type	,   )
                     _UpperCAmelCase       :       Union[str, Any]							   =True
                    # Store the module class in case we need to transpose the weight later
               _UpperCAmelCase       :       int							   =type(SCREAMING_SNAKE_CASE_							)
               # Force requires grad to False to avoid unexpected errors
               model._modules[name].requires_grad_(SCREAMING_SNAKE_CASE_							)
      if len(list(module.children()							)							) > 0:
         _UpperCAmelCase   ,	_UpperCAmelCase       :       int							   =_replace_with_bnb_linear(
             SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   has_been_replaced=SCREAMING_SNAKE_CASE_	,   )
      # Remove the last key for recursion
      current_key_name.pop(-1							)
   return model, has_been_replaced





def lowerCamelCase__      (						__lowerCamelCase	:							Any	,   __lowerCamelCase	:							Dict=None	,   __lowerCamelCase	:							Tuple=None	,   __lowerCamelCase	:							Optional[Any]=None							):







   '''simple docstring'''




   _UpperCAmelCase       :       Optional[Any]							   =['lm_head'] if modules_to_not_convert is None else modules_to_not_convert
   _UpperCAmelCase   ,	_UpperCAmelCase       :       Tuple							   =_replace_with_bnb_linear(
       SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_							)

   if not has_been_replaced:
      logger.warning(
          'You are loading your model in 8bit or 4bit but no linear modules were found in your model.'
          ' Please double check your model architecture, or submit an issue on github if you think this is'
          ' a bug.'							)

   return model





def lowerCamelCase__      (						*__lowerCamelCase	:							str	,   **__lowerCamelCase	:							Any							):







   '''simple docstring'''




   warnings.warn(
       '`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead'	,   SCREAMING_SNAKE_CASE_	,   )
   return replace_with_bnb_linear(*SCREAMING_SNAKE_CASE_	,   **SCREAMING_SNAKE_CASE_							)





def lowerCamelCase__      (						*__lowerCamelCase	:							Any	,   **__lowerCamelCase	:							Optional[int]							):







   '''simple docstring'''




   warnings.warn(
       '`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead'	,   SCREAMING_SNAKE_CASE_	,   )
   return set_module_quantized_tensor_to_device(*SCREAMING_SNAKE_CASE_	,   **SCREAMING_SNAKE_CASE_							)





def lowerCamelCase__      (						__lowerCamelCase	:							Any							):







   '''simple docstring'''




   _UpperCAmelCase       :       Dict							   =deepcopy(SCREAMING_SNAKE_CASE_							)  # this has 0 cost since it is done inside `init_empty_weights` context manager`
   tied_model.tie_weights()

   _UpperCAmelCase       :       Optional[Any]							   =find_tied_parameters(SCREAMING_SNAKE_CASE_							)
   # For compatibility with Accelerate < 0.18
   if isinstance(SCREAMING_SNAKE_CASE_	,   SCREAMING_SNAKE_CASE_							):
      _UpperCAmelCase       :       Union[str, Any]							   =sum(list(tied_params.values()							)	,   []							) + list(tied_params.keys()							)
   else:
      _UpperCAmelCase       :       str							   =sum(SCREAMING_SNAKE_CASE_	,   []							)
   _UpperCAmelCase       :       List[str]							   =len(SCREAMING_SNAKE_CASE_							) > 0

   # Check if it is a base model
   _UpperCAmelCase       :       int							   =not hasattr(SCREAMING_SNAKE_CASE_	,   model.base_model_prefix							)

   # Ignore this for base models (BertModel, GPT2Model, etc.)
   if (not has_tied_params) and is_base_model:
      return []

   # otherwise they have an attached head
   _UpperCAmelCase       :       int							   =list(model.named_children()							)
   _UpperCAmelCase       :       Tuple							   =[list_modules[-1][0]]

   # add last module together with tied weights
   _UpperCAmelCase       :       int							   =set(SCREAMING_SNAKE_CASE_							) - set(SCREAMING_SNAKE_CASE_							)
   _UpperCAmelCase       :       Optional[int]							   =list(set(SCREAMING_SNAKE_CASE_							)							) + list(SCREAMING_SNAKE_CASE_							)

   # remove ".weight" from the keys
   _UpperCAmelCase       :       List[Any]							   =['.weight', '.bias']
   _UpperCAmelCase       :       Optional[int]							   =[]
   for name in list_untouched:
      for name_to_remove in names_to_remove:
         if name_to_remove in name:
            _UpperCAmelCase       :       Optional[Any]							   =name.replace(SCREAMING_SNAKE_CASE_	,   ''							)
      filtered_module_names.append(SCREAMING_SNAKE_CASE_							)

   return filtered_module_names





'''simple docstring'''






import jax.numpy as jnp

from ...utils import logging
from ..ta.modeling_flax_ta import FlaxTaEncoderModel, FlaxTaForConditionalGeneration, FlaxTaModel
from .configuration_mta import MTaConfig


lowercase							=logging.get_logger(__name__)

lowercase							='T5Config'





def lowerCamelCase__      (						__lowerCamelCase	:							jnp.array	,   __lowerCamelCase	:							int	,   __lowerCamelCase	:							int							):







   '''simple docstring'''




   _UpperCAmelCase       :       List[Any]							   =jnp.zeros_like(__lowerCamelCase							)
   _UpperCAmelCase       :       Union[str, Any]							   =shifted_input_ids.at[:, 1:].set(input_ids[:, :-1]							)
   _UpperCAmelCase       :       List[Any]							   =shifted_input_ids.at[:, 0].set(__lowerCamelCase							)

   _UpperCAmelCase       :       str							   =jnp.where(shifted_input_ids == -1_0_0	,   __lowerCamelCase	,   __lowerCamelCase							)
   return shifted_input_ids






class  __magic_name__							(		lowerCAmelCase    ):
       UpperCAmelCase										="mt5"
       UpperCAmelCase										=MTaConfig






class  __magic_name__							(		lowerCAmelCase    ):
       UpperCAmelCase										="mt5"
       UpperCAmelCase										=MTaConfig






class  __magic_name__							(		lowerCAmelCase    ):
       UpperCAmelCase										="mt5"
       UpperCAmelCase										=MTaConfig

