



import tensorflow as tf

from ...tf_utils import shape_list







class 				A_      (					tf.keras.layers.Layer    ):



						"""simple docstring"""






						def __init__( self			:List[Any]       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:List[Any]       ,  lowercase_			:List[Any]       ,  lowercase_			:str       ,  lowercase_			:List[Any]=1       ,  lowercase_			:Dict=False       ,  **lowercase_			:List[Any] )    ->      Union[str, Any]:
												super().__init__(**lowercase_ )

												UpperCAmelCase					=			vocab_size
												UpperCAmelCase					=			d_embed
												UpperCAmelCase					=			d_proj

												UpperCAmelCase					=			cutoffs + [vocab_size]
												UpperCAmelCase					=			[0] + self.cutoffs
												UpperCAmelCase					=			div_val

												UpperCAmelCase					=			self.cutoffs[0]
												UpperCAmelCase					=			len(self.cutoffs ) - 1
												UpperCAmelCase					=			self.shortlist_size + self.n_clusters
												UpperCAmelCase					=			keep_order

												UpperCAmelCase					=			[]
												UpperCAmelCase					=			[]






						def      UpperCAmelCase__					( self			:Any       ,  lowercase_			:Any )    ->      Tuple:
												if self.n_clusters > 0:
																		UpperCAmelCase					=			self.add_weight(
																		    shape=(self.n_clusters, self.d_embed)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name='cluster_weight' )
																		UpperCAmelCase					=			self.add_weight(
																		    shape=(self.n_clusters,)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name='cluster_bias' )

												if self.div_val == 1:
																		for i in range(len(self.cutoffs ) ):
																								if self.d_proj != self.d_embed:
																														UpperCAmelCase					=			self.add_weight(
																														    shape=(self.d_embed, self.d_proj)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name=f"""out_projs_._{i}"""       ,  )
																														self.out_projs.append(lowercase_ )
																								else:
																														self.out_projs.append(lowercase_ )
																								UpperCAmelCase					=			self.add_weight(
																								    shape=(self.vocab_size, self.d_embed)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name=f"""out_layers_._{i}_._weight"""       ,  )
																								UpperCAmelCase					=			self.add_weight(
																								    shape=(self.vocab_size,)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name=f"""out_layers_._{i}_._bias"""       ,  )
																								self.out_layers.append((weight, bias) )
												else:
																		for i in range(len(self.cutoffs ) ):
																								UpperCAmelCase				,   UpperCAmelCase					=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
																								UpperCAmelCase					=			self.d_embed // (self.div_val**i)

																								UpperCAmelCase					=			self.add_weight(
																								    shape=(d_emb_i, self.d_proj)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name=f"""out_projs_._{i}""" )
																								self.out_projs.append(lowercase_ )
																								UpperCAmelCase					=			self.add_weight(
																								    shape=(r_idx - l_idx, d_emb_i)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name=f"""out_layers_._{i}_._weight"""       ,  )
																								UpperCAmelCase					=			self.add_weight(
																								    shape=(r_idx - l_idx,)       ,  initializer='zeros'       ,  trainable=lowercase_       ,  name=f"""out_layers_._{i}_._bias"""       ,  )
																								self.out_layers.append((weight, bias) )
												super().build(lowercase_ )






						@staticmethod
						def      UpperCAmelCase__					( lowercase_			:str       ,  lowercase_			:Union[str, Any]       ,  lowercase_			:List[str]       ,  lowercase_			:Tuple=None )    ->      Optional[int]:
												UpperCAmelCase					=			x
												if proj is not None:
																		UpperCAmelCase					=			tf.einsum('ibd,ed->ibe'       ,  lowercase_       ,  lowercase_ )
												return tf.einsum('ibd,nd->ibn'       ,  lowercase_       ,  lowercase_ ) + b






						@staticmethod
						def      UpperCAmelCase__					( lowercase_			:Dict       ,  lowercase_			:Optional[int] )    ->      Optional[Any]:
												UpperCAmelCase					=			shape_list(lowercase_ )
												UpperCAmelCase					=			tf.range(lp_size[0]       ,  dtype=target.dtype )
												UpperCAmelCase					=			tf.stack([r, target]       ,  1 )
												return tf.gather_nd(lowercase_       ,  lowercase_ )






						def      UpperCAmelCase__					( self			:Optional[Any]       ,  lowercase_			:int       ,  lowercase_			:Dict       ,  lowercase_			:int=True       ,  lowercase_			:Any=False )    ->      Optional[Any]:
												UpperCAmelCase					=			0
												if self.n_clusters == 0:
																		UpperCAmelCase					=			self._logit(lowercase_       ,  self.out_layers[0][0]       ,  self.out_layers[0][1]       ,  self.out_projs[0] )
																		if target is not None:
																								UpperCAmelCase					=			tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowercase_       ,  logits=lowercase_ )
																		UpperCAmelCase					=			tf.nn.log_softmax(lowercase_       ,  axis=-1 )
												else:
																		UpperCAmelCase					=			shape_list(lowercase_ )
																		UpperCAmelCase					=			[]
																		UpperCAmelCase					=			tf.zeros(hidden_sizes[:2] )
																		for i in range(len(self.cutoffs ) ):
																								UpperCAmelCase				,   UpperCAmelCase					=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
																								if target is not None:
																														UpperCAmelCase					=			(target >= l_idx) & (target < r_idx)
																														UpperCAmelCase					=			tf.where(lowercase_ )
																														UpperCAmelCase					=			tf.boolean_mask(lowercase_       ,  lowercase_ ) - l_idx

																								if self.div_val == 1:
																														UpperCAmelCase					=			self.out_layers[0][0][l_idx:r_idx]
																														UpperCAmelCase					=			self.out_layers[0][1][l_idx:r_idx]
																								else:
																														UpperCAmelCase					=			self.out_layers[i][0]
																														UpperCAmelCase					=			self.out_layers[i][1]

																								if i == 0:
																														UpperCAmelCase					=			tf.concat([cur_W, self.cluster_weight]       ,  0 )
																														UpperCAmelCase					=			tf.concat([cur_b, self.cluster_bias]       ,  0 )

																														UpperCAmelCase					=			self._logit(lowercase_       ,  lowercase_       ,  lowercase_       ,  self.out_projs[0] )
																														UpperCAmelCase					=			tf.nn.log_softmax(lowercase_ )
																														out.append(head_logprob[..., : self.cutoffs[0]] )
																														if target is not None:
																																				UpperCAmelCase					=			tf.boolean_mask(lowercase_       ,  lowercase_ )
																																				UpperCAmelCase					=			self._gather_logprob(lowercase_       ,  lowercase_ )
																								else:
																														UpperCAmelCase					=			self._logit(lowercase_       ,  lowercase_       ,  lowercase_       ,  self.out_projs[i] )
																														UpperCAmelCase					=			tf.nn.log_softmax(lowercase_ )
																														UpperCAmelCase					=			self.cutoffs[0] + i - 1  # No probability for the head cluster
																														UpperCAmelCase					=			head_logprob[..., cluster_prob_idx, None] + tail_logprob
																														out.append(lowercase_ )
																														if target is not None:
																																				UpperCAmelCase					=			tf.boolean_mask(lowercase_       ,  lowercase_ )
																																				UpperCAmelCase					=			tf.boolean_mask(lowercase_       ,  lowercase_ )
																																				UpperCAmelCase					=			self._gather_logprob(lowercase_       ,  lowercase_ )
																																				cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																								if target is not None:
																														loss += tf.scatter_nd(lowercase_       ,  -cur_logprob       ,  shape_list(lowercase_ ) )
																		UpperCAmelCase					=			tf.concat(lowercase_       ,  axis=-1 )

												if target is not None:
																		if return_mean:
																								UpperCAmelCase					=			tf.reduce_mean(lowercase_ )
																		# Add the training-time loss value to the layer using `self.add_loss()`.
																		self.add_loss(lowercase_ )

																		# Log the loss as a metric (we could log arbitrary metrics,
																		# including different metrics for training and inference.
																		self.add_metric(lowercase_       ,  name=self.name       ,  aggregation='mean' if return_mean else '' )

												return out










"""simple docstring"""


def 			_lowerCAmelCase				(       lowercase_						):

						if not isinstance(lowercase_		,     lowercase_						):
												UpperCAmelCase					=			F"""Input value of [number={number}] must be an integer"""
												raise TypeError(lowercase_						)

						if number < 1:
												UpperCAmelCase					=			F"""Input value of [number={number}] must be > 0"""
												raise ValueError(lowercase_						)

						UpperCAmelCase					=			1

						for i in range(1		,     lowercase_						):
												current_number *= 4 * i - 2
												current_number //= i + 1

						return current_number


if __name__ == "__main__":
				import doctest

				doctest.testmod()





