fixed attentions for MoE

vall_e/models/arch/mixtral.py

@@ -1,5 +1,6 @@
 # https://github.com/huggingface/transformers/blob/main/src/transformers/models/mixtral/modeling_mixtral.py
 
+import math
 import torch
 import torch.nn.functional as F
 from typing import Literal, overload, Optional, Tuple
@@ -13,6 +14,16 @@ try:
 except Exception as e:
 	pass
 
+try:
+	from .llama import fused_attn_func
+except Exception as e:
+	pass
+
+try:
+	from .llama import memory_efficient_attention
+except Exception as e:
+	pass
+
 # This is required because batch sizes > 1 throws errors
 def MixtralSparseMoeBlock_forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
 	""" """
@@ -99,6 +110,7 @@ class MixtralAttention_Adapted(MixtralAttention):
 				use_cache=use_cache,
 			)
 
+		dropout_rate = self.attention_dropout if self.training else 0.0
 		bsz, q_len, _ = hidden_states.size()
 
 		query_states = self.q_proj(hidden_states)
@@ -127,98 +139,59 @@ class MixtralAttention_Adapted(MixtralAttention):
 		key_states = repeat_kv(key_states, self.num_key_value_groups)
 		value_states = repeat_kv(value_states, self.num_key_value_groups)
 
-		causal_mask = attention_mask
+		if self.mode in ["xformers", "flash_attn"]:
-		if attention_mask is not None:  # no matter the length, we just slice it
+			# TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
-			causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+			# to be able to avoid many of these transpose/reshape/view.
+			query_states = query_states.transpose(1, 2)
+			key_states = key_states.transpose(1, 2)
+			value_states = value_states.transpose(1, 2)
 
-		# SDPA with memory-efficient backend is currently (torch==2.1.2) bugged with non-contiguous inputs with custom attn_mask,
+			"""
-		# Reference: https://github.com/pytorch/pytorch/issues/112577.
+			# In PEFT, usually we cast the layer norms in float32 for training stability reasons
-		if query_states.device.type == "cuda" and attention_mask is not None:
+			# therefore the input hidden states gets silently casted in float32. Hence, we need
-			query_states = query_states.contiguous()
+			# cast them back in the correct dtype just to be sure everything works as expected.
-			key_states = key_states.contiguous()
+			# This might slowdown training & inference so it is recommended to not cast the LayerNorms
-			value_states = value_states.contiguous()
+			# in fp32. (LlamaRMSNorm handles it correctly)
 
-		# We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
+			input_dtype = query_states.dtype
-		# in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
+			if input_dtype == torch.float32:
-		# The q_len > 1 is necessary to match with AttentionMaskConverter.to_causal_4d that does not create a causal mask in case q_len == 1.
+				if torch.is_autocast_enabled():
-		is_causal = True if causal_mask is None and q_len > 1 else False
+					target_dtype = torch.get_autocast_gpu_dtype()
+				# Handle the case where the model is quantized
+				elif hasattr(self.config, "_pre_quantization_dtype"):
+					target_dtype = self.config._pre_quantization_dtype
+				else:
+					target_dtype = self.q_proj.weight.dtype
 
-		#with torch.backends.cuda.sdp_kernel(enable_flash=self.mode == "flash", enable_math=self.mode == "math", enable_mem_efficient=self.mode == "mem_efficient"):
+				query_states = query_states.to(target_dtype)
-		if self.mode == "flash_attn":
+				key_states = key_states.to(target_dtype)
-			attn_output = flash_attn_func(
+				value_states = value_states.to(target_dtype)
-				query_states,
+			"""
-				key_states,
+
-				value_states,
+			if self.mode == "flash_attn":
-				causal=True,
+				attn_output = flash_attn_func(
-				softmax_scale=None, # 1, / math.sqrt(cfg.head_dim),
-				dropout_p=self.attention_dropout if self.training else 0.0,
-			)
-		else:
-			with torch.nn.attention.sdpa_kernel(self.mode):
-				attn_output = torch.nn.functional.scaled_dot_product_attention(
 					query_states,
 					key_states,
 					value_states,
-					attn_mask=causal_mask,
+					causal=True,
-					dropout_p=self.attention_dropout if self.training else 0.0,
+					softmax_scale=1.0 / math.sqrt(self.head_dim),
-					is_causal=is_causal,
+					dropout_p=dropout_rate,
 				)
+				
+				attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
+			elif self.mode == "xformers":
+				attn_output = memory_efficient_attention(
+					query_states,
+					key_states,
+					value_states,
+					attn_bias = LowerTriangularMask() if attention_mask is None or attention_mask[0, 0, 0, 1] == 0 else None,
+					scale = 1.0 / math.sqrt(self.head_dim),
+					p=dropout_rate
+				)
+				attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
 
-		attn_output = attn_output.transpose(1, 2).contiguous()
+			attn_output = self.o_proj(attn_output)
-		attn_output = attn_output.view(bsz, q_len, self.hidden_size)
+			return attn_output, None, past_key_value
-
-		attn_output = self.o_proj(attn_output)
-
-		return attn_output, None, past_key_value
-	"""
-	def forward(
-		self,
-		hidden_states: torch.Tensor,
-		attention_mask: Optional[torch.Tensor] = None,
-		position_ids: Optional[torch.LongTensor] = None,
-		past_key_value: Optional[Cache] = None,
-		output_attentions: bool = False,
-		use_cache: bool = False,
-		cache_position: Optional[torch.LongTensor] = None,
-		position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # will become mandatory in v4.45
-		**kwargs,
-	) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
-		if output_attentions:
-			# TODO: Improve this warning with e.g. `model.config.attn_implementation = "manual"` once this is implemented.
-			return super().forward(
-				hidden_states=hidden_states,
-				attention_mask=attention_mask,
-				position_ids=position_ids,
-				past_key_value=past_key_value,
-				output_attentions=output_attentions,
-				use_cache=use_cache,
-				cache_position=cache_position,
-				position_embeddings=position_embeddings,
-			)
-
-		bsz, q_len, _ = hidden_states.size()
-
-		query_states = self.q_proj(hidden_states)
-		key_states = self.k_proj(hidden_states)
-		value_states = self.v_proj(hidden_states)
-
-		query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
-		key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
-		value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
-
-		if position_embeddings is None:
-			cos, sin = self.rotary_emb(value_states, position_ids)
-		else:
-			cos, sin = position_embeddings
-		query_states, key_states = apply_rotary_pos_emb(query_states, key_states, cos, sin)
-
-		if past_key_value is not None:
-			# sin and cos are specific to RoPE models; cache_position needed for the static cache
-			cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
-			key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
-
-		key_states = repeat_kv(key_states, self.num_key_value_groups)
-		value_states = repeat_kv(value_states, self.num_key_value_groups)
 
 		causal_mask = attention_mask
 		if attention_mask is not None:
@@ -234,22 +207,30 @@ class MixtralAttention_Adapted(MixtralAttention):
 		# We dispatch to SDPA's Flash Attention or Efficient kernels via this `is_causal` if statement instead of an inline conditional assignment
 		# in SDPA to support both torch.compile's dynamic shapes and full graph options. An inline conditional prevents dynamic shapes from compiling.
 		is_causal = True if causal_mask is None and q_len > 1 else False
-
+		
-		#with torch.backends.cuda.sdp_kernel(enable_flash=self.mode == "flash", enable_math=self.mode == "math", enable_mem_efficient=self.mode == "mem_efficient"):
+		if self.mode in ["fused_attn"]:
-		with torch.nn.attention.sdpa_kernel(self.mode):
+			attn_output = fused_attn_func(
-			attn_output = torch.nn.functional.scaled_dot_product_attention(
 				query_states,
 				key_states,
 				value_states,
-				attn_mask=causal_mask,
+				causal=True,
-				dropout_p=self.attention_dropout if self.training else 0.0,
+				softmax_scale=1.0 / math.sqrt(self.head_dim),
-				is_causal=is_causal,
+				dropout_p=dropout_rate,
 			)
+		else:
+			with torch.nn.attention.sdpa_kernel(self.mode):
+				attn_output = torch.nn.functional.scaled_dot_product_attention(
+					query_states,
+					key_states,
+					value_states,
+					attn_mask=causal_mask,
+					dropout_p=dropout_rate,
+					is_causal=is_causal,
+				)
 
 		attn_output = attn_output.transpose(1, 2).contiguous()
 		attn_output = attn_output.view(bsz, q_len, -1)
 
 		attn_output = self.o_proj(attn_output)
 
-		return attn_output, None, past_key_value
+		return attn_output, None, past_key_value
-	"""