Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
This commit is contained in:
Woosuk Kwon
2026-04-09 02:13:11 +00:00
parent c8661431e0
commit d9b481e248
3 changed files with 3 additions and 1 deletions
@@ -421,7 +421,7 @@ class DeepseekV32DecoderLayer(nn.Module):
)
return out
shared_experts.forward = _fused_forward
shared_experts.forward = _fused_forward # type: ignore[method-assign]
class DeepseekV32MLAAttention(nn.Module):
@@ -43,6 +43,7 @@ class DeepSeekMultiTokenPredictorLayer(DeepSeekMultiTokenPredictorLayerBase):
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
nn.Module.__init__(self)
assert vllm_config.speculative_config is not None
config = vllm_config.speculative_config.draft_model_config.hf_config
quant_config = vllm_config.quant_config
@@ -48,6 +48,7 @@ def sparse_attn_indexer(
_ = torch.empty(max_logits_elems, dtype=torch.uint8, device=q_fp8.device)
return None
assert isinstance(attn_metadata, dict)
attn_metadata = attn_metadata[k_cache_prefix]
assert isinstance(attn_metadata, DeepseekV32IndexerMetadata)
has_decode = attn_metadata.num_decodes > 0