forked from Karylab-cklius/vllm
@@ -421,7 +421,7 @@ class DeepseekV32DecoderLayer(nn.Module):
|
||||
)
|
||||
return out
|
||||
|
||||
shared_experts.forward = _fused_forward
|
||||
shared_experts.forward = _fused_forward # type: ignore[method-assign]
|
||||
|
||||
|
||||
class DeepseekV32MLAAttention(nn.Module):
|
||||
|
||||
@@ -43,6 +43,7 @@ class DeepSeekMultiTokenPredictorLayer(DeepSeekMultiTokenPredictorLayerBase):
|
||||
def __init__(self, vllm_config: VllmConfig, prefix: str) -> None:
|
||||
nn.Module.__init__(self)
|
||||
|
||||
assert vllm_config.speculative_config is not None
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ def sparse_attn_indexer(
|
||||
_ = torch.empty(max_logits_elems, dtype=torch.uint8, device=q_fp8.device)
|
||||
return None
|
||||
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata[k_cache_prefix]
|
||||
assert isinstance(attn_metadata, DeepseekV32IndexerMetadata)
|
||||
has_decode = attn_metadata.num_decodes > 0
|
||||
|
||||
Reference in New Issue
Block a user