[Bugfix] BailingMoeV2.5: rotate full qk_rope_head_dim in MLA RoPE (#41185)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
Jiangyun Zhu
2026-04-29 18:08:55 +08:00
committed by GitHub
parent 3f1a4bb639
commit 6d7d4da99e
@@ -205,13 +205,19 @@ class BailingMoeV25MLAAttention(nn.Module):
self.q_a_layernorm = None
self.q_b_proj = None
rope_parameters = _build_rope_parameters(config)
rope_parameters = _build_rope_parameters(config) or {}
# MLA rotates the full qk_rope_head_dim,
# partial_rotary_factor is for the linear-attn head only.
rope_parameters = {
k: v for k, v in rope_parameters.items() if k != "partial_rotary_factor"
}
rope_parameters["rope_dim"] = self.qk_rope_head_dim
max_position = getattr(config, "max_position_embeddings", 8192)
self.rotary_emb = get_rope(
head_size=self.qk_rope_head_dim,
max_position=max_position,
is_neox_style=False,
rope_parameters=rope_parameters or None,
rope_parameters=rope_parameters,
)
# Build MLAModules for MultiHeadLatentAttentionWrapper