forked from Karylab-cklius/vllm
[CI/Build][Bugfix] Fix SD LoRA (#45941)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -5,6 +5,7 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.model_executor.model_loader import get_model
|
||||
|
||||
|
||||
@@ -48,6 +49,13 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod
|
||||
target_embed = getattr(target_inner, "embed_tokens", None) or getattr(
|
||||
target_inner, "embedding", None
|
||||
)
|
||||
# If the target's embedding is LoRA-wrapped, share the underlying base
|
||||
# layer. The draft is not part of the LoRA adapter; sharing the wrapper
|
||||
# would make the draft run the LoRA embedding kernel with the target's
|
||||
# punica metadata (sized for the target's token count), causing an
|
||||
# out-of-bounds GPU access during multi-step draft decode.
|
||||
if isinstance(target_embed, BaseLayerWithLoRA):
|
||||
target_embed = target_embed.base_layer
|
||||
draft_embed = getattr(draft_inner, "embed_tokens", None)
|
||||
if target_embed is not None and _should_share(
|
||||
eagle_model, "has_own_embed_tokens", draft_embed, target_embed
|
||||
|
||||
Reference in New Issue
Block a user