karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 1 Packages Activity

[Bugfix][Granite4Vision] Fix deepstack buffer causing decode slowdown in compiled mode (#40917)

Browse Source 
Signed-off-by: artemspector <artems@il.ibm.com>
Co-authored-by: artemspector <artems@il.ibm.com>
This commit is contained in:
artem-spector
2026-04-28 07:43:30 +00:00
committed by GitHub
co-authored by artemspector
parent ea74f701db
commit bde0efdbb7
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/model_executor/models/granite4_vision.py
+2 -1
View File
@@ -887,9 +887,10 @@ class Granite4VisionForConditionalGeneration(
and get_pp_group().is_first_rank
and self._ds_layer_indices
):
n = inputs_embeds.size(0)
ds: IntermediateTensors | None = IntermediateTensors(
{
f"ds_{llm_layer}": self._ds_buffers[lvl]
f"ds_{llm_layer}": self._ds_buffers[lvl][:n]
for lvl, llm_layer in enumerate(self._ds_layer_indices)
}
)