karylab_agents/vllm
1
0
Fork 0
forked from Karylab-cklius/vllm
Code Pull Requests 2 Actions 1 Packages Activity

[DSv4] Tune default value of VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD (#41526)

Browse Source 
Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
Roger Wang
2026-05-02 18:32:09 -07:00
committed by GitHub
co-authored by Copilot
parent 1c607d7b2c
commit 856ec4804a
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/envs.py
+4 -4
View File
@@ -245,7 +245,7 @@ if TYPE_CHECKING:
VLLM_DEBUG_WORKSPACE: bool = False
VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 4096
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
VLLM_USE_V2_MODEL_RUNNER: bool = False
VLLM_LOG_MODEL_INSPECTION: bool = False
@@ -1686,10 +1686,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
# tokens the FP8 main GEMM has idle SMs to share with the bf16 aux GEMMs
# and overlap is a 5-45% win; above it the FP8 GEMM saturates the device
# and the cross-stream sync becomes pure overhead. Set to 0 to disable
# the multi-stream path entirely. Empirical crossover on B300 (148 SMs)
# is ~4096; B200 (132 SMs) is expected ~3072.
# the multi-stream path entirely. See #PR 41526 for the empirical result
# for the default value of 1024 tokens.
"VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD": lambda: int(
os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "4096")
os.getenv("VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD", "1024")
),
# Format for saving torch.compile cache artifacts
# - "binary": saves as binary file