From 674e6ffdb6aac3782419c9ffdd86700bdd672858 Mon Sep 17 00:00:00 2001 From: Zijing Liu Date: Fri, 17 Apr 2026 10:17:27 +0000 Subject: [PATCH] [KV Connector] Opt DecodeBenchConnector into SupportsHMA Previously the HMA (hybrid KV cache manager) layer refused to activate when DecodeBenchConnector was in use, because the connector did not advertise SupportsHMA. That forced decode-only benchmark recipes to pass --disable-hybrid-kv-cache-manager, which collapsed hybrid-model KV cache groups (SWA / MLA compress=4 / MLA compress=128 / sparse indexer) into a single uniform page size via unify_kv_cache_spec_ page_size, throwing away the compression savings and capping concurrent capacity on hybrid models (e.g. DeepSeek-V4 saw ~43 concurrent 8k/1k requests instead of the model's true ceiling). This connector is a dummy fill that owns no external per-block state, so the HMA path has nothing extra to do. Implementation is minimal: - Inherit from SupportsHMA. - Implement request_finished_all_groups: delegates to the same scheduler.request_finished() cleanup as the single-group variant, ignoring block_ids (no per-block state to release). With this change, recipes can drop --disable-hybrid-kv-cache-manager and let HMA size each KV cache group correctly. Co-Authored-By: Claude Opus 4.7 (1M context) Signed-off-by: Zijing Liu --- .../kv_connector/v1/decode_bench_connector.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py index 6e9e757ffbd..9a39ec658ff 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/decode_bench_connector.py @@ -40,7 +40,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import ( KVConnectorBase_V1, KVConnectorRole, ) -from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata +from vllm.distributed.kv_transfer.kv_connector.v1.base import ( + KVConnectorMetadata, + SupportsHMA, +) from vllm.logger import init_logger from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionMetadata @@ -71,7 +74,7 @@ class DecodeBenchConnectorMetadata(KVConnectorMetadata): reqs_to_fill: dict[str, tuple[tuple[list[int], ...], int]] -class DecodeBenchConnector(KVConnectorBase_V1): +class DecodeBenchConnector(KVConnectorBase_V1, SupportsHMA): """ A KV Connector for decode instance performance testing. @@ -164,6 +167,17 @@ class DecodeBenchConnector(KVConnectorBase_V1): self.connector_scheduler.request_finished(request) return False, None + def request_finished_all_groups( + self, + request: "Request", + block_ids: tuple[list[int], ...], + ) -> tuple[bool, dict[str, Any] | None]: + # HMA-enabled path: same cleanup as the single-group variant since + # this connector owns no external state per block. + assert self.connector_scheduler is not None + self.connector_scheduler.request_finished(request) + return False, None + class DecodeBenchConnectorScheduler: """Scheduler-side implementation for DecodeBenchConnector."""