Merge branch 'main' into kimi-k3

Co-authored-by: Codex <codex@openai.com>

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>

.buildkite/check-torch-abi.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Audit vLLM compiled libraries for PyTorch stable ABI compliance."""
+
+import fnmatch
+import sys
+from pathlib import Path
+
+from torch_abi_audit import inspect_package
+from torch_abi_audit.report import ExtensionReport, PackageReport
+
+# Temporary allowlist of extensions not yet on the stable ABI.
+# Shrink and remove over time.
+ALLOWED_UNSTABLE_LIBRARIES: tuple[str, ...] = (
+    "_flashkda_C.abi3.so",
+    "vllm_flash_attn/_vllm_fa2_C.abi3.so",
+    "vllm_flash_attn/_vllm_fa3_C.abi3.so",
+    "third_party/deep_gemm/_C*.so",
+)
+
+
+def _relative_path(lib: ExtensionReport, package_root: Path) -> str:
+    try:
+        return lib.path.relative_to(package_root).as_posix()
+    except ValueError:
+        return lib.path.name
+
+
+def _is_torch_unstable(lib: ExtensionReport) -> bool:
+    return lib.error is None and lib.torch.uses_torch and not lib.torch.stable
+
+
+def _matches_allowlist(rel_path: str, patterns: tuple[str, ...]) -> bool:
+    return any(fnmatch.fnmatch(rel_path, pattern) for pattern in patterns)
+
+
+def _iter_libs(report: PackageReport) -> tuple[ExtensionReport, ...]:
+    return (*report.extensions, *report.bundled_libs)
+
+
+def _collect_unstable(report: PackageReport) -> list[str]:
+    return sorted(
+        _relative_path(lib, report.root)
+        for lib in _iter_libs(report)
+        if _is_torch_unstable(lib)
+    )
+
+
+def _find_stale_allowlist_entries(
+    report: PackageReport, patterns: tuple[str, ...]
+) -> list[str]:
+    """Allowlist patterns that match a built library which is no longer unstable."""
+    stale: list[str] = []
+    for pattern in patterns:
+        for lib in _iter_libs(report):
+            if lib.error is not None:
+                continue
+            if not fnmatch.fnmatch(_relative_path(lib, report.root), pattern):
+                continue
+            if not _is_torch_unstable(lib):
+                stale.append(pattern)
+                break
+    return stale
+
+
+def check_torch_abi(
+    package: str = "vllm",
+    patterns: tuple[str, ...] = ALLOWED_UNSTABLE_LIBRARIES,
+) -> int:
+    report = inspect_package(package)
+    if report.error:
+        print(f"error: failed to inspect {package!r}: {report.error}", file=sys.stderr)
+        return 2
+
+    unstable = _collect_unstable(report)
+    unexpected = [
+        rel_path for rel_path in unstable if not _matches_allowlist(rel_path, patterns)
+    ]
+    stale = _find_stale_allowlist_entries(report, patterns)
+
+    if unexpected or stale:
+        if unexpected:
+            print(
+                "Not allowed: torch-unstable libraries outside "
+                f"ALLOWED_UNSTABLE_LIBRARIES: {', '.join(unexpected)}",
+                file=sys.stderr,
+            )
+        if stale:
+            print(
+                "Not allowed: stale ALLOWED_UNSTABLE_LIBRARIES entries: "
+                f"{', '.join(stale)}",
+                file=sys.stderr,
+            )
+        return 1
+
+    print("Torch stable ABI check passed.")
+    return 0
+
+
+if __name__ == "__main__":
+    print(">>> Auditing vLLM extension modules for PyTorch stable ABI compliance")
+    sys.exit(check_torch_abi())

.buildkite/ci_config.yaml

@@ -14,6 +14,7 @@ run_all_patterns:
   - "setup.py"
   - "csrc/"
   - "cmake/"
+  - ".buildkite/check-torch-abi.py"
 run_all_exclude_patterns:
   - "docker/Dockerfile."
   - "csrc/cpu/"

.buildkite/intel_jobs/test-intel.yaml

@@ -147,6 +147,30 @@ steps:
         'cd tests &&
         pytest -v -s quantization/test_auto_round.py &&
         pytest -v -s quantization/test_online.py'
+  - label: "XPU GPQA Eval (GPT-OSS)"
+    depends_on:
+      - image-build-xpu
+    timeout_in_minutes: 60
+    device: intel_gpu
+    agent_tags:
+      label: production
+      gpu: 1+
+      mem: 24+
+    no_plugin: true
+    env:
+      REGISTRY: "public.ecr.aws/q9t5s3a7"
+      REPO: "vllm-ci-test-repo"
+      VLLM_TEST_DEVICE: "xpu"
+    source_file_dependencies:
+      - vllm/
+      - tests/evals/gpt_oss/
+      - .buildkite/intel_jobs/test-intel.yaml
+    commands:
+      - >-
+        bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+        'pip install "gpt-oss[eval]==0.0.5" &&
+        cd tests &&
+        pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
   - label: "XPU compressed tensors FP8 test"
     depends_on:
       - image-build-xpu

.buildkite/test_areas/cuda.yaml

@@ -16,6 +16,7 @@ steps:
   commands:
     - pytest -v -s cuda/test_cuda_context.py
     - pytest -v -s cuda/test_platform_no_cuda_init.py
+    - pytest -v -s cuda/test_cuda_compatibility_path.py
 
 - label: Cudagraph
   device: h200_35gb

.buildkite/test_areas/disaggregated.yaml

@@ -131,6 +131,22 @@ steps:
         - uv pip install --system -r /vllm-workspace/requirements/kv_connectors_rocm.txt
         - HYBRID_SSM=1 ATTENTION_BACKEND=TRITON_ATTN bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh
 
+- label: NixlConnector PD edge case test (2 GPUs)
+  key: nixlconnector-pd-edge-cases-2-gpus
+  timeout_in_minutes: 40
+  working_dir: "/vllm-workspace/tests"
+  num_devices: 2
+  source_file_dependencies:
+    - vllm/distributed/kv_transfer/kv_connector/v1/nixl/
+    - vllm/v1/core/sched/
+    - tests/v1/kv_connector/nixl_integration/
+  env:
+    PREFILL_GPU_ID: "0"
+    DECODE_GPU_ID: "1"
+  commands:
+    - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
+    - bash v1/kv_connector/nixl_integration/run_edge_case_test.sh
+
 - label: Hybrid SSM NixlConnector PD prefix cache test (2 GPUs)
   key: hybrid-ssm-nixlconnector-pd-prefix-cache-2-gpus
   timeout_in_minutes: 25

.buildkite/test_areas/engine.yaml

@@ -40,9 +40,11 @@ steps:
   source_file_dependencies:
     - vllm/v1/engine/
     - tests/v1/engine/
+    - tests/v1/test_tensor_ipc_queue.py
   commands:
     - pytest -v -s v1/engine/test_preprocess_error_handling.py
     - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
+    - pytest -v -s v1/test_tensor_ipc_queue.py
   mirror:
     amd:
       device: mi250_1

.buildkite/test_areas/expert_parallelism.yaml

@@ -52,4 +52,5 @@ steps:
   - vllm/compilation/
   - tests/distributed/
   commands:
+  - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
   - pytest -v -s distributed/test_elastic_ep.py

.buildkite/test_areas/kernels.yaml

@@ -61,9 +61,45 @@ steps:
   source_file_dependencies:
   - csrc/fused_deepseek_v4_qnorm_rope_kv_insert_kernel.cu
   - vllm/models/deepseek_v4/common/ops/
+  - vllm/models/deepseek_v4/nvidia/
   - tests/kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
+  - tests/models/test_deepseek_v4_mega_moe.py
   commands:
     - pytest -v -s kernels/test_fused_deepseek_v4_*.py
+    - pytest -v -s models/test_deepseek_v4_mega_moe.py
+
+# Catch-all for test files at the tests/kernels root. This job collects
+# the whole root so new files are wired by default.
+# Files with dedicated jobs elsewhere in this file are excluded via --ignore
+# (test_kda, test_bf16x3_router_gemm_cutedsl and test_ll_bf16_gemm run in
+# their own jobs / Kernels (B200)).
+- label: Kernels Root Misc Test (B200)
+  key: kernels-root-misc-test-b200
+  timeout_in_minutes: 45
+  device: b200-k8s
+  source_file_dependencies:
+  - csrc/
+  - vllm/
+  - tests/kernels/
+  commands:
+    - pytest -v -s kernels/
+        --ignore=kernels/attention
+        --ignore=kernels/core
+        --ignore=kernels/helion
+        --ignore=kernels/ir
+        --ignore=kernels/mamba
+        --ignore=kernels/moe
+        --ignore=kernels/quantization
+        --ignore=kernels/test_concat_mla_q.py
+        --ignore=kernels/test_fused_qk_norm_rope_gate.py
+        --ignore=kernels/test_fused_deepseek_v4_qnorm_rope_kv_insert.py
+        --ignore=kernels/test_top_k_per_row.py
+        --ignore=kernels/test_kda.py
+        --ignore=kernels/test_bf16x3_router_gemm_cutedsl.py
+        --ignore=kernels/test_ll_bf16_gemm.py
+        --ignore=kernels/test_shuffle_rows.py
+    # BROKEN on main, pending kernel fixes (B200):
+    #   test_shuffle_rows.py (1: test_shuffle_rows_edge_cases)
 
 - label: Kernels Attention Test %N
   key: kernels-attention-test
@@ -178,17 +214,6 @@ steps:
   commands:
     - pytest -v -s kernels/mamba
 
-- label: Kernels KDA Test
-  timeout_in_minutes: 25
-  device: h200_18gb
-  source_file_dependencies:
-  - vllm/third_party/flash_linear_attention/ops/kda.py
-  - vllm/third_party/flash_linear_attention/ops/chunk_delta_h.py
-  - vllm/third_party/flash_linear_attention/ops/l2norm.py
-  - tests/kernels/test_kda.py
-  commands:
-    - pytest -v -s kernels/test_kda.py
-
 - label: Kernels DeepGEMM Test (H100)
   key: kernels-deepgemm-test-h100
   timeout_in_minutes: 35

.buildkite/test_areas/misc.yaml

@@ -148,6 +148,7 @@ steps:
     - pytest -v -s -m 'cpu_test' v1/core
     - pytest -v -s v1/structured_output
     - pytest -v -s v1/test_serial_utils.py
+    - pytest -v -s v1/test_kv_cache_spec_registry.py
     - pytest -v -s v1/cudagraph/test_cudagraph_manager.py
     - pytest -v -s -m 'cpu_test' v1/kv_connector/unit
     - pytest -v -s -m 'cpu_test' v1/metrics
@@ -265,6 +266,7 @@ steps:
   - vllm/utils/
   - vllm/v1/
   - tests/v1/tracing
+  - tests/tracing/
   commands:
   - "pip install \
       'opentelemetry-sdk>=1.26.0' \
@@ -272,6 +274,7 @@ steps:
       'opentelemetry-exporter-otlp>=1.26.0' \
       'opentelemetry-semantic-conventions-ai>=0.4.1'"
   - pytest -v -s v1/tracing
+  - pytest -v -s tracing
   mirror:
     amd:
       dind: false
@@ -395,7 +398,7 @@ steps:
 
 - label: Batch Invariance (A100)
   key: batch-invariance-a100
-  timeout_in_minutes: 40
+  timeout_in_minutes: 60
   device: a100
   source_file_dependencies:
     - vllm/v1/attention
@@ -405,11 +408,11 @@ steps:
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
     - pip install pytest-timeout pytest-forked
     - pytest -v -s v1/determinism/test_batch_invariance.py
-    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
+    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
 
 - label: Batch Invariance (H100)
   key: batch-invariance-h100
-  timeout_in_minutes: 40
+  timeout_in_minutes: 60
   device: h100
   source_file_dependencies:
     - vllm/v1/attention
@@ -420,12 +423,12 @@ steps:
     - pip install pytest-timeout pytest-forked
     - pytest -v -s v1/determinism/test_batch_invariance.py
     - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
-    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
-    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
+    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
+    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
 
 - label: Batch Invariance (B200)
   key: batch-invariance-b200
-  timeout_in_minutes: 35
+  timeout_in_minutes: 45
   device: b200-k8s
   source_file_dependencies:
     - vllm/v1/attention
@@ -436,11 +439,14 @@ steps:
     - pip install pytest-timeout pytest-forked
     - pytest -v -s v1/determinism/test_batch_invariance.py
     - pytest -v -s v1/determinism/test_rms_norm_batch_invariant.py
-    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[TRITON_MLA]
-    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle[FLASH_ATTN]
+    - VLLM_TEST_MODEL=deepseek-ai/DeepSeek-V2-Lite-Chat pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k TRITON_MLA
+    - VLLM_TEST_MODEL=Qwen/Qwen3-30B-A3B-Thinking-2507-FP8 pytest -v -s v1/determinism/test_batch_invariance.py::test_v1_generation_is_deterministic_across_batch_sizes_with_needle -k FLASH_ATTN
     - pytest -v -s v1/determinism/test_nvfp4_batch_invariant.py
     - pytest -v -s v1/determinism/test_nvfp4_batch_invariant_scaled_mm.py
-  
+    - pytest -v -s v1/determinism/test_matmul_batch_invariant.py
+    - pytest -v -s v1/determinism/test_cutlass_batch_invariance.py
+    - pytest -v -s v1/determinism/test_online_batch_invariance.py
+
 - label: Acceptance Length Test (Large Models) # optional
   device: h200_35gb
   key: acceptance-length-test-large-models

.buildkite/test_areas/models_basic.yaml

@@ -61,6 +61,20 @@ steps:
     # FA4 kernel tests require SM100; the suite skips them elsewhere.
     - pytest -v -s models/inkling
 
+- label: Kimi K3 Unit Tests (B200)
+  key: kimi-k3-unit-tests-b200
+  timeout_in_minutes: 40
+  device: b200-k8s
+  source_file_dependencies:
+  - vllm/models/kimi_k3/
+  - csrc/libtorch_stable/kimi_k3/
+  - tests/models/kimi_k3/
+  - tests/kernels/attention/test_kimi_k3_mla_fused_epilogue.py
+  - tests/kernels/test_bf16_skinny_gemm.py
+  commands:
+    # The native NVIDIA Kimi K3 kernels require the SM100 family.
+    - pytest -v -s models/kimi_k3 kernels/attention/test_kimi_k3_mla_fused_epilogue.py kernels/test_bf16_skinny_gemm.py
+
 - label: Basic Models Test (Other CPU) # 5min
   key: basic-models-test-other-cpu
   depends_on:
@@ -70,7 +84,8 @@ steps:
   - vllm/
   - tests/models/test_utils.py
   - tests/models/test_vision.py
+  - tests/models/test_adapters.py
   - tests/models/transformers/fusers/
   device: cpu-small
   commands:
-    - pytest -v -s models/test_utils.py models/test_vision.py models/transformers/fusers/
+    - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/transformers/fusers/

.buildkite/test_areas/spec_decode.yaml

@@ -90,8 +90,10 @@ steps:
     - vllm/v1/spec_decode/
     - vllm/v1/worker/gpu/spec_decode/
     - tests/v1/e2e/spec_decode/
+    - tests/spec_decode/
   commands:
     - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
+    - python3 spec_decode/test_custom_proposer.py
   mirror:
     amd:
       dind: false

.buildkite/test_areas/torch_abi.yaml

@@ -0,0 +1,14 @@
+group: Torch ABI
+depends_on:
+  - image-build
+steps:
+- label: Torch Stable ABI Audit
+  key: torch-stable-abi-audit
+  timeout_in_minutes: 5
+  source_file_dependencies:
+  - .buildkite/check-torch-abi.py
+  - csrc/
+  - cmake/
+  - setup.py
+  commands:
+  - python3 /vllm-workspace/.buildkite/check-torch-abi.py

.github/workflows/new_pr_bot.yml

@@ -80,9 +80,9 @@ jobs:
                   '',
                   '\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.',
                   '',
-                  'PRs do not trigger a full CI run by default. Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.',
+                  'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.',
                   '',
-                  'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.',
+                  'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run` or `/ci retry`. New commits do not start CI automatically.',
                   '',
                   'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.',
                   '',

.github/workflows/pre-commit.yml

@@ -41,7 +41,7 @@ jobs:
           if (hasReadyLabel || hasVerifiedLabel || mergedCount >= 4) {
             core.info(`Check passed: verified label=${hasVerifiedLabel}, ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`);
           } else {
-            core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label (the ready labels also trigger tests) or the author must have at least 4 merged PRs (found ${mergedCount}).`);
+            core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label to run pre-commit, or the author must have at least 4 merged PRs (found ${mergedCount}).`);
           }
 
   pre-commit:

.github/workflows/run-ci-command.yml

@@ -0,0 +1,40 @@
+name: Run CI from PR comment
+
+on:
+  issue_comment:
+    types: [created]
+
+concurrency:
+  group: run-ci-comment-${{ github.event.issue.number }}
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+  issues: write
+  pull-requests: read
+
+jobs:
+  run-ci-command:
+    if: >-
+      github.event.issue.pull_request &&
+      (github.event.comment.body == '/ci run' ||
+      github.event.comment.body == '/ci retry')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          persist-credentials: false
+      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          python-version: "3.12"
+      - name: Authorize and run CI command
+        run: >-
+          uv run --no-project --python 3.12
+          .github/workflows/scripts/run_ci_command.py
+        env:
+          BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
+          BUILDKITE_ORGANIZATION: vllm
+          BUILDKITE_PIPELINE: ci
+          CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }}
+          GH_TOKEN: ${{ github.token }}

.github/workflows/scripts/run_ci_command.py

@@ -0,0 +1,581 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+COMMAND_RUN_CI = "/ci run"
+COMMAND_RETRY_FAILED = "/ci retry"
+READY_LABELS = {"ready", "ready-run-all-tests"}
+TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
+ACTIVE_BUILD_STATES = {
+    "blocked",
+    "creating",
+    "scheduled",
+    "running",
+    "failing",
+    "canceling",
+    "waiting",
+    "waiting_failed",
+}
+RETRY_STATES = "failed,timed_out,expired"
+
+
+class ApiError(RuntimeError):
+    def __init__(self, status: int | None, message: str) -> None:
+        super().__init__(message)
+        self.status = status
+
+
+class HttpTransport:
+    def request(
+        self,
+        url: str,
+        *,
+        body: Mapping[str, Any] | None = None,
+        headers: Mapping[str, str] | None = None,
+        method: str = "GET",
+    ) -> Any:
+        data = None if body is None else json.dumps(body).encode()
+        request = urllib.request.Request(
+            url,
+            data=data,
+            headers=dict(headers or {}),
+            method=method,
+        )
+        try:
+            with urllib.request.urlopen(request, timeout=30) as response:
+                response_body = response.read().decode()
+        except urllib.error.HTTPError as error:
+            response_body = error.read().decode()
+            message = self._error_message(response_body, error.reason)
+            raise ApiError(
+                error.code,
+                f"API returned {error.code}: {message}",
+            ) from error
+        except urllib.error.URLError as error:
+            raise ApiError(None, f"API request failed: {error.reason}") from error
+
+        if not response_body:
+            return None
+        try:
+            return json.loads(response_body)
+        except json.JSONDecodeError as error:
+            raise ApiError(None, "API returned a non-JSON response.") from error
+
+    @staticmethod
+    def _error_message(response_body: str, fallback: str) -> str:
+        try:
+            parsed = json.loads(response_body)
+        except json.JSONDecodeError:
+            return fallback
+        return str(parsed.get("message", fallback))
+
+
+class GitHubClient:
+    def __init__(
+        self,
+        token: str,
+        repository: str,
+        transport: HttpTransport | None = None,
+    ) -> None:
+        if not token:
+            raise RuntimeError("GH_TOKEN is not set.")
+        self.owner, self.repo = repository.split("/", maxsplit=1)
+        self.transport = transport or HttpTransport()
+        self.headers = {
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "Content-Type": "application/json",
+            "User-Agent": "vllm-ci-command",
+            "X-GitHub-Api-Version": "2022-11-28",
+        }
+
+    def _request(
+        self,
+        path: str,
+        *,
+        body: Mapping[str, Any] | None = None,
+        method: str = "GET",
+    ) -> Any:
+        return self.transport.request(
+            f"https://api.github.com{path}",
+            body=body,
+            headers=self.headers,
+            method=method,
+        )
+
+    def _repo_path(self, suffix: str) -> str:
+        owner = urllib.parse.quote(self.owner, safe="")
+        repo = urllib.parse.quote(self.repo, safe="")
+        return f"/repos/{owner}/{repo}{suffix}"
+
+    def _paginate(self, path: str) -> list[dict[str, Any]]:
+        results: list[dict[str, Any]] = []
+        separator = "&" if "?" in path else "?"
+        for page in range(1, 101):
+            response = self._request(f"{path}{separator}per_page=100&page={page}")
+            if not isinstance(response, list):
+                raise ApiError(None, "GitHub API returned an invalid list response.")
+            results.extend(response)
+            if len(response) < 100:
+                return results
+        raise ApiError(None, "GitHub API pagination exceeded 10,000 results.")
+
+    def get_pr(self, number: int) -> dict[str, Any]:
+        return self._request(self._repo_path(f"/pulls/{number}"))
+
+    def get_permission(self, actor: str) -> str:
+        username = urllib.parse.quote(actor, safe="")
+        try:
+            response = self._request(
+                self._repo_path(f"/collaborators/{username}/permission")
+            )
+        except ApiError as error:
+            if error.status == 404:
+                return "none"
+            raise
+        return str(response["permission"])
+
+    def get_review_decision(self, number: int) -> str | None:
+        query = """
+          query($owner: String!, $repo: String!, $number: Int!) {
+            repository(owner: $owner, name: $repo) {
+              pullRequest(number: $number) {
+                reviewDecision
+              }
+            }
+          }
+        """
+        response = self._request(
+            "/graphql",
+            body={
+                "query": query,
+                "variables": {
+                    "number": number,
+                    "owner": self.owner,
+                    "repo": self.repo,
+                },
+            },
+            method="POST",
+        )
+        return response["data"]["repository"]["pullRequest"]["reviewDecision"]
+
+    def list_reviews(self, number: int) -> list[dict[str, Any]]:
+        return self._paginate(self._repo_path(f"/pulls/{number}/reviews"))
+
+    def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
+        return self._paginate(
+            self._repo_path(f"/issues/comments/{comment_id}/reactions")
+        )
+
+    def add_reaction(self, comment_id: int, content: str) -> None:
+        self._request(
+            self._repo_path(f"/issues/comments/{comment_id}/reactions"),
+            body={"content": content},
+            method="POST",
+        )
+
+    def add_comment(self, issue_number: int, body: str) -> None:
+        self._request(
+            self._repo_path(f"/issues/{issue_number}/comments"),
+            body={"body": body},
+            method="POST",
+        )
+
+
+class BuildkiteClient:
+    def __init__(
+        self,
+        token: str,
+        organization: str,
+        pipeline: str,
+        transport: HttpTransport | None = None,
+    ) -> None:
+        self.token = token
+        self.transport = transport or HttpTransport()
+        organization = urllib.parse.quote(organization, safe="")
+        pipeline = urllib.parse.quote(pipeline, safe="")
+        self.base_url = (
+            "https://api.buildkite.com/v2/organizations/"
+            f"{organization}/pipelines/{pipeline}/builds"
+        )
+
+    def _request(
+        self,
+        *,
+        body: Mapping[str, Any] | None = None,
+        method: str = "GET",
+        path: str = "",
+        query: Sequence[tuple[str, str]] = (),
+    ) -> Any:
+        if not self.token:
+            raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
+        url = f"{self.base_url}{path}"
+        if query:
+            url = f"{url}?{urllib.parse.urlencode(query)}"
+        return self.transport.request(
+            url,
+            body=body,
+            headers={
+                "Authorization": f"Bearer {self.token}",
+                "Content-Type": "application/json",
+                "User-Agent": "vllm-ci-command",
+            },
+            method=method,
+        )
+
+    def list_builds(
+        self,
+        commit: str,
+        *,
+        metadata: tuple[str, str] | None = None,
+    ) -> list[dict[str, Any]]:
+        query = [
+            ("commit", commit),
+            ("exclude_jobs", "true"),
+            ("exclude_pipeline", "true"),
+            ("per_page", "100"),
+        ]
+        if metadata:
+            key, value = metadata
+            query.append((f"meta_data[{key}]", value))
+        response = self._request(query=query)
+        if not isinstance(response, list):
+            raise ApiError(None, "Buildkite API returned an invalid build list.")
+        return response
+
+    def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]:
+        return self._request(body=body, method="POST")
+
+    def retry_failed_jobs(
+        self,
+        build_number: int,
+        states: str,
+    ) -> dict[str, Any]:
+        number = urllib.parse.quote(str(build_number), safe="")
+        return self._request(
+            body={"states": states},
+            method="PUT",
+            path=f"/{number}/retry_failed_jobs",
+        )
+
+
+def parse_command(body: str) -> str | None:
+    if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
+        return body
+    return None
+
+
+def parse_trusted_users(value: str = "") -> set[str]:
+    return {
+        user.casefold() for item in value.split(",") for user in item.split() if user
+    }
+
+
+def has_ready_label(pr: Mapping[str, Any]) -> bool:
+    return any(label["name"] in READY_LABELS for label in pr["labels"])
+
+
+def is_trusted_permission(permission: str) -> bool:
+    return permission in TRUSTED_PERMISSIONS
+
+
+def authorize(
+    *,
+    actor: str,
+    permission: str,
+    pr: Mapping[str, Any],
+    trusted_approval: bool = False,
+    trusted_users: set[str] | None = None,
+) -> tuple[bool, str]:
+    trusted_users = trusted_users or set()
+    if is_trusted_permission(permission):
+        return True, f"repository {permission} permission"
+    if actor.casefold() in trusted_users:
+        return True, "configured trusted contributor"
+    if actor.casefold() != pr["user"]["login"].casefold():
+        return (
+            False,
+            "Only reviewers with write access can run CI before it is "
+            "delegated to the PR author.",
+        )
+    if pr["draft"]:
+        return False, "PR authors cannot run CI while the PR is a draft."
+    if has_ready_label(pr):
+        return True, "ready label"
+    if trusted_approval:
+        return True, "approval from a trusted reviewer"
+    return (
+        False,
+        "A reviewer with write access must run `/ci run`, approve the PR, "
+        "or add the `ready` label first.",
+    )
+
+
+def has_trusted_approval(
+    github: GitHubClient,
+    number: int,
+    trusted_users: set[str],
+) -> bool:
+    if github.get_review_decision(number) != "APPROVED":
+        return False
+
+    latest_review_states: dict[str, tuple[str, str]] = {}
+    for review in github.list_reviews(number):
+        user = review.get("user") or {}
+        login = user.get("login")
+        state = review.get("state")
+        if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}:
+            latest_review_states[login.casefold()] = (login, state)
+
+    for login, state in latest_review_states.values():
+        if state != "APPROVED":
+            continue
+        if login.casefold() in trusted_users:
+            return True
+        if is_trusted_permission(github.get_permission(login)):
+            return True
+    return False
+
+
+def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool:
+    pull_request = build.get("pull_request")
+    if isinstance(pull_request, Mapping):
+        build_pr_number = pull_request.get("id", pull_request.get("number"))
+        if build_pr_number is not None:
+            return str(build_pr_number) == str(pr_number)
+    metadata = build.get("meta_data") or {}
+    return str(metadata.get("github-pr-number")) == str(pr_number)
+
+
+def is_active_build(build: Mapping[str, Any]) -> bool:
+    return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES
+
+
+def select_latest_build(
+    builds: Sequence[dict[str, Any]],
+    pr_number: int,
+) -> dict[str, Any] | None:
+    matching = [build for build in builds if is_build_for_pr(build, pr_number)]
+    return max(matching, key=lambda build: build.get("created_at", ""), default=None)
+
+
+def create_build_payload(
+    *,
+    actor: str,
+    comment_id: int,
+    pr: Mapping[str, Any],
+) -> dict[str, Any]:
+    return {
+        "commit": pr["head"]["sha"],
+        "branch": pr["head"]["ref"],
+        "message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
+        "pull_request_id": pr["number"],
+        "pull_request_base_branch": pr["base"]["ref"],
+        "pull_request_repository": pr["head"]["repo"]["clone_url"],
+        "pull_request_labels": [label["name"] for label in pr["labels"]],
+        "ignore_pipeline_branch_filters": True,
+        "env": {
+            "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
+            "VLLM_CI_TRIGGERED_BY": actor,
+        },
+        "meta_data": {
+            "github-comment-id": str(comment_id),
+            "github-pr-number": str(pr["number"]),
+            "github-triggered-by": actor,
+        },
+    }
+
+
+def add_reaction_safely(
+    github: GitHubClient,
+    comment_id: int,
+    content: str,
+) -> None:
+    try:
+        github.add_reaction(comment_id, content)
+    except Exception as error:
+        print(f"Could not add {content} reaction: {error}", file=sys.stderr)
+
+
+def is_already_handled(github: GitHubClient, comment_id: int) -> bool:
+    return any(
+        reaction.get("content") in {"rocket", "-1"}
+        and (reaction.get("user") or {}).get("login") == "github-actions[bot]"
+        for reaction in github.list_reactions(comment_id)
+    )
+
+
+def handle_run_ci(
+    *,
+    actor: str,
+    buildkite: BuildkiteClient,
+    comment_id: int,
+    github: GitHubClient,
+    pr: Mapping[str, Any],
+) -> str:
+    duplicate_builds = buildkite.list_builds(
+        pr["head"]["sha"],
+        metadata=("github-comment-id", str(comment_id)),
+    )
+    duplicate = select_latest_build(duplicate_builds, pr["number"])
+    if duplicate:
+        return f"CI was already requested by this comment: {duplicate['web_url']}"
+
+    current_builds = buildkite.list_builds(pr["head"]["sha"])
+    active_build = next(
+        (
+            build
+            for build in current_builds
+            if is_build_for_pr(build, pr["number"]) and is_active_build(build)
+        ),
+        None,
+    )
+    if active_build:
+        return f"CI is already running for this commit: {active_build['web_url']}"
+
+    current_pr = github.get_pr(pr["number"])
+    if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
+        return (
+            "The PR head changed while processing the command. Comment `/ci run` again."
+        )
+
+    build = buildkite.create_build(
+        create_build_payload(
+            actor=actor,
+            comment_id=comment_id,
+            pr=current_pr,
+        )
+    )
+    return (
+        f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) "
+        f"for commit `{current_pr['head']['sha'][:12]}`."
+    )
+
+
+def handle_retry_failed(
+    *,
+    buildkite: BuildkiteClient,
+    pr: Mapping[str, Any],
+) -> str:
+    builds = buildkite.list_builds(pr["head"]["sha"])
+    build = select_latest_build(builds, pr["number"])
+    if not build:
+        return "No CI build exists for the current PR commit. Use `/ci run` first."
+    if not build.get("finished_at") or is_active_build(build):
+        return f"CI is still running for this commit: {build['web_url']}"
+
+    retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
+    if retried["retried_jobs_count"] == 0:
+        return (
+            f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}"
+        )
+    return (
+        f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
+        f"[Buildkite CI #{build['number']}]({build['web_url']})."
+    )
+
+
+def run(
+    event: Mapping[str, Any],
+    github: GitHubClient,
+    buildkite: BuildkiteClient,
+    trusted_users_value: str = "",
+) -> None:
+    command = parse_command(event["comment"]["body"])
+    if not command or "pull_request" not in event["issue"]:
+        return
+
+    issue_number = event["issue"]["number"]
+    comment_id = event["comment"]["id"]
+    actor = event["comment"]["user"]["login"]
+
+    if is_already_handled(github, comment_id):
+        print(f"Comment {comment_id} was already handled.")
+        return
+    add_reaction_safely(github, comment_id, "eyes")
+
+    try:
+        pr = github.get_pr(issue_number)
+        permission = github.get_permission(actor)
+        if pr["state"] != "open":
+            github.add_comment(issue_number, "CI commands require an open PR.")
+            return
+
+        trusted_users = parse_trusted_users(trusted_users_value)
+        should_check_approval = (
+            not is_trusted_permission(permission)
+            and actor.casefold() not in trusted_users
+            and actor.casefold() == pr["user"]["login"].casefold()
+            and not pr["draft"]
+            and not has_ready_label(pr)
+        )
+        trusted_approval = should_check_approval and has_trusted_approval(
+            github,
+            issue_number,
+            trusted_users,
+        )
+        allowed, reason = authorize(
+            actor=actor,
+            permission=permission,
+            pr=pr,
+            trusted_approval=trusted_approval,
+            trusted_users=trusted_users,
+        )
+        if not allowed:
+            add_reaction_safely(github, comment_id, "-1")
+            github.add_comment(issue_number, f"@{actor}, {reason}")
+            return
+
+        print(f"Authorized @{actor}: {reason}")
+        if command == COMMAND_RUN_CI:
+            message = handle_run_ci(
+                actor=actor,
+                buildkite=buildkite,
+                comment_id=comment_id,
+                github=github,
+                pr=pr,
+            )
+        else:
+            message = handle_retry_failed(buildkite=buildkite, pr=pr)
+        add_reaction_safely(github, comment_id, "rocket")
+        github.add_comment(issue_number, message)
+    except Exception:
+        add_reaction_safely(github, comment_id, "confused")
+        raise
+
+
+def main() -> None:
+    event_path = os.environ["GITHUB_EVENT_PATH"]
+    with open(event_path, encoding="utf-8") as event_file:
+        event = json.load(event_file)
+
+    if not parse_command(event["comment"]["body"]):
+        return
+
+    github = GitHubClient(
+        os.environ.get("GH_TOKEN", ""),
+        os.environ["GITHUB_REPOSITORY"],
+    )
+    buildkite = BuildkiteClient(
+        os.environ.get("BUILDKITE_API_TOKEN", ""),
+        os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
+        os.environ.get("BUILDKITE_PIPELINE", "ci"),
+    )
+    run(
+        event,
+        github,
+        buildkite,
+        os.environ.get("CI_TRUSTED_USERS", ""),
+    )
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/scripts/test_run_ci_command.py

@@ -0,0 +1,363 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import unittest
+from typing import Any
+
+from run_ci_command import (
+    COMMAND_RETRY_FAILED,
+    COMMAND_RUN_CI,
+    RETRY_STATES,
+    BuildkiteClient,
+    authorize,
+    create_build_payload,
+    has_trusted_approval,
+    is_active_build,
+    is_build_for_pr,
+    parse_command,
+    parse_trusted_users,
+    run,
+    select_latest_build,
+)
+
+
+def make_pr(**overrides: Any) -> dict[str, Any]:
+    pr = {
+        "base": {"ref": "main"},
+        "draft": False,
+        "head": {
+            "ref": "feature",
+            "repo": {"clone_url": "https://github.com/contributor/vllm.git"},
+            "sha": "0123456789abcdef",
+        },
+        "labels": [],
+        "number": 42,
+        "state": "open",
+        "user": {"login": "author"},
+    }
+    pr.update(overrides)
+    return pr
+
+
+def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]:
+    return {
+        "comment": {
+            "body": command,
+            "id": 99,
+            "user": {"login": actor},
+        },
+        "issue": {
+            "number": 42,
+            "pull_request": {},
+        },
+    }
+
+
+class FakeGitHub:
+    def __init__(
+        self,
+        *,
+        permission: str = "write",
+        permissions: dict[str, str] | None = None,
+        pr: dict[str, Any] | None = None,
+        review_decision: str = "REVIEW_REQUIRED",
+        reviews: list[dict[str, Any]] | None = None,
+    ) -> None:
+        self.comments: list[str] = []
+        self.permission = permission
+        self.permissions = permissions or {}
+        self.pr = pr or make_pr()
+        self.reactions: list[str] = []
+        self.review_decision = review_decision
+        self.reviews = reviews or []
+
+    def get_pr(self, number: int) -> dict[str, Any]:
+        return self.pr
+
+    def get_permission(self, actor: str) -> str:
+        return self.permissions.get(actor, self.permission)
+
+    def get_review_decision(self, number: int) -> str:
+        return self.review_decision
+
+    def list_reviews(self, number: int) -> list[dict[str, Any]]:
+        return self.reviews
+
+    def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
+        return []
+
+    def add_reaction(self, comment_id: int, content: str) -> None:
+        self.reactions.append(content)
+
+    def add_comment(self, issue_number: int, body: str) -> None:
+        self.comments.append(body)
+
+
+class FakeBuildkite:
+    def __init__(
+        self,
+        build_lists: list[list[dict[str, Any]]] | None = None,
+    ) -> None:
+        self.build_lists = build_lists or []
+        self.created_builds: list[dict[str, Any]] = []
+        self.list_calls: list[tuple[str, tuple[str, str] | None]] = []
+        self.retry_calls: list[tuple[int, str]] = []
+
+    def list_builds(
+        self,
+        commit: str,
+        *,
+        metadata: tuple[str, str] | None = None,
+    ) -> list[dict[str, Any]]:
+        self.list_calls.append((commit, metadata))
+        return self.build_lists.pop(0)
+
+    def create_build(self, body: dict[str, Any]) -> dict[str, Any]:
+        self.created_builds.append(body)
+        return {
+            "number": 123,
+            "web_url": "https://buildkite.example/builds/123",
+        }
+
+    def retry_failed_jobs(
+        self,
+        build_number: int,
+        states: str,
+    ) -> dict[str, Any]:
+        self.retry_calls.append((build_number, states))
+        return {"retried_jobs_count": 3}
+
+
+class FakeTransport:
+    def __init__(self, response: Any) -> None:
+        self.calls: list[dict[str, Any]] = []
+        self.response = response
+
+    def request(self, url: str, **kwargs: Any) -> Any:
+        self.calls.append({"url": url, **kwargs})
+        return self.response
+
+
+class RunCiCommandTest(unittest.TestCase):
+    def test_only_exact_ci_commands_are_accepted(self) -> None:
+        self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
+        self.assertEqual(
+            parse_command(COMMAND_RETRY_FAILED),
+            COMMAND_RETRY_FAILED,
+        )
+        self.assertIsNone(parse_command("/ci run please"))
+        self.assertIsNone(parse_command(" /ci run"))
+
+    def test_write_access_authorizes_reviewers_and_authors(self) -> None:
+        allowed, _ = authorize(
+            actor="reviewer",
+            permission="write",
+            pr=make_pr(),
+        )
+        self.assertTrue(allowed)
+
+    def test_configured_trusted_contributors_can_run_ci(self) -> None:
+        trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO")
+        allowed, _ = authorize(
+            actor="trusted-two",
+            permission="read",
+            pr=make_pr(),
+            trusted_users=trusted_users,
+        )
+        self.assertTrue(allowed)
+
+    def test_authors_need_an_approval_or_ready_label(self) -> None:
+        pending, _ = authorize(
+            actor="author",
+            permission="read",
+            pr=make_pr(),
+        )
+        approved, _ = authorize(
+            actor="author",
+            permission="read",
+            pr=make_pr(),
+            trusted_approval=True,
+        )
+        ready, _ = authorize(
+            actor="author",
+            permission="read",
+            pr=make_pr(labels=[{"name": "ready"}]),
+        )
+        self.assertFalse(pending)
+        self.assertTrue(approved)
+        self.assertTrue(ready)
+
+    def test_non_author_contributors_without_write_are_denied(self) -> None:
+        allowed, _ = authorize(
+            actor="contributor",
+            permission="read",
+            pr=make_pr(),
+            trusted_approval=True,
+        )
+        self.assertFalse(allowed)
+
+    def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None:
+        allowed, _ = authorize(
+            actor="author",
+            permission="read",
+            pr=make_pr(draft=True, labels=[{"name": "ready"}]),
+            trusted_approval=True,
+        )
+        self.assertFalse(allowed)
+
+    def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None:
+        approved_review = {
+            "state": "APPROVED",
+            "user": {"login": "reviewer"},
+        }
+        trusted = FakeGitHub(
+            permission="read",
+            permissions={"reviewer": "write"},
+            review_decision="APPROVED",
+            reviews=[approved_review],
+        )
+        untrusted = FakeGitHub(
+            permission="read",
+            review_decision="APPROVED",
+            reviews=[approved_review],
+        )
+        self.assertTrue(has_trusted_approval(trusted, 42, set()))
+        self.assertFalse(has_trusted_approval(untrusted, 42, set()))
+
+    def test_build_matching_is_scoped_to_the_pr(self) -> None:
+        self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42))
+        self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42))
+        self.assertTrue(
+            is_build_for_pr(
+                {"meta_data": {"github-pr-number": "42"}},
+                42,
+            )
+        )
+
+    def test_latest_build_selection_ignores_other_prs(self) -> None:
+        latest = select_latest_build(
+            [
+                {
+                    "created_at": "2026-07-28T02:00:00Z",
+                    "number": 3,
+                    "pull_request": {"id": 43},
+                },
+                {
+                    "created_at": "2026-07-28T01:00:00Z",
+                    "number": 2,
+                    "pull_request": {"id": 42},
+                },
+                {
+                    "created_at": "2026-07-28T00:00:00Z",
+                    "number": 1,
+                    "pull_request": {"id": 42},
+                },
+            ],
+            42,
+        )
+        self.assertEqual(latest["number"], 2)
+
+    def test_active_build_states_prevent_duplicate_runs(self) -> None:
+        self.assertTrue(is_active_build({"state": "scheduled"}))
+        self.assertTrue(is_active_build({"state": "running"}))
+        self.assertTrue(is_active_build({"state": "waiting"}))
+        self.assertTrue(is_active_build({"blocked": True, "state": "passed"}))
+        self.assertFalse(is_active_build({"state": "failed"}))
+
+    def test_build_payload_preserves_pr_context(self) -> None:
+        payload = create_build_payload(
+            actor="reviewer",
+            comment_id=99,
+            pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]),
+        )
+        self.assertEqual(
+            payload,
+            {
+                "commit": "0123456789abcdef",
+                "branch": "feature",
+                "message": "PR #42 /ci run by @reviewer",
+                "pull_request_id": 42,
+                "pull_request_base_branch": "main",
+                "pull_request_repository": ("https://github.com/contributor/vllm.git"),
+                "pull_request_labels": ["ready", "v1"],
+                "ignore_pipeline_branch_filters": True,
+                "env": {
+                    "VLLM_CI_GITHUB_COMMENT_ID": "99",
+                    "VLLM_CI_TRIGGERED_BY": "reviewer",
+                },
+                "meta_data": {
+                    "github-comment-id": "99",
+                    "github-pr-number": "42",
+                    "github-triggered-by": "reviewer",
+                },
+            },
+        )
+
+    def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None:
+        github = FakeGitHub()
+        buildkite = FakeBuildkite([[], []])
+        run(make_event(COMMAND_RUN_CI), github, buildkite)
+
+        self.assertEqual(len(buildkite.created_builds), 1)
+        self.assertEqual(
+            buildkite.created_builds[0]["message"],
+            "PR #42 /ci run by @reviewer",
+        )
+        self.assertEqual(github.reactions, ["eyes", "rocket"])
+        self.assertIn("Buildkite CI #123", github.comments[0])
+
+    def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
+        github = FakeGitHub(
+            permission="read",
+            pr=make_pr(),
+            review_decision="REVIEW_REQUIRED",
+        )
+        buildkite = FakeBuildkite()
+        run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
+
+        self.assertEqual(buildkite.list_calls, [])
+        self.assertEqual(github.reactions, ["eyes", "-1"])
+        self.assertIn("approve the PR", github.comments[0])
+
+    def test_ci_retry_uses_latest_current_sha_build(self) -> None:
+        github = FakeGitHub(
+            permission="read",
+            pr=make_pr(labels=[{"name": "ready"}]),
+        )
+        buildkite = FakeBuildkite(
+            [
+                [
+                    {
+                        "created_at": "2026-07-28T01:00:00Z",
+                        "finished_at": "2026-07-28T02:00:00Z",
+                        "number": 123,
+                        "pull_request": {"id": 42},
+                        "state": "failed",
+                        "web_url": "https://buildkite.example/builds/123",
+                    }
+                ]
+            ]
+        )
+        run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
+
+        self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
+        self.assertIn("Queued 3 failed job", github.comments[0])
+
+    def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None:
+        transport = FakeTransport({"retried_jobs_count": 2})
+        client = BuildkiteClient(
+            "secret",
+            "vllm",
+            "ci",
+            transport=transport,
+        )
+        client.retry_failed_jobs(123, RETRY_STATES)
+
+        call = transport.calls[0]
+        self.assertEqual(call["method"], "PUT")
+        self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
+        self.assertEqual(call["body"], {"states": RETRY_STATES})
+
+
+if __name__ == "__main__":
+    unittest.main()

CMakeLists.txt

@@ -219,10 +219,8 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
   # the set of architectures we want to compile for and remove the from the
   # CMAKE_CUDA_FLAGS so that they are not applied globally.
   #
-  # `+PTX` in TORCH_CUDA_ARCH_LIST is not preserved here. It is emitted by torch
-  # as `code=compute_*`, while extract_unique_cuda_archs_ascending() records only
-  # `arch=compute_*`. If a kernel really needs PTX, add `+PTX` to that kernel's
-  # component-specific arch list below.
+  # `+PTX` in TORCH_CUDA_ARCH_LIST is not preserved here. If a kernel really
+  # needs PTX, add `+PTX` to that kernel's component-specific arch list below.
   #
   clear_cuda_arches(CUDA_ARCH_FLAGS)
   extract_unique_cuda_archs_ascending(CUDA_ARCHS "${CUDA_ARCH_FLAGS}")
@@ -232,6 +230,13 @@ if(VLLM_GPU_LANG STREQUAL "CUDA")
   cuda_archs_loose_intersection(CUDA_ARCHS
     "${CUDA_SUPPORTED_ARCHS}" "${CUDA_ARCHS}")
   message(STATUS "CUDA supported target architectures: ${CUDA_ARCHS}")
+  if(NOT CUDA_ARCHS)
+    message(FATAL_ERROR
+      "No supported CUDA architectures; the build would produce a binary "
+      "with no usable kernels. Detected gencode flags: ${CUDA_ARCH_FLAGS}; "
+      "supported: ${CUDA_SUPPORTED_ARCHS}. "
+      "Set TORCH_CUDA_ARCH_LIST for your GPU (e.g. 12.0).")
+  endif()
 else()
   #
   # For other GPU targets override the GPU architectures detected by cmake/torch

cmake/utils.cmake

@@ -241,14 +241,15 @@ endmacro()
 # `<major>.<minor>`, dedupes them and then sorts them in ascending order and 
 # stores them in `OUT_ARCHES`.
 #
-# Example:
-#   CUDA_ARCH_FLAGS="-gencode arch=compute_75,code=sm_75;...;-gencode arch=compute_90a,code=sm_90a" 
-#   extract_unique_cuda_archs_ascending(OUT_ARCHES CUDA_ARCH_FLAGS)
-#   OUT_ARCHES="7.5;...;9.0"
+# Prefer `code=sm_*`; fall back to `arch=compute_*` for PTX-only flags.
+# This handles mismatches such as `arch=compute_20,code=sm_121`.
 function(extract_unique_cuda_archs_ascending OUT_ARCHES CUDA_ARCH_FLAGS)
   set(_CUDA_ARCHES)
   foreach(_ARCH ${CUDA_ARCH_FLAGS})
-    string(REGEX MATCH "arch=compute_\([0-9]+[af]?\)" _COMPUTE ${_ARCH})
+    string(REGEX MATCH "code=sm_\([0-9]+[af]?\)" _COMPUTE ${_ARCH})
+    if (NOT _COMPUTE)
+      string(REGEX MATCH "arch=compute_\([0-9]+[af]?\)" _COMPUTE ${_ARCH})
+    endif()
     if (_COMPUTE)
       set(_COMPUTE ${CMAKE_MATCH_1})
     endif()

csrc/cpu/cpu_types_vxe.hpp

@@ -269,7 +269,7 @@ struct FP32Vec4 : public Vec<FP32Vec4> {
 
   explicit FP32Vec4(__vector float data) : reg(data) {}
 
-  explicit FP32Vec4(const FP32Vec4& data) : reg(data.reg) {}
+  FP32Vec4(const FP32Vec4& data) : reg(data.reg) {}
 };
 
 struct FP32Vec8 : public Vec<FP32Vec8> {
@@ -298,7 +298,7 @@ struct FP32Vec8 : public Vec<FP32Vec8> {
 
   explicit FP32Vec8(f32x4x2_t data) : reg(data) {}
 
-  explicit FP32Vec8(const FP32Vec8& data) {
+  FP32Vec8(const FP32Vec8& data) {
     reg.val[0] = data.reg.val[0];
     reg.val[1] = data.reg.val[1];
   }
@@ -643,7 +643,7 @@ struct FP32Vec16 : public Vec<FP32Vec16> {
 
   explicit FP32Vec16(f32x4x4_t data) : reg(data) {}
 
-  explicit FP32Vec16(const FP32Vec16& data) {
+  FP32Vec16(const FP32Vec16& data) {
     reg.val[0] = data.reg.val[0];
     reg.val[1] = data.reg.val[1];
     reg.val[2] = data.reg.val[2];

csrc/libtorch_stable/layernorm_kernels.cu

@@ -249,7 +249,9 @@ void rms_norm(torch::stable::Tensor& out,    // [..., hidden_size]
   int64_t input_shape_d3 = (num_dims >= 4) ? input.size(-3) : 0;
 
   // For large num_tokens, use smaller blocks to increase SM concurrency.
-  const int max_block_size = (num_tokens < 256) ? 1024 : 256;
+  const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
+  const int max_block_size =
+      batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
   dim3 grid(num_tokens);
   const torch::stable::accelerator::DeviceGuard device_guard(
       input.get_device_index());
@@ -325,8 +327,13 @@ void fused_add_rms_norm(torch::stable::Tensor& input,     // [..., hidden_size]
   /* This kernel is memory-latency bound in many scenarios.
      When num_tokens is large, a smaller block size allows
      for increased block occupancy on CUs and better latency
-     hiding on global mem ops. */
-  const int max_block_size = (num_tokens < 256) ? 1024 : 256;
+     hiding on global mem ops. In batch-invariant mode the block size must
+     not depend on num_tokens, otherwise the same token would use a different
+     reduction width (and thus a different floating-point summation order)
+     across batches; lock it to 1024 to keep results bit-exact. */
+  const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
+  const int max_block_size =
+      batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
   dim3 block(std::min(hidden_size, max_block_size));
   const torch::stable::accelerator::DeviceGuard device_guard(
       input.get_device_index());
@@ -337,7 +344,6 @@ void fused_add_rms_norm(torch::stable::Tensor& input,     // [..., hidden_size]
   auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
   bool offsets_are_multiple_of_vector_width =
       hidden_size % vector_width == 0 && input_stride % vector_width == 0;
-  bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
   const bool has_weight = weight.has_value();
   if (has_weight) {
     auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight->data_ptr());

csrc/libtorch_stable/layernorm_quant_kernels.cu

@@ -215,7 +215,9 @@ void rms_norm_static_fp8_quant(
   int num_tokens = input.numel() / hidden_size;
 
   // For large num_tokens, use smaller blocks to increase SM concurrency.
-  const int max_block_size = (num_tokens < 256) ? 1024 : 256;
+  const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
+  const int max_block_size =
+      batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
   dim3 grid(num_tokens);
   const torch::stable::accelerator::DeviceGuard device_guard(
       input.get_device_index());
@@ -279,7 +281,9 @@ void fused_add_rms_norm_static_fp8_quant(
      When num_tokens is large, a smaller block size allows
      for increased block occupancy on CUs and better latency
      hiding on global mem ops. */
-  const int max_block_size = (num_tokens < 256) ? 1024 : 256;
+  const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
+  const int max_block_size =
+      batch_invariant_launch ? 1024 : ((num_tokens < 256) ? 1024 : 256);
   dim3 block(std::min(hidden_size, max_block_size));
   const torch::stable::accelerator::DeviceGuard device_guard(
       input.get_device_index());
@@ -296,7 +300,6 @@ void fused_add_rms_norm_static_fp8_quant(
   auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
   bool ptrs_are_aligned =
       inp_ptr % 16 == 0 && res_ptr % 16 == 0 && wt_ptr % 16 == 0;
-  bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
   if (ptrs_are_aligned && hidden_size % 8 == 0 && input_stride % 8 == 0 &&
       !batch_invariant_launch) {
     LAUNCH_FUSED_ADD_RMS_NORM(8);

csrc/libtorch_stable/quantization/fused_kernels/fused_layernorm_dynamic_per_token_quant.cu

@@ -2,6 +2,7 @@
 #include "../../torch_utils.h"
 
 #include "../../dispatch_utils.h"
+#include "../../../core/batch_invariant.hpp"
 #include "layernorm_utils.cuh"
 #include "quant_conversions.cuh"
 
@@ -231,7 +232,9 @@ void rms_norm_per_block_quant_dispatch(
   auto num_tokens = input.numel() / hidden_size;
 
   dim3 grid(num_tokens);
-  const int max_block_size = (num_tokens <= 256) ? 512 : 256;
+  const bool batch_invariant_launch = vllm::vllm_is_batch_invariant();
+  const int max_block_size =
+      batch_invariant_launch ? 512 : ((num_tokens <= 256) ? 512 : 256);
   dim3 block(std::min(hidden_size, max_block_size));
   const torch::stable::accelerator::DeviceGuard device_guard(
       input.get_device_index());

csrc/libtorch_stable/torch_bindings.cpp

@@ -591,7 +591,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
       "limit=7.0) "
       "-> ()");
 
-  // Kimi SITU (SituGLU) gated activation. linear_beta<=0 means unset.
+  // SituGLU implementation used in Kimi models.
   ops.def(
       "situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
       "linear_beta=-1.0) -> ()");

csrc/spinloop.cpp

@@ -7,7 +7,7 @@ extern "C" {
 
 #if defined(__i386__) || defined(__x86_64__)
   #include <cpuid.h>
-  #include <mwaitxintrin.h>
+  #include <x86intrin.h>
 #endif
 
 #if defined(CLOCK_MONOTONIC_RAW)

docker/Dockerfile.s390x

@@ -61,13 +61,13 @@ ENV C_INCLUDE_PATH="/usr/local/include:$C_INCLUDE_PATH"
 
 FROM python-install AS torch-vision
 # Install torchvision
-ARG TORCH_VISION_VERSION=v0.26.0
+ARG TORCH_VISION_VERSION=v0.28.0
 WORKDIR /tmp
 RUN --mount=type=cache,target=/root/.cache/uv \
     git clone https://github.com/pytorch/vision.git && \
     cd vision && \
     git checkout $TORCH_VISION_VERSION && \
-    uv pip install torch==2.11.0 --index-url https://download.pytorch.org/whl/cpu && \
+    uv pip install torch==2.13.0 --index-url https://download.pytorch.org/whl/cpu && \
     python setup.py bdist_wheel
 
 FROM python-install AS hf-xet-builder

docs/contributing/README.md

@@ -301,8 +301,10 @@ review process:
   isn't clear or you disagree with a suggestion, feel free to ask for
   clarification or discuss the suggestion.
 - Note that not all CI checks will be executed due to limited computational
-  resources. The reviewer will add `ready` label to the PR when the PR is
-  ready to merge or a full CI run is needed.
+  resources. Reviewers with write access and configured trusted contributors
+  can comment `/ci run` when CI signals are needed before a PR is ready. After
+  the PR is approved or has the `ready` label, the PR author can use `/ci run`
+  or `/ci retry`. New commits do not start CI automatically.
 
 ### Pull Request Limits and Escalation
 

docs/deployment/integrations/llm-d.md

@@ -1,5 +1,37 @@
 # llm-d
 
-vLLM can be deployed with [llm-d](https://github.com/llm-d/llm-d), a Kubernetes-native distributed inference serving stack providing well-lit paths for anyone to serve large generative AI models at scale. It helps achieve the fastest "time to state-of-the-art (SOTA) performance" for key OSS models across most hardware accelerators and infrastructure providers.
+[llm-d](https://llm-d.ai/) is a Kubernetes-native distributed inference framework for serving large language models at scale, with vLLM as its primary inference engine. llm-d coordinates a fleet of vLLM instances across a cluster so that performance holds up under real production traffic, achieving the fastest "time to state-of-the-art (SOTA) performance" for key OSS models across most hardware accelerators.
 
-You can use vLLM with llm-d directly by following [the official guides](https://llm-d.ai/docs/guides) or via [KServe's LLMInferenceService](https://kserve.github.io/website/docs/model-serving/generative-inference/llmisvc/llmisvc-overview).
+It is a [CNCF Sandbox project](https://www.cncf.io/blog/2026/03/24/welcome-llm-d-to-the-cncf-evolving-kubernetes-into-sota-ai-infrastructure/) founded by Red Hat, Google Cloud, IBM Research, CoreWeave, and NVIDIA.
+
+## What llm-d adds to vLLM
+
+A single vLLM server is fast, but at scale the picture changes: across many replicas, cache locality breaks under round-robin load balancing, long prompts inflate time-to-first-token, and accelerators sit underused. llm-d adds the cluster-level layer that vLLM does not aim to provide on its own:
+
+- **[Prefix-aware routing](https://llm-d.ai/docs/guides/precise-prefix-cache-aware).** Instead of round-robin, llm-d reads vLLM's KV-cache events and routes each request to the replica that already holds its prefix, reusing cache instead of recomputing it.
+- **[Distributed KV-cache management](https://llm-d.ai/docs/guides#advanced-kv-cache-management).** A global index tracks which token blocks live on which replica, and [tiered offloading](https://llm-d.ai/docs/guides/tiered-prefix-cache) spills cache to CPU memory or local SSD, extending the working set beyond accelerator HBM.
+- **[Prefill/decode disaggregation](https://llm-d.ai/docs/guides/pd-disaggregation).** Prompt processing and token generation run on separate vLLM workers, with KV-cache moved over the vLLM [NIXL connector](https://docs.vllm.ai/en/latest/features/nixl_connector_usage/), lowering TTFT and steadying per-token latency on long prompts.
+- **[Wide expert-parallelism](https://llm-d.ai/docs/guides/wide-expert-parallelism).** Serve large Mixture-of-Experts models such as DeepSeek-R1 and GPT-OSS across nodes with combined data and expert parallelism, for more KV-cache capacity and throughput.
+- **SLO-aware [autoscaling](https://llm-d.ai/docs/guides/workload-autoscaling) and [flow control](https://llm-d.ai/docs/guides/flow-control).** Scale vLLM pools on real inference signals (queue depth, true demand) rather than raw GPU utilization, with multi-tenant fairness and priority dispatch.
+
+These are composable. Most teams start by adding prefix-aware routing over an existing vLLM pool, then layer in the rest as specific bottlenecks appear.
+
+## Performance
+
+Representative benchmarked results across accelerators:
+
+- **3x higher output throughput** and **2x faster TTFT** from prefix-aware routing vs round-robin (Llama 3.1 70B, AMD MI300X)
+- **Up to 70% higher tokens/sec** from prefill/decode disaggregation (GPT-OSS, NVIDIA B200)
+- **13.9x throughput** from hierarchical KV offloading at high concurrency vs GPU-only (NVIDIA H100)
+
+See the [full list](https://github.com/llm-d/llm-d#performance-highlights) and reproducible benchmarks on [Prism](https://prism.llm-d.ai/).
+
+## Get started
+
+1. Deploy the [Optimized Baseline](https://llm-d.ai/docs/guides/optimized-baseline) with the [Quickstart](https://llm-d.ai/docs/getting-started/quickstart). It stands up an intelligent router over a vLLM pool on Kubernetes in a tested configuration.
+2. Browse the [well-lit path guides](https://llm-d.ai/docs/guides), each a tested recipe for one of the capabilities above, and add the optimization that fits your workload.
+3. Read the [Introduction](https://llm-d.ai/docs/getting-started) and [Architecture overview](https://llm-d.ai/docs/architecture) to see how the pieces wrap your vLLM deployment.
+
+You can also deploy vLLM with llm-d via [KServe's LLMInferenceService](https://kserve.github.io/website/docs/model-serving/generative-inference/llmisvc/llmisvc-overview).
+
+Questions and contributions are welcome on [GitHub](https://github.com/llm-d/llm-d) and [Slack](https://llm-d.ai/slack).

docs/features/kv_offloading_usage.md

@@ -70,7 +70,8 @@ vllm serve <model> \
 | `cpu_bytes_to_use` | yes | — | both | Total bytes of host memory reserved for the CPU tier across all workers (not per-worker). |
 | `block_size` | no | GPU block size | both | Offloaded block size in tokens; must be a multiple of the GPU block size. Mutually exclusive with `blocks_per_chunk`. |
 | `blocks_per_chunk` | no | `1` | both | Offloaded chunk size in GPU blocks; must be > 0. Alternative to `block_size` for models whose KV cache groups have different block sizes. |
-| `eviction_policy` | no | `lru` | both | Primary tier policy: `lru` or `arc`. |
+| `eviction_policy` | no | `lru` | both | Primary tier policy: built-in `lru`/`arc`, or a custom `CachePolicy` name (see [Custom Eviction Policies](#custom-eviction-policies)). |
+| `cache_policy_module_path` | no | — | both | Python import path for a custom `CachePolicy` not in the built-in registry. Required only when `eviction_policy` is not built-in and wasn't pre-registered via `CachePolicyFactory` (advanced). |
 | `store_threshold` | no | `0` | single-tier | Min lookups before a block is offloaded. Values ≥ 2 are rejected by `TieringOffloadingSpec`. |
 | `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. |
 | `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). |
@@ -78,6 +79,36 @@ vllm serve <model> \
 | `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
 | `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). |
 
+## Custom Eviction Policies
+
+`eviction_policy` resolves through `CachePolicyFactory` (`vllm/v1/kv_offload/cpu/policies/factory.py`), which pre-registers the built-in `lru` and `arc` policies.
+
+### Out-of-tree (recommended)
+
+Implement `CachePolicy` (`vllm/v1/kv_offload/cpu/policies/base.py`) in your own package — no vLLM fork or patch required — and point `kv_connector_extra_config` at it directly:
+
+```json
+{
+  "cpu_bytes_to_use": 10737418240,
+  "eviction_policy": "MyCachePolicy",
+  "cache_policy_module_path": "my_package.my_module"
+}
+```
+
+`eviction_policy` is checked against the built-in registry first; if it isn't a registered name, vLLM imports `cache_policy_module_path` and looks up `eviction_policy` as a class name in that module — the same fallback `spec_module_path` provides for a custom `OffloadingSpec`. No import or registration call needs to run before the server starts.
+
+### Registering a friendly short name (in-process only)
+
+If you control the process that constructs the vLLM engine (e.g. an embedding application), you can register a short name once at startup instead of repeating the module path in every config:
+
+```python
+from vllm.v1.kv_offload.cpu.policies.factory import CachePolicyFactory
+
+CachePolicyFactory.register_cache_policy("my_policy", "my_package.my_module", "MyCachePolicy")
+```
+
+Then set `"eviction_policy": "my_policy"` in `kv_connector_extra_config`, the same as `"lru"`/`"arc"`. This only takes effect within the process that ran the `register_cache_policy` call — it does not help when the server is launched as a separate process (e.g. via the `vllm serve` CLI), where the out-of-tree `cache_policy_module_path` config above is the only option.
+
 ## Secondary Tiers
 
 Each entry in `secondary_tiers` is a dict with a required `type` field plus tier-specific fields.

docs/serving/offline_inference.md

@@ -65,6 +65,8 @@ For further details on Weight Transfer, please refer to [this page](../training/
 - `LLM.start_weight_update` - Starts a new weight update cycle.
 - `LLM.update_weights` - Updates the model weights.
 - `LLM.finish_weight_update` - Finishes the current weight update cycle.
+- `LLM.update_weight_version` - Sets the weight version without updating model weights.
+- `LLM.get_weight_version` - Returns the latest committed weight version.
 
 ## Additional APIs
 

docs/serving/online_serving/README.md

@@ -179,6 +179,8 @@ For further details on Weight Transfer, please refer to [this page](../../traini
 - `/start_weight_update` - Prepares the inference engine for a weight update.
 - `/update_weights` - Update model weights (can alter model behavior)
 - `/finish_weight_update` - Finalizes the weight update
+- `/update_weight_version` - Set the weight version without updating model weights
+- `/weight_info` - Get the latest committed weight version
 - `/get_world_size` - Get distributed world size
 
 ### Collective RPC

docs/training/async_rl.md

@@ -38,11 +38,12 @@ Resumes the scheduler after a pause. Any requests frozen with `mode="keep"` will
 
 ### HTTP Endpoints
 
-When using the vLLM HTTP server, the same functionality is available via:
+With `VLLM_SERVER_DEV_MODE=1`, the vLLM HTTP server exposes the same functionality via:
 
 - `POST /pause?mode=keep` - Pause generation
 - `POST /resume` - Resume generation
 - `POST /abort_requests` - Abort in-flight requests without pausing the scheduler (send `{}` to abort all, or `{"request_ids": [...]}`)
+- `GET /weight_info` - Return the latest committed `weight_version`
 
 !!! note "Data Parallelism"
     When using data parallelism with vLLM's **internal load balancer** (i.e. `data_parallel_backend="ray"`), pause and resume are handled automatically across all DP ranks -- a single call is sufficient. When using an **external load balancer** (i.e. multiple independent vLLM instances behind a proxy), you must send pause and resume requests to **every** engine instance individually before and after the weight update.

docs/training/weight_transfer/README.md

@@ -53,7 +53,9 @@ When running vLLM as an HTTP server, the following endpoints are available for w
 | `/init_weight_transfer_engine` | POST | Initialize the weight transfer engine with backend-specific info |
 | `/start_weight_update` | POST | Start a weight update |
 | `/update_weights` | POST | Transfer a batch of weights with backend-specific metadata |
-| `/finish_weight_update` | POST | Finish the weight update and run post-processing |
+| `/finish_weight_update` | POST | Finish the update and optionally commit its `weight_version` |
+| `/update_weight_version` | POST | Update `weight_version` without changing model weights |
+| `/weight_info` | GET | Get the latest committed weight version |
 | `/pause` | POST | Pause generation before weight sync to handle inflight requests |
 | `/resume` | POST | Resume generation after weight sync |
 | `/get_world_size` | GET | Get the number of inference workers (useful for NCCL world size calculation) |
@@ -79,7 +81,7 @@ EngineClass.trainer_send_weights(
 )
 
 # 4. Finish weight update on inference side
-llm.finish_weight_update()
+llm.finish_weight_update(weight_version="step-42")
 ```
 
 See the [NCCL](nccl.md) and [IPC](ipc.md) pages for backend-specific trainer APIs and full examples.

pyproject.toml

@@ -129,10 +129,9 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
     "tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py",
     "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html", 
     "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
-    "rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
-    "rust/src/parser/src/tool/gemma4.rs", "rust/src/parser/src/unified/gemma4.rs",
+    "rust/src/chat/src/renderer/deepseek_v32/fixtures/*", "rust/src/parser/**",
     "rust/src/text/src/output/decoded.rs",
-    "rust/src/tokenizer/src/incremental.rs", "rust/src/parser/src/reasoning/tests.rs"]
+    "rust/src/tokenizer/src/incremental.rs"]
 ignore-hidden = false
 
 [tool.typos.default]

requirements/test/cpu.txt

@@ -1,5 +1,7 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile requirements/test/cuda.in -o requirements/test/cpu.txt --index-strategy unsafe-best-match --torch-backend cpu --python-platform x86_64-manylinux_2_28 --python-version 3.12
+abi3info==2025.11.29
+    # via torch-abi-audit
 absl-py==2.1.0
     # via rouge-score
 accelerate==1.13.0
@@ -763,6 +765,8 @@ pycparser==2.22
     # via cffi
 pycryptodomex==3.22.0
     # via blobfile
+pycxxfilt==0.1.0
+    # via torch-abi-audit
 pydantic==2.12.0
     # via
     #   -r requirements/test/../common.txt
@@ -1127,6 +1131,8 @@ torch==2.13.0+cpu
     #   vector-quantize-pytorch
     #   vocos
     #   xgrammar
+torch-abi-audit==0.0.1
+    # via -r requirements/test/cuda.in
 torchaudio==2.11.0+cpu
     # via
     #   -r requirements/test/cuda.in

requirements/test/cuda.in

@@ -45,7 +45,7 @@ schemathesis>=4.0.0 # Required for openai schema test.
 # quantization
 bitsandbytes==0.49.2
 buildkite-test-collector==0.1.9
-
+torch-abi-audit # CI check for PyTorch stable ABI compliance
 
 genai_perf>=0.0.8
 tritonclient>=2.51.0

requirements/test/cuda.txt

@@ -1,5 +1,7 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile requirements/test/cuda.in -c requirements/cuda.txt -o requirements/test/cuda.txt --index-strategy unsafe-best-match --torch-backend cu130 --python-platform x86_64-manylinux_2_28 --python-version 3.12
+abi3info==2025.11.29
+    # via torch-abi-audit
 absl-py==2.1.0
     # via rouge-score
 accelerate==1.13.0
@@ -850,6 +852,8 @@ pycparser==2.22
     # via cffi
 pycryptodomex==3.22.0
     # via blobfile
+pycxxfilt==0.1.0
+    # via torch-abi-audit
 pydantic==2.12.0
     # via
     #   -c requirements/common.txt
@@ -1225,6 +1229,8 @@ torch==2.13.0+cu130
     #   vector-quantize-pytorch
     #   vocos
     #   xgrammar
+torch-abi-audit==0.0.1
+    # via -r requirements/test/cuda.in
 torchaudio==2.11.0+cu130
     # via
     #   -c requirements/cuda.txt

rust/Cargo.lock

@@ -5503,9 +5503,9 @@ dependencies = [
  "tokio",
  "tokio-stream",
  "tracing",
- "tracing-subscriber",
  "url",
  "uuid",
+ "vllm-tracing",
 ]
 
 [[package]]
@@ -5570,17 +5570,16 @@ dependencies = [
  "serde_json",
  "serde_with",
  "thiserror-ext",
- "time",
  "tokio",
  "tokio-util",
  "tracing",
- "tracing-subscriber",
  "uuid",
  "vllm-bench",
  "vllm-chat",
  "vllm-engine-core-client",
  "vllm-managed-engine",
  "vllm-server",
+ "vllm-tracing",
 ]
 
 [[package]]
@@ -5827,6 +5826,15 @@ dependencies = [
  "vllm-parser",
 ]
 
+[[package]]
+name = "vllm-tracing"
+version = "0.1.0"
+dependencies = [
+ "time",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.5.0"

rust/Cargo.toml

@@ -13,6 +13,7 @@ members = [
     "src/server",
     "src/text",
     "src/tokenizer",
+    "src/tracing",
 ]
 resolver = "3"
 
@@ -143,6 +144,7 @@ vllm-parser = { path = "src/parser" }
 vllm-server = { path = "src/server" }
 vllm-text = { path = "src/text" }
 vllm-tokenizer = { path = "src/tokenizer" }
+vllm-tracing = { path = "src/tracing" }
 winnow = { version = "1.0.2", features = ["simd"] }
 xgrammar-structural-tag = "0.2.0"
 zeromq = { version = "0.6.0", default-features = false, features = [

rust/src/bench/Cargo.toml

@@ -32,9 +32,9 @@ tokenizers.workspace = true
 tokio.workspace = true
 tokio-stream.workspace = true
 tracing.workspace = true
-tracing-subscriber.workspace = true
 url.workspace = true
 uuid.workspace = true
+vllm-tracing.workspace = true
 
 [lints]
 workspace = true

rust/src/bench/src/main.rs

@@ -19,18 +19,8 @@ struct Cli {
     args: vllm_bench::BenchServeArgs,
 }
 
-// TODO: unify the tracing subscriber used by different binaries.
-fn init_tracing() {
-    let filter = tracing_subscriber::EnvFilter::try_from_default_env()
-        .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info"));
-    let _ = tracing_subscriber::fmt()
-        .with_env_filter(filter)
-        .with_writer(std::io::stderr)
-        .try_init();
-}
-
 fn main() -> anyhow::Result<()> {
-    init_tracing();
+    vllm_tracing::init_tracing("Bench");
 
     let cli = Cli::parse();
     vllm_bench::prepare_process();

rust/src/chat/src/backend/hf.rs

@@ -73,7 +73,7 @@ impl HfChatBackend {
             RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
             RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
             RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
-            RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new()),
+            RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
         };
 
         info!(

rust/src/chat/src/renderer/inkling/tests.rs

@@ -31,6 +31,10 @@ impl Tokenizer for FixtureTokenizer {
         Ok(text.bytes().map(u32::from).collect())
     }
 
+    fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
+        self.encode(text, false)
+    }
+
     fn decode(
         &self,
         token_ids: &[u32],

rust/src/chat/src/renderer/kimi_k3/encoding.rs

@@ -6,9 +6,9 @@
 //! Port of Moonshot remote-code `encoding_k3.py::build_chat_segments()`.
 
 use std::collections::HashMap;
-use std::fmt::Write as _;
 
 use serde_json::{Map, Value, json};
+use vllm_tokenizer::Tokenizer;
 
 use crate::error::{Error, Result};
 use crate::request::{
@@ -25,12 +25,47 @@ pub(super) const IMAGE_PLACEHOLDER: &str = "<|media_pad|>";
 const DEFAULT_THINKING_EFFORT: &str = "max";
 const VALID_THINKING_EFFORTS: &[&str] = &["low", "high", "max"];
 
-/// Render one chat request into the K3 XTML prompt string.
-pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
+/// K3 prompt encoder preserving Python's per-segment tokenization boundaries.
+pub(super) struct K3TokenWriter<'a> {
+    tokenizer: &'a dyn Tokenizer,
+    token_ids: Vec<u32>,
+}
+
+impl<'a> K3TokenWriter<'a> {
+    pub(super) fn new(tokenizer: &'a dyn Tokenizer) -> Self {
+        Self {
+            tokenizer,
+            token_ids: Vec::new(),
+        }
+    }
+
+    /// Encode one trusted segment with normal added-token recognition.
+    pub(super) fn control(&mut self, text: &str) -> Result<()> {
+        if !text.is_empty() {
+            self.token_ids.extend(self.tokenizer.encode(text, false)?);
+        }
+        Ok(())
+    }
+
+    /// Encode one literal segment while bypassing every added-token matcher.
+    pub(super) fn ordinary(&mut self, text: &str) -> Result<()> {
+        if !text.is_empty() {
+            self.token_ids.extend(self.tokenizer.encode_ordinary(text)?);
+        }
+        Ok(())
+    }
+
+    pub(super) fn finish(self) -> Vec<u32> {
+        self.token_ids
+    }
+}
+
+/// Render and tokenize one chat request using K3's segment-aware contract.
+pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -> Result<Vec<u32>> {
     let thinking = thinking_enabled(request)?;
     let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
     let tools = request_tools(request);
-    let mut out = String::new();
+    let mut out = K3TokenWriter::new(tokenizer);
 
     if !tools.is_empty() {
         write_tool_declare(&mut out, tools, false)?;
@@ -48,14 +83,14 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
                  supported values include `low`, `medium`, `high`, and `max`.\n\
                  Now the system is invoked with `thinking_effort={effort}`."
             ),
-        );
+        )?;
     }
 
     // Track prior assistant tool-call ids for tool-result reordering / naming.
     let mut tool_call_id_index: HashMap<String, (usize, String)> = HashMap::new();
     let mut pending_tool_run: Vec<(usize, ChatMessage)> = Vec::new();
 
-    let flush_tool_run = |out: &mut String,
+    let flush_tool_run = |out: &mut K3TokenWriter<'_>,
                           run: &mut Vec<(usize, ChatMessage)>,
                           id_index: &HashMap<String, (usize, String)>|
      -> Result<()> {
@@ -163,7 +198,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
                 "tool-choice",
                 "The system is invoked with `tool_choice=required`.\n\
                  You MUST call tools in the next message.",
-            );
+            )?;
         }
         // Emit only when tools are present: Rust defaults tool_choice to None
         // for tool-free requests, which must not inject a tool-choice message.
@@ -173,7 +208,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
                 "tool-choice",
                 "The system is invoked with `tool_choice=none`.\n\
                  You MUST NOT call any tools in the next message.",
-            );
+            )?;
         }
         ChatToolChoice::None | ChatToolChoice::Auto | ChatToolChoice::Function { .. } => {}
     }
@@ -181,11 +216,11 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
     write_response_format(&mut out, request)?;
 
     if request.chat_options.add_generation_prompt() {
-        write_open_tag(&mut out, "message", &[("role", "assistant")]);
-        write_open_tag(&mut out, if thinking { "think" } else { "response" }, &[]);
+        write_open_tag(&mut out, "message", &[("role", "assistant")])?;
+        write_open_tag(&mut out, if thinking { "think" } else { "response" }, &[])?;
     }
 
-    Ok(out)
+    Ok(out.finish())
 }
 
 fn request_tools(request: &ChatRequest) -> &[ChatTool] {
@@ -248,7 +283,11 @@ fn content_is_empty(content: &ChatContent) -> bool {
     }
 }
 
-fn write_tool_declare(out: &mut String, tools: &[ChatTool], dynamic: bool) -> Result<()> {
+fn write_tool_declare(
+    out: &mut K3TokenWriter<'_>,
+    tools: &[ChatTool],
+    dynamic: bool,
+) -> Result<()> {
     let mut specs = Vec::with_capacity(tools.len());
     for tool in tools {
         let mut function = Map::new();
@@ -285,23 +324,26 @@ fn write_tool_declare(out: &mut String, tools: &[ChatTool], dynamic: bool) -> Re
         )
     };
 
-    write_internal_system(out, "tool-declare", &body);
-    Ok(())
+    write_internal_system(out, "tool-declare", &body)
 }
 
-fn write_internal_system(out: &mut String, message_type: &str, body: &str) {
+fn write_internal_system(
+    out: &mut K3TokenWriter<'_>,
+    message_type: &str,
+    body: &str,
+) -> Result<()> {
     write_open_tag(
         out,
         "message",
         &[("role", "system"), ("type", message_type)],
-    );
-    out.push_str(body.trim());
-    write_close_tag(out, "message");
-    out.push_str(END_OF_MSG);
+    )?;
+    out.ordinary(body.trim())?;
+    write_close_tag(out, "message")?;
+    out.control(END_OF_MSG)
 }
 
 fn write_role_message(
-    out: &mut String,
+    out: &mut K3TokenWriter<'_>,
     role: &str,
     name: Option<&str>,
     content: &ChatContent,
@@ -311,15 +353,14 @@ fn write_role_message(
         attrs.push(("name", name.to_string()));
     }
     let attr_refs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
-    write_open_tag(out, "message", &attr_refs);
+    write_open_tag(out, "message", &attr_refs)?;
     write_content(out, content)?;
-    write_close_tag(out, "message");
-    out.push_str(END_OF_MSG);
-    Ok(())
+    write_close_tag(out, "message")?;
+    out.control(END_OF_MSG)
 }
 
 fn write_tool_message(
-    out: &mut String,
+    out: &mut K3TokenWriter<'_>,
     tool_name: &str,
     index: usize,
     content: &ChatContent,
@@ -329,19 +370,18 @@ fn write_tool_message(
         out,
         "message",
         &[("role", "tool"), ("tool", tool_name), ("index", &index_str)],
-    );
+    )?;
     write_content(out, content)?;
-    write_close_tag(out, "message");
-    out.push_str(END_OF_MSG);
-    Ok(())
+    write_close_tag(out, "message")?;
+    out.control(END_OF_MSG)
 }
 
 fn write_assistant_message(
-    out: &mut String,
+    out: &mut K3TokenWriter<'_>,
     content: &[AssistantContentBlock],
     thinking: bool,
 ) -> Result<()> {
-    write_open_tag(out, "message", &[("role", "assistant")]);
+    write_open_tag(out, "message", &[("role", "assistant")])?;
 
     let mut reasoning = String::new();
     let mut response = String::new();
@@ -358,32 +398,31 @@ fn write_assistant_message(
     // message carries open/close tags even when there is no reasoning content.
     // In non-thinking mode the channel is dropped entirely.
     if thinking {
-        write_open_tag(out, "think", &[]);
+        write_open_tag(out, "think", &[])?;
         if !reasoning.trim().is_empty() {
-            out.push_str(&reasoning);
+            out.ordinary(&reasoning)?;
         }
-        write_close_tag(out, "think");
+        write_close_tag(out, "think")?;
     }
 
-    write_open_tag(out, "response", &[]);
-    out.push_str(&response);
-    write_close_tag(out, "response");
+    write_open_tag(out, "response", &[])?;
+    out.ordinary(&response)?;
+    write_close_tag(out, "response")?;
 
     if !tool_calls.is_empty() {
-        write_open_tag(out, "tools", &[]);
+        write_open_tag(out, "tools", &[])?;
         for (index, tool_call) in tool_calls.into_iter().enumerate() {
             write_assistant_tool_call(out, tool_call, index + 1)?;
         }
-        write_close_tag(out, "tools");
+        write_close_tag(out, "tools")?;
     }
 
-    write_close_tag(out, "message");
-    out.push_str(END_OF_MSG);
-    Ok(())
+    write_close_tag(out, "message")?;
+    out.control(END_OF_MSG)
 }
 
 fn write_assistant_tool_call(
-    out: &mut String,
+    out: &mut K3TokenWriter<'_>,
     tool_call: &AssistantToolCall,
     index: usize,
 ) -> Result<()> {
@@ -395,34 +434,33 @@ fn write_assistant_tool_call(
             ("tool", tool_call.name.as_str()),
             ("index", index_str.as_str()),
         ],
-    );
+    )?;
 
     let (args, json_block) = normalize_tool_arguments(&tool_call.arguments)?;
     if let Some(raw) = json_block {
-        write_open_tag(out, "json", &[("type", "object")]);
-        out.push_str(&raw);
-        write_close_tag(out, "json");
+        write_open_tag(out, "json", &[("type", "object")])?;
+        out.ordinary(&raw)?;
+        write_close_tag(out, "json")?;
     } else {
         for (key, value) in args {
             let typ = xtml_type(&value);
-            write_open_tag(out, "argument", &[("key", key.as_str()), ("type", typ)]);
-            out.push_str(&xtml_value(&value));
-            write_close_tag(out, "argument");
+            write_open_tag(out, "argument", &[("key", key.as_str()), ("type", typ)])?;
+            out.ordinary(&xtml_value(&value))?;
+            write_close_tag(out, "argument")?;
         }
     }
 
-    write_close_tag(out, "call");
-    Ok(())
+    write_close_tag(out, "call")
 }
 
-fn write_content(out: &mut String, content: &ChatContent) -> Result<()> {
+fn write_content(out: &mut K3TokenWriter<'_>, content: &ChatContent) -> Result<()> {
     match content {
         ChatContent::Text(text) => write_text_with_images(out, text),
         ChatContent::Parts(parts) => {
             for part in parts {
                 match part {
                     ChatContentPart::Text { text } => write_text_with_images(out, text)?,
-                    ChatContentPart::ImageUrl { .. } => out.push_str(IMAGE_PLACEHOLDER),
+                    ChatContentPart::ImageUrl { .. } => out.control(IMAGE_PLACEHOLDER)?,
                     ChatContentPart::VideoUrl { .. } => {
                         return Err(Error::UnsupportedMultimodalContent("video_url"));
                     }
@@ -439,15 +477,14 @@ fn write_content(out: &mut String, content: &ChatContent) -> Result<()> {
     }
 }
 
-fn write_text_with_images(out: &mut String, text: &str) -> Result<()> {
+fn write_text_with_images(out: &mut K3TokenWriter<'_>, text: &str) -> Result<()> {
     // Placeholder expansion is left as the literal K3 image token; multimodal
     // preprocessing can replace it once image prompts are known.
-    out.push_str(text);
-    Ok(())
+    out.ordinary(text)
 }
 
-fn write_response_format(out: &mut String, request: &ChatRequest) -> Result<()> {
-    let Some(rf) = request.chat_options.template_kwargs.get("response_format") else {
+fn write_response_format(out: &mut K3TokenWriter<'_>, request: &ChatRequest) -> Result<()> {
+    let Some(rf) = request.chat_options.response_format.as_ref() else {
         return Ok(());
     };
 
@@ -461,7 +498,7 @@ fn write_response_format(out: &mut String, request: &ChatRequest) -> Result<()>
                 "The system is invoked with `response_format=json_object`.\n\
                  Your response must be raw JSON data without markdown code \
                  blocks (```json) or any additional formatting.",
-            );
+            )?;
         }
         "json_schema" => {
             let schema = extract_response_schema(rf);
@@ -478,7 +515,7 @@ fn write_response_format(out: &mut String, request: &ChatRequest) -> Result<()>
                      {schema_json}\n\
                      ```"
                 ),
-            );
+            )?;
         }
         _ => {}
     }
@@ -528,19 +565,22 @@ fn xtml_value(value: &Value) -> String {
     }
 }
 
-fn write_open_tag(out: &mut String, tag: &str, attrs: &[(&str, &str)]) {
-    out.push_str(OPEN);
-    out.push_str(tag);
+fn write_open_tag(out: &mut K3TokenWriter<'_>, tag: &str, attrs: &[(&str, &str)]) -> Result<()> {
+    out.control(OPEN)?;
+    out.ordinary(tag)?;
     for (key, value) in attrs {
-        let _ = write!(out, " {key}=\"{}\"", escape_attr_value(value));
+        out.ordinary(&format!(" {key}"))?;
+        out.ordinary("=\"")?;
+        out.ordinary(&escape_attr_value(value))?;
+        out.ordinary("\"")?;
     }
-    out.push_str(SEP);
+    out.control(SEP)
 }
 
-fn write_close_tag(out: &mut String, tag: &str) {
-    out.push_str(CLOSE);
-    out.push_str(tag);
-    out.push_str(SEP);
+fn write_close_tag(out: &mut K3TokenWriter<'_>, tag: &str) -> Result<()> {
+    out.control(CLOSE)?;
+    out.ordinary(tag)?;
+    out.control(SEP)
 }
 
 fn escape_attr_value(value: &str) -> String {

rust/src/chat/src/renderer/kimi_k3/fixtures/controls_thinking_off_input.json

@@ -6,10 +6,10 @@
     }
   ],
   "add_generation_prompt": true,
+  "response_format": {
+    "type": "json_object"
+  },
   "template_kwargs": {
-    "thinking": false,
-    "response_format": {
-      "type": "json_object"
-    }
+    "thinking": false
   }
 }

rust/src/chat/src/renderer/kimi_k3/mod.rs

@@ -8,19 +8,22 @@ mod encoding;
 mod tests;
 
 use vllm_text::Prompt;
+use vllm_text::tokenizer::DynTokenizer;
 
 use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
 use crate::Result;
 use crate::request::ChatRequest;
 
 /// Dedicated Kimi K3 XTML renderer.
-#[derive(Debug, Clone, Copy, Default)]
-pub struct KimiK3ChatRenderer;
+#[derive(Clone)]
+pub struct KimiK3ChatRenderer {
+    tokenizer: DynTokenizer,
+}
 
 impl KimiK3ChatRenderer {
     /// Create a Kimi K3 renderer.
-    pub fn new() -> Self {
-        Self
+    pub fn new(tokenizer: DynTokenizer) -> Self {
+        Self { tokenizer }
     }
 }
 
@@ -29,7 +32,7 @@ impl ChatRenderer for KimiK3ChatRenderer {
         request.validate()?;
 
         Ok(RenderedPrompt {
-            prompt: Prompt::Text(encoding::render_request(request)?),
+            prompt: Prompt::TokenIds(encoding::render_request(request, self.tokenizer.as_ref())?),
             effective_template_kwargs: request_template_kwargs(request),
         })
     }

rust/src/chat/src/renderer/kimi_k3/tests.rs

@@ -4,23 +4,49 @@
 //! Golden fixtures generated from HF remote-code `encoding_k3.py`.
 
 use std::path::PathBuf;
+use std::sync::Arc;
 
 use expect_test::{expect, expect_file};
 use serde_json::json;
+use vllm_text::Prompt;
+use vllm_text::tokenizer::DynTokenizer;
+use vllm_tokenizer::Tokenizer;
+use vllm_tokenizer::test_utils::TestTokenizer;
 
 use super::KimiK3ChatRenderer;
 use crate::AssistantContentBlock;
 use crate::ChatRenderer;
+use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
-use crate::request::{ChatMessage, GenerationPromptMode, ReasoningEffort};
+use crate::request::{ChatContentPart, ChatMessage, GenerationPromptMode, ReasoningEffort};
+
+const OPEN_ID: u32 = 256;
+const CLOSE_ID: u32 = 257;
+const SEP_ID: u32 = 258;
+const END_OF_MSG_ID: u32 = 259;
+const MEDIA_ID: u32 = 260;
+
+fn test_tokenizer() -> TestTokenizer {
+    TestTokenizer::new()
+        .with_special_token(OPEN, OPEN_ID)
+        .with_special_token(CLOSE, CLOSE_ID)
+        .with_special_token(SEP, SEP_ID)
+        .with_special_token(END_OF_MSG, END_OF_MSG_ID)
+        .with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID)
+}
+
+fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec<u32> {
+    let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt;
+    let Prompt::TokenIds(token_ids) = prompt else {
+        panic!("kimi k3 renderer should return token IDs")
+    };
+    token_ids
+}
 
 fn render_request(request: &crate::request::ChatRequest) -> String {
-    KimiK3ChatRenderer::new()
-        .render(request)
-        .unwrap()
-        .prompt
-        .into_text()
-        .expect("kimi k3 renderer should return text prompt")
+    let tokenizer: DynTokenizer = Arc::new(test_tokenizer());
+    let token_ids = render_token_ids(request, tokenizer.clone());
+    tokenizer.decode(&token_ids, false).unwrap()
 }
 
 fn fixture_path(name: &str) -> PathBuf {
@@ -64,6 +90,44 @@ fn golden_dynamic_system_tool_declare() {
     assert_golden("dynamic_system_tool_declare");
 }
 
+#[test]
+fn token_writer_protects_literal_control_and_media_markers() {
+    let tokenizer = Arc::new(test_tokenizer());
+    let user_text = format!("literal {OPEN} and {}", super::encoding::IMAGE_PLACEHOLDER);
+    let mut request = crate::request::ChatRequest::for_test();
+    request.messages = vec![ChatMessage::user(vec![
+        ChatContentPart::text(user_text),
+        ChatContentPart::image_url("data:image/png;base64,test"),
+    ])];
+    request
+        .chat_options
+        .template_kwargs
+        .insert("thinking".to_string(), json!(false));
+    request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
+
+    let token_ids = render_token_ids(&request, tokenizer.clone());
+
+    assert_eq!(
+        token_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
+        1
+    );
+    assert_eq!(
+        token_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
+        1
+    );
+
+    let flattened = tokenizer.decode(&token_ids, false).unwrap();
+    let flattened_ids = tokenizer.encode(&flattened, false).unwrap();
+    assert_eq!(
+        flattened_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
+        2
+    );
+    assert_eq!(
+        flattened_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
+        2
+    );
+}
+
 #[test]
 fn thinking_history_renders_empty_think_channel() {
     let mut request = crate::request::ChatRequest::for_test();
@@ -190,7 +254,9 @@ fn rejects_removed_medium_thinking_effort() {
         .template_kwargs
         .insert("thinking_effort".to_string(), json!("medium"));
 
-    let error = KimiK3ChatRenderer::new().render(&request).unwrap_err();
+    let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()))
+        .render(&request)
+        .unwrap_err();
 
     expect![[r#"
         ChatTemplate(

rust/src/chat/src/renderer/test_utils.rs

@@ -48,7 +48,10 @@ pub(crate) struct FixtureRequest {
     messages: Vec<FixtureMessage>,
     add_generation_prompt: Option<bool>,
     reasoning_effort: Option<ReasoningEffort>,
-    /// Extra chat-template kwargs (thinking, preserve_thinking, response_format, …).
+    /// Standard response format passed to model-specific renderers.
+    #[serde(default)]
+    response_format: Option<Value>,
+    /// Extra chat-template kwargs (thinking, preserve_thinking, …).
     #[serde(default)]
     template_kwargs: HashMap<String, Value>,
     /// When omitted, defaults to `auto` if tools are present, otherwise `none`.
@@ -65,6 +68,7 @@ impl FixtureFile {
                 messages,
                 add_generation_prompt: None,
                 reasoning_effort: None,
+                response_format: None,
                 template_kwargs: HashMap::new(),
                 tool_choice: None,
             },
@@ -182,6 +186,7 @@ impl FixtureRequest {
             request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
         }
         request.chat_options.reasoning_effort = self.reasoning_effort;
+        request.chat_options.response_format = self.response_format;
         request.chat_options.template_kwargs.extend(self.template_kwargs);
 
         // Options supply a default thinking toggle only when the fixture did not.

rust/src/chat/src/request.rs

@@ -397,6 +397,10 @@ pub struct ChatOptions {
     /// Effort level exposed to chat templates for reasoning models.
     pub reasoning_effort: Option<ReasoningEffort>,
 
+    /// Standard response format available to model-specific renderers.
+    #[serde(default)]
+    pub response_format: Option<Value>,
+
     /// Additional keyword arguments exposed to the chat template.
     pub template_kwargs: HashMap<String, Value>,
 }
@@ -407,6 +411,7 @@ impl Default for ChatOptions {
             generation_prompt_mode: GenerationPromptMode::StartNewAssistant,
             chat_template: None,
             reasoning_effort: None,
+            response_format: None,
             template_kwargs: HashMap::new(),
         }
     }

rust/src/chat/tests/roundtrip.rs

@@ -272,7 +272,7 @@ impl RoundtripCase {
     fn gpt_oss() -> Self {
         Self {
             model_id: "openai/gpt-oss-20b",
-            assistant_stop_suffix: "", // not applicable for token-id cases
+            assistant_stop_suffix: "",
             tool_call_parser: ParserSelection::Auto,
             reasoning_parser: ParserSelection::Auto,
             thinking_behavior: ThinkingBehavior::Always { value: true },
@@ -285,7 +285,7 @@ impl RoundtripCase {
     fn inkling() -> Self {
         Self {
             model_id: "thinkingmachines/Inkling",
-            assistant_stop_suffix: "",
+            assistant_stop_suffix: "<|content_model_end_sampling|>",
             tool_call_parser: ParserSelection::Auto,
             reasoning_parser: ParserSelection::Auto,
             thinking_behavior: ThinkingBehavior::Always { value: true },
@@ -689,14 +689,23 @@ fn decoded_completion_stream(
                 .collect()
         }
         Prompt::TokenIds(token_ids) => {
-            ensure!(
-                assistant_stop_suffix.is_empty(),
-                "token-id roundtrip cases do not support text stop suffixes"
-            );
+            let body = if assistant_stop_suffix.is_empty() {
+                token_ids.as_slice()
+            } else {
+                let stop_token_ids = tokenizer
+                    .encode(assistant_stop_suffix, false)
+                    .context("failed to encode token-id completion stop suffix")?;
+                token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
+                    format!(
+                        "token-id completion did not end with {:?}: {:?}",
+                        assistant_stop_suffix, token_ids
+                    )
+                })?
+            };
             incremental_decode_chunks(
                 tokenizer,
                 &prompt_token_ids,
-                token_ids,
+                body,
                 TOKEN_COMPLETION_CHUNK_TOKENS,
             )?
         }

rust/src/cmd/Cargo.toml

@@ -23,17 +23,16 @@ serde.workspace = true
 serde_json.workspace = true
 serde_with.workspace = true
 thiserror-ext.workspace = true
-time.workspace = true
 tokio = { workspace = true, features = ["signal"] }
 tokio-util.workspace = true
 tracing.workspace = true
-tracing-subscriber.workspace = true
 uuid.workspace = true
 vllm-bench.workspace = true
 vllm-chat.workspace = true
 vllm-engine-core-client.workspace = true
 vllm-managed-engine.workspace = true
 vllm-server.workspace = true
+vllm-tracing.workspace = true
 
 [dev-dependencies]
 expect-test.workspace = true

rust/src/cmd/src/main.rs

@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 mod cli;
-mod logging;
 
 use std::env;
 use std::ffi::OsStr;
@@ -89,7 +88,7 @@ fn main() -> Result<()> {
             "serve" | "frontend" => "RustFrontend",
             _ => "Rust",
         };
-    logging::init_tracing(process_label);
+    vllm_tracing::init_tracing(process_label);
 
     let cli = Cli::parse();
 

rust/src/parser/benches/utils/adapter.rs

@@ -19,6 +19,10 @@ impl Tokenizer for BenchTokenizer {
         Ok(text.chars().map(|_| u32::MAX).collect())
     }
 
+    fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
+        self.encode(text, false)
+    }
+
     fn decode(
         &self,
         token_ids: &[u32],

rust/src/parser/src/unified/inkling.rs

@@ -414,6 +414,10 @@ mod tests {
             Ok(text.chars().map(u32::from).collect())
         }
 
+        fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
+            self.encode(text, false)
+        }
+
         fn decode(
             &self,
             token_ids: &[u32],
@@ -733,6 +737,10 @@ mod tests {
                 Ok(vec![])
             }
 
+            fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
+                self.encode(text, false)
+            }
+
             fn decode(
                 &self,
                 _token_ids: &[u32],

rust/src/server/src/error.rs

@@ -141,6 +141,22 @@ mod tests {
         assert!(response.error.message.contains("max_tokens=4"));
     }
 
+    #[test]
+    fn sampling_params_validation_maps_to_invalid_request() {
+        let api_error = text_submit_error(
+            "failed to submit completion request",
+            vllm_text::Error::SamplingParams(vllm_text::SamplingParamsError::OutOfRange {
+                parameter: "top_p",
+                value: 0.0,
+                expected: "(0, 1]",
+            }),
+        );
+        assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
+        let response = api_error.to_error_response();
+        assert_eq!(response.error.error_type, "invalid_request_error");
+        assert!(response.error.message.contains("top_p"));
+    }
+
     #[test]
     fn chat_wrapped_prompt_too_long_maps_to_invalid_request() {
         let error = vllm_chat::Error::Text(vllm_text::Error::PromptTooLong {

rust/src/server/src/grpc/tests.rs

@@ -648,6 +648,35 @@ async fn unary_generate_min_tokens_above_max_tokens_returns_invalid_argument() {
     server_task.abort();
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn unary_generate_invalid_sampling_params_returns_invalid_argument() {
+    let (mut client, server_task, _engine_task) = grpc_test_server(
+        b"engine-grpc-invalid-sampling",
+        default_stream_output_specs(),
+    )
+    .await;
+
+    let status = client
+        .generate(pb::GenerateRequest {
+            request_id: "test-invalid-sampling".to_string(),
+            model: "test-model".to_string(),
+            prompt: Some(pb::generate_request::Prompt::Text("hi".to_string())),
+            sampling: Some(pb::RandomSampling {
+                top_p: 2.0,
+                ..Default::default()
+            }),
+            ..Default::default()
+        })
+        .await
+        .expect_err("should fail when top_p is out of range");
+
+    assert_eq!(status.code(), tonic::Code::InvalidArgument);
+    assert!(status.message().contains("top_p"));
+
+    server_task.abort();
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 async fn streaming_generate_yields_incremental_responses() {

rust/src/server/src/routes/openai/chat_completions.rs

@@ -131,6 +131,7 @@ async fn collect_chat_completion(
         echo,
         return_token_ids,
         return_tokens_as_token_ids,
+        is_named_tool_choice,
     }: ResponseOptions,
 ) -> Result<ChatCompletionResponse, ApiError> {
     let collected = stream.collect_message().await.map_err(|error| {
@@ -157,7 +158,9 @@ async fn collect_chat_completion(
     // When reasoning is hidden, omit them rather than leaking hidden reasoning
     // tokens through per-token metadata.
     let include_output_metadata = include_reasoning || reasoning.is_none();
-    let finish_reason = chat_finish_reason_to_openai(&finish_reason, saw_tool_calls)?.to_string();
+    let finish_reason =
+        chat_finish_reason_to_openai(&finish_reason, saw_tool_calls && !is_named_tool_choice)?
+            .to_string();
     let tool_calls = message
         .tool_calls()
         .map(|call| ToolCall {
@@ -254,6 +257,7 @@ async fn chat_completion_chunk_stream(
         echo,
         return_token_ids,
         return_tokens_as_token_ids,
+        is_named_tool_choice,
     }: ResponseOptions,
     mut y: TryYielder<ChatCompletionStreamResponse, ApiError>,
 ) -> Result<(), ApiError> {
@@ -454,7 +458,7 @@ async fn chat_completion_chunk_stream(
                     &response_model,
                     created,
                     finish_reason,
-                    saw_tool_calls,
+                    saw_tool_calls && !is_named_tool_choice,
                 ) {
                     Ok(chunk) => yield_chunk!(chunk),
                     Err(error) => {
@@ -787,10 +791,10 @@ fn final_chunk(
     response_model: &str,
     created: u64,
     finish_reason: FinishReason,
-    saw_tool_calls: bool,
+    use_tool_calls_finish_reason: bool,
 ) -> Result<ChatCompletionStreamResponse, ApiError> {
     let stop_reason = finish_reason.as_stop_reason().map(stop_reason_to_json);
-    let finish_reason = chat_finish_reason_to_openai(&finish_reason, saw_tool_calls)?;
+    let finish_reason = chat_finish_reason_to_openai(&finish_reason, use_tool_calls_finish_reason)?;
 
     debug!(
         finish_reason = %finish_reason,
@@ -809,10 +813,10 @@ fn final_chunk(
 
 fn chat_finish_reason_to_openai(
     finish_reason: &FinishReason,
-    saw_tool_calls: bool,
+    use_tool_calls_finish_reason: bool,
 ) -> Result<&'static str, ApiError> {
     match finish_reason {
-        FinishReason::Stop(_) if saw_tool_calls => Ok("tool_calls"),
+        FinishReason::Stop(_) if use_tool_calls_finish_reason => Ok("tool_calls"),
         FinishReason::Stop(_) => Ok("stop"),
         FinishReason::Length => Ok("length"),
         FinishReason::Abort => Ok("abort"),

rust/src/server/src/routes/openai/chat_completions/convert.rs

@@ -54,6 +54,8 @@ pub(super) struct ResponseOptions {
     pub return_token_ids: bool,
     /// Whether to format logprob tokens as `token_id:{id}`.
     pub return_tokens_as_token_ids: bool,
+    /// Whether the request forces one named function tool.
+    pub is_named_tool_choice: bool,
 }
 
 /// Validate and lower one OpenAI chat completion request into the internal chat
@@ -87,6 +89,15 @@ pub(super) fn prepare_chat_request(
     )?;
 
     let template_kwargs = request.chat_template_kwargs.unwrap_or_default();
+    let response_format =
+        request.response_format.as_ref().map(serde_json::to_value).transpose().map_err(
+            |error| {
+                ApiError::invalid_request(
+                    format!("failed to serialize response_format: {error}"),
+                    Some("response_format"),
+                )
+            },
+        )?;
 
     let include_usage = (request.stream_options.as_ref())
         .and_then(|options| options.include_usage)
@@ -98,6 +109,7 @@ pub(super) fn prepare_chat_request(
             .and_then(|options| options.continuous_usage_stats)
             .unwrap_or(false);
     let requested_logprobs = request.logprobs;
+    let is_named_tool_choice = matches!(&request.tool_choice, Some(ToolChoice::Function { .. }));
 
     // Auto-enable prompt logprobs for non-streaming echo, matching Python vLLM's
     // behavior.
@@ -147,6 +159,7 @@ pub(super) fn prepare_chat_request(
             generation_prompt_mode,
             chat_template: request.chat_template,
             reasoning_effort: request.reasoning_effort,
+            response_format,
             template_kwargs,
         },
         tools: convert_tools(request.tools)?,
@@ -180,6 +193,7 @@ pub(super) fn prepare_chat_request(
             echo,
             return_token_ids: request.return_token_ids.unwrap_or(false),
             return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
+            is_named_tool_choice,
         },
         chat_request,
     })
@@ -396,6 +410,7 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
+    use std::sync::Arc;
 
     use axum::http::HeaderMap;
     use expect_test::expect;
@@ -403,16 +418,18 @@ mod tests {
     use serde_json::json;
     use vllm_chat::{
         AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
-        ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
-        SamplingParams as VllmSamplingParams,
+        ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
+        KimiK3ChatRenderer, SamplingParams as VllmSamplingParams,
     };
-    use vllm_text::output::TextDecodeOptions;
+    use vllm_text::{Prompt, output::TextDecodeOptions};
+    use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
 
     use super::prepare_chat_request;
     use crate::lora::LoraModelResolution;
     use crate::routes::openai::chat_completions::types::{
         AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
     };
+    use crate::routes::openai::utils::structured_outputs::{JsonSchemaFormat, ResponseFormat};
     use crate::routes::openai::utils::types::{
         AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
         MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
@@ -469,6 +486,60 @@ mod tests {
         assert!(prepared.chat_request.parallel_tool_calls);
     }
 
+    #[test]
+    fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() {
+        let mut request = base_request();
+        request.model = "moonshotai/Kimi-K3".to_string();
+        let response_format = ResponseFormat::JsonSchema {
+            json_schema: JsonSchemaFormat {
+                name: "answer".to_string(),
+                description: None,
+                schema: json!({
+                    "type": "object",
+                    "properties": {
+                        "answer": {"type": "string"}
+                    },
+                    "required": ["answer"]
+                }),
+                strict: Some(true),
+            },
+        };
+        request.response_format = Some(response_format.clone());
+        let template_response_format = json!({"type": "json_object"});
+        request.chat_template_kwargs = Some(HashMap::from([(
+            "response_format".to_string(),
+            template_response_format.clone(),
+        )]));
+
+        let prepared = prepare_chat_request(
+            request,
+            &served(&["moonshotai/Kimi-K3"]),
+            ResolvedRequestContext::default(),
+        )
+        .expect("request is valid");
+
+        assert_eq!(
+            prepared.chat_request.chat_options.response_format.as_ref(),
+            Some(&serde_json::to_value(response_format).unwrap())
+        );
+        assert_eq!(
+            prepared.chat_request.chat_options.template_kwargs.get("response_format"),
+            Some(&template_response_format)
+        );
+
+        let tokenizer = Arc::new(TestTokenizer::new());
+        let prompt = KimiK3ChatRenderer::new(tokenizer.clone())
+            .render(&prepared.chat_request)
+            .expect("Kimi K3 rendering succeeds")
+            .prompt;
+        let Prompt::TokenIds(prompt_token_ids) = prompt else {
+            panic!("Kimi K3 renders token IDs");
+        };
+        let prompt = tokenizer.decode(&prompt_token_ids, false).expect("Kimi K3 prompt decodes");
+        assert!(prompt.contains("response_format=json_schema"));
+        assert!(prompt.contains(r#""answer":{"type":"string"}"#));
+    }
+
     #[test]
     fn prepare_chat_request_maps_text_parts() {
         let mut request = base_request();
@@ -1068,6 +1139,7 @@ mod tests {
         .expect("request is valid");
 
         assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
+        assert!(!prepared.options.is_named_tool_choice);
     }
 
     #[test]
@@ -1107,6 +1179,7 @@ mod tests {
                 name: "get_weather".to_string(),
             }
         );
+        assert!(prepared.options.is_named_tool_choice);
     }
 
     #[test]

rust/src/server/src/routes/tests.rs

@@ -153,6 +153,20 @@ fn default_stream_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason
     ]
 }
 
+fn weather_tool_call_output_specs() -> Vec<(Vec<u32>, Option<EngineCoreFinishReason>)> {
+    vec![
+        (bytes_to_token_ids(b"<think>Need tool.</think>"), None),
+        (
+            bytes_to_token_ids(b"<tool_call>\n{\"name\":\"get_weather\", "),
+            None,
+        ),
+        (
+            bytes_to_token_ids(b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>"),
+            Some(EngineCoreFinishReason::Stop),
+        ),
+    ]
+}
+
 fn assert_adapter_a_lora_request(request: &EngineCoreRequest) {
     let lora = request.lora_request.as_ref().expect("lora request");
     assert_eq!(lora.lora_name, "adapter-a");
@@ -4554,17 +4568,7 @@ async fn include_reasoning_false_suppresses_non_stream_output_metadata() {
 async fn tool_calls_are_mapped_to_tool_call_sse_chunks() {
     let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
         Arc::new(FakeChatBackend::with_model_id("Qwen/Qwen3-0.6B")),
-        vec![
-            (bytes_to_token_ids(b"<think>Need tool.</think>"), None),
-            (
-                bytes_to_token_ids(b"<tool_call>\n{\"name\":\"get_weather\", "),
-                None,
-            ),
-            (
-                bytes_to_token_ids(b"\"arguments\":{\"city\":\"Paris\"}}\n</tool_call>"),
-                Some(EngineCoreFinishReason::Stop),
-            ),
-        ],
+        weather_tool_call_output_specs(),
     )
     .await;
 
@@ -4613,6 +4617,63 @@ async fn tool_calls_are_mapped_to_tool_call_sse_chunks() {
     assert!(text.contains("\"finish_reason\":\"tool_calls\""), "{text}");
 }
 
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn named_tool_choice_uses_stop_finish_reason() {
+    for stream in [false, true] {
+        let (app, engine_task) = test_app_with_backend_and_stream_output_specs(
+            Arc::new(FakeChatBackend::with_model_id("Qwen/Qwen3-0.6B")),
+            weather_tool_call_output_specs(),
+        )
+        .await;
+
+        let response = app
+            .clone()
+            .call(
+                Request::builder()
+                    .method("POST")
+                    .uri("/v1/chat/completions")
+                    .header("content-type", "application/json")
+                    .body(Body::from(
+                        json!({
+                            "model": "Qwen/Qwen1.5-0.5B-Chat",
+                            "stream": stream,
+                            "messages": [{"role": "user", "content": "hello"}],
+                            "tools": [{
+                                "type": "function",
+                                "function": {
+                                    "name": "get_weather",
+                                    "description": "Get weather",
+                                    "parameters": {
+                                        "type": "object",
+                                        "properties": {"city": {"type": "string"}}
+                                    }
+                                }
+                            }],
+                            "tool_choice": {
+                                "type": "function",
+                                "function": {"name": "get_weather"}
+                            }
+                        })
+                        .to_string(),
+                    ))
+                    .expect("build request"),
+            )
+            .await
+            .expect("call app");
+
+        assert_eq!(response.status(), StatusCode::OK);
+        let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
+        engine_task.await.expect("mock engine task");
+        let text = String::from_utf8(body.to_vec()).expect("utf8 body");
+
+        assert!(text.contains("\"tool_calls\":"), "{text}");
+        assert!(text.contains("\"name\":\"get_weather\""), "{text}");
+        assert!(text.contains("\"finish_reason\":\"stop\""), "{text}");
+        assert!(!text.contains("\"finish_reason\":\"tool_calls\""), "{text}");
+    }
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 async fn tool_call_sse_chunks_can_carry_logprobs() {

rust/src/server/src/routes/tokenize/types.rs

@@ -82,6 +82,7 @@ impl TokenizeChatRequest {
                 generation_prompt_mode,
                 chat_template: self.chat_template,
                 reasoning_effort: None,
+                response_format: None,
                 template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
             },
             tools: convert_tools(self.tools)?,

rust/src/text/src/backend/hf/mod.rs

@@ -177,6 +177,10 @@ mod tests {
             Ok(vec![])
         }
 
+        fn encode_ordinary(&self, text: &str) -> vllm_tokenizer::Result<Vec<u32>> {
+            self.encode(text, false)
+        }
+
         fn decode(
             &self,
             _token_ids: &[u32],

rust/src/text/src/error.rs

@@ -6,6 +6,7 @@ use vllm_engine_core_client::Error as EngineCoreError;
 use vllm_llm::Error as LlmError;
 
 pub use crate::lower::logprobs::LogprobsError;
+pub use crate::lower::sampling::SamplingParamsError;
 pub use crate::lower::token_ids::TokenIdsError;
 
 #[derive(Debug, Error)]
@@ -23,6 +24,8 @@ pub enum Error {
     Logprobs(#[from] LogprobsError),
     #[error(transparent)]
     TokenIds(#[from] TokenIdsError),
+    #[error(transparent)]
+    SamplingParams(#[from] SamplingParamsError),
     #[error(
         "`min_tokens` must be less than or equal to `max_tokens`, \
          got min_tokens={min_tokens}, max_tokens={max_tokens}"
@@ -50,6 +53,7 @@ impl Error {
             | Self::EmptyPromptTokenIds { .. }
             | Self::Logprobs(_)
             | Self::TokenIds(_)
+            | Self::SamplingParams(_)
             | Self::MinTokensExceedsMaxTokens { .. }
             | Self::InvalidThinkingTokenBudget
             | Self::InvalidRepetitionDetection { .. }

rust/src/text/src/lib.rs

@@ -10,7 +10,7 @@
 use std::mem::take;
 
 pub use backend::{DynTextBackend, SamplingHints, SamplingLimits, TextBackend};
-pub use error::{Error, LogprobsError, Result, TokenIdsError};
+pub use error::{Error, LogprobsError, Result, SamplingParamsError, TokenIdsError};
 use futures::Stream;
 pub use lower::{
     PreparedTextRequest, lower_sampling_params, lower_text_request, resolve_max_tokens,

rust/src/text/src/lower.rs

@@ -4,9 +4,11 @@
 use std::collections::BTreeSet;
 
 pub(crate) mod logprobs;
+pub(crate) mod sampling;
 pub(crate) mod token_ids;
 
 use logprobs::validate_logprobs;
+use sampling::validate_resolved_sampling_params;
 use token_ids::{validate_prompt_token_ids, validate_vocab_range};
 use vllm_engine_core_client::protocol::sampling::{
     EngineCoreSamplingParams, RepetitionDetectionParams,
@@ -186,6 +188,7 @@ pub fn lower_sampling_params(
         skip_reading_prefix_cache,
         extra_args: vllm_xargs,
     };
+    validate_resolved_sampling_params(&params)?;
     validate_vocab_range(&params, &sampling_limits)?;
     Ok(params)
 }
@@ -319,7 +322,7 @@ mod tests {
     use super::*;
     use crate::backend::hf::HfTextBackend;
     use crate::backend::{SamplingHints, TextBackend as _};
-    use crate::error::{LogprobsError, TokenIdsError};
+    use crate::error::{LogprobsError, SamplingParamsError, TokenIdsError};
     use crate::request::{Prompt, TextRequest};
 
     fn stub_tokenizer() -> TestTokenizer {
@@ -482,6 +485,120 @@ mod tests {
         assert!(message.contains("min_count=1"));
     }
 
+    #[test]
+    fn lower_sampling_params_rejects_invalid_sampling_ranges() {
+        let cases = [
+            (
+                "temperature",
+                SamplingParams {
+                    temperature: Some(5.0),
+                    ..SamplingParams::default()
+                },
+            ),
+            (
+                "top_p",
+                SamplingParams {
+                    top_p: Some(0.0),
+                    ..SamplingParams::default()
+                },
+            ),
+            (
+                "min_p",
+                SamplingParams {
+                    min_p: Some(2.0),
+                    ..SamplingParams::default()
+                },
+            ),
+            (
+                "repetition_penalty",
+                SamplingParams {
+                    repetition_penalty: Some(0.0),
+                    ..SamplingParams::default()
+                },
+            ),
+            (
+                "frequency_penalty",
+                SamplingParams {
+                    frequency_penalty: Some(100.0),
+                    ..SamplingParams::default()
+                },
+            ),
+            (
+                "presence_penalty",
+                SamplingParams {
+                    presence_penalty: Some(100.0),
+                    ..SamplingParams::default()
+                },
+            ),
+        ];
+
+        for (expected_parameter, sampling_params) in cases {
+            let error =
+                lower_sampling_params_with_limits(sampling_params, sample_sampling_limits())
+                    .unwrap_err();
+
+            assert!(
+                matches!(
+                    error,
+                    Error::SamplingParams(SamplingParamsError::OutOfRange {
+                        parameter,
+                        ..
+                    }) if parameter == expected_parameter
+                ),
+                "{expected_parameter} should be rejected"
+            );
+        }
+    }
+
+    #[test]
+    fn lower_sampling_params_rejects_non_finite_sampling_values() {
+        for (expected_parameter, sampling_params) in [
+            (
+                "temperature",
+                SamplingParams {
+                    temperature: Some(f32::INFINITY),
+                    ..SamplingParams::default()
+                },
+            ),
+            (
+                "repetition_penalty",
+                SamplingParams {
+                    repetition_penalty: Some(f32::NAN),
+                    ..SamplingParams::default()
+                },
+            ),
+        ] {
+            let error =
+                lower_sampling_params_with_limits(sampling_params, sample_sampling_limits())
+                    .unwrap_err();
+
+            assert!(
+                matches!(
+                    error,
+                    Error::SamplingParams(SamplingParamsError::NotFinite {
+                        parameter,
+                        ..
+                    }) if parameter == expected_parameter
+                ),
+                "{expected_parameter} should reject non-finite values"
+            );
+        }
+    }
+
+    #[test]
+    fn lower_sampling_params_accepts_python_compatible_repetition_penalty_above_two() {
+        let params = lower_sampling_params_with_limits(
+            SamplingParams {
+                repetition_penalty: Some(2.5),
+                ..SamplingParams::default()
+            },
+            sample_sampling_limits(),
+        )
+        .unwrap();
+
+        assert_eq!(params.repetition_penalty, 2.5);
+    }
+
     #[test]
     fn lower_text_request_applies_python_style_eos_hints() {
         let prepared = lower_text_request(

rust/src/text/src/lower/sampling.rs

@@ -0,0 +1,92 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+use thiserror::Error;
+use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
+
+#[derive(Debug, Error, PartialEq)]
+pub enum SamplingParamsError {
+    #[error("{parameter} must be a finite number, got {value}")]
+    NotFinite { parameter: &'static str, value: f32 },
+    #[error("{parameter} must be in {expected}, got {value}")]
+    OutOfRange {
+        parameter: &'static str,
+        value: f32,
+        expected: &'static str,
+    },
+}
+
+fn validate_frequency_penalty(value: f32) -> Result<(), SamplingParamsError> {
+    validate_closed_range("frequency_penalty", value, -2.0, 2.0, "[-2, 2]")
+}
+
+fn validate_presence_penalty(value: f32) -> Result<(), SamplingParamsError> {
+    validate_closed_range("presence_penalty", value, -2.0, 2.0, "[-2, 2]")
+}
+
+fn validate_temperature(value: f32) -> Result<(), SamplingParamsError> {
+    validate_finite("temperature", value)?;
+    validate_closed_range("temperature", value, 0.0, 2.0, "[0, 2]")
+}
+
+fn validate_top_p(value: f32) -> Result<(), SamplingParamsError> {
+    if value > 0.0 && value <= 1.0 {
+        return Ok(());
+    }
+    Err(SamplingParamsError::OutOfRange {
+        parameter: "top_p",
+        value,
+        expected: "(0, 1]",
+    })
+}
+
+fn validate_min_p(value: f32) -> Result<(), SamplingParamsError> {
+    validate_closed_range("min_p", value, 0.0, 1.0, "[0, 1]")
+}
+
+fn validate_repetition_penalty(value: f32) -> Result<(), SamplingParamsError> {
+    validate_finite("repetition_penalty", value)?;
+    if value > 0.0 {
+        return Ok(());
+    }
+    Err(SamplingParamsError::OutOfRange {
+        parameter: "repetition_penalty",
+        value,
+        expected: "(0, inf)",
+    })
+}
+
+pub(crate) fn validate_resolved_sampling_params(
+    params: &EngineCoreSamplingParams,
+) -> Result<(), SamplingParamsError> {
+    validate_temperature(params.temperature)?;
+    validate_top_p(params.top_p)?;
+    validate_min_p(params.min_p)?;
+    validate_frequency_penalty(params.frequency_penalty)?;
+    validate_presence_penalty(params.presence_penalty)?;
+    validate_repetition_penalty(params.repetition_penalty)
+}
+
+fn validate_finite(parameter: &'static str, value: f32) -> Result<(), SamplingParamsError> {
+    if value.is_finite() {
+        return Ok(());
+    }
+    Err(SamplingParamsError::NotFinite { parameter, value })
+}
+
+fn validate_closed_range(
+    parameter: &'static str,
+    value: f32,
+    min: f32,
+    max: f32,
+    expected: &'static str,
+) -> Result<(), SamplingParamsError> {
+    if value >= min && value <= max {
+        return Ok(());
+    }
+    Err(SamplingParamsError::OutOfRange {
+        parameter,
+        value,
+        expected,
+    })
+}

rust/src/tokenizer/src/hf.rs

@@ -1,13 +1,20 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+use std::borrow::Cow;
 use std::path::Path;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use fastokens::Tokenizer as FastokensTokenizer;
 use fastokens::decoders::Decoder as FastokensDecoder;
+use fastokens::pre_tokenized::{
+    PreTokenizedString as FastokensPreTokenizedString, Split as FastokensSplit,
+};
+use fastokens::{PreTokenizer as FastokensPreTokenizer, Split as FastokensSplitPreTokenizer};
 use thiserror_ext::AsReport as _;
-use tokenizers::Tokenizer as HfTokenizer;
+use tokenizers::{
+    AddedVocabulary, Model as _, OffsetType, PreTokenizer as _, Tokenizer as HfTokenizer,
+};
 use tracing::{info, warn};
 
 use crate::byte_level_decode::decode_byte_level;
@@ -16,6 +23,8 @@ use crate::{Result, Tokenizer};
 
 mod added_tokens;
 
+static EMPTY_HF_ADDED_VOCABULARY: LazyLock<AddedVocabulary> = LazyLock::new(AddedVocabulary::new);
+
 enum Backend {
     Hf(Box<HfTokenizer>),
     Fastokens(Box<FastokensTokenizer>),
@@ -53,6 +62,85 @@ fn decode_fastokens_byte_level(
     Ok(decode_byte_level(tokens))
 }
 
+fn encode_hf_ordinary(tokenizer: &HfTokenizer, text: &str) -> tokenizers::Result<Vec<u32>> {
+    let mut pretokenized =
+        EMPTY_HF_ADDED_VOCABULARY.extract_and_normalize(tokenizer.get_normalizer(), text);
+
+    if let Some(pre_tokenizer) = tokenizer.get_pre_tokenizer() {
+        pre_tokenizer.pre_tokenize(&mut pretokenized)?;
+    }
+    pretokenized.tokenize(|normalized| tokenizer.get_model().tokenize(normalized.get()))?;
+    let encoding = pretokenized.into_encoding(None, 0, OffsetType::Byte)?;
+    let encoding = tokenizer.post_process(encoding, None, false)?;
+    Ok(encoding.get_ids().to_vec())
+}
+
+fn fastokens_fused_split(tokenizer: &FastokensTokenizer) -> Option<&FastokensSplitPreTokenizer> {
+    // Keep this predicate aligned with fastokens::Tokenizer::detect_fused_byte_level.
+    let FastokensPreTokenizer::Sequence(steps) = tokenizer.pre_tokenizer()? else {
+        return None;
+    };
+    let [
+        FastokensPreTokenizer::Split(split),
+        FastokensPreTokenizer::ByteLevel(byte_level),
+    ] = steps.as_slice()
+    else {
+        return None;
+    };
+    byte_level.is_bulk_only().then_some(split)
+}
+
+fn fastokens_pre_tokenized_ordinary(
+    tokenizer: &FastokensTokenizer,
+    text: &str,
+) -> FastokensPreTokenizedString {
+    // This is fastokens::Tokenizer::build_pre_tokenized with added_tokens = None.
+    let normalized = tokenizer
+        .normalizer()
+        .map_or(Cow::Borrowed(text), |normalizer| normalizer.normalize(text));
+    match normalized {
+        Cow::Borrowed(_) => FastokensPreTokenizedString::from_text(text),
+        Cow::Owned(text) => {
+            let len = text.len();
+            FastokensPreTokenizedString::new(
+                text,
+                vec![FastokensSplit {
+                    range: 0..len,
+                    token_id: None,
+                }],
+            )
+        }
+    }
+}
+
+fn encode_fastokens_ordinary(
+    tokenizer: &FastokensTokenizer,
+    text: &str,
+) -> std::result::Result<Vec<u32>, fastokens::Error> {
+    if text.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let mut pretokenized = fastokens_pre_tokenized_ordinary(tokenizer, text);
+    let ids = if let Some(split) = fastokens_fused_split(tokenizer) {
+        split.pre_tokenize(&mut pretokenized)?;
+        pretokenized
+            .tokenize_batched(|buffer, splits, output| {
+                tokenizer.model().tokenize_batch_fused(buffer, splits, output)
+            })
+            .map_err(fastokens::Error::Model)?
+    } else {
+        if let Some(pre_tokenizer) = tokenizer.pre_tokenizer() {
+            pre_tokenizer.pre_tokenize(&mut pretokenized)?;
+        }
+        pretokenized
+            .tokenize(|text, output| tokenizer.model().tokenize_into(text, output))
+            .map_err(fastokens::Error::Model)?
+    };
+
+    Ok(tokenizer.post_process(ids, false))
+}
+
 /// Tokenizer from `tokenizer.json` in HuggingFace format.
 ///
 /// This tries to load with `fastokens` first for better performance, then falls
@@ -156,6 +244,17 @@ impl Tokenizer for HuggingFaceTokenizer {
         }
     }
 
+    fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
+        match &self.backend {
+            Backend::Hf(tokenizer) => encode_hf_ordinary(tokenizer, text)
+                .map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report())),
+            Backend::Fastokens(tokenizer) | Backend::FastokensByteLevel(tokenizer) => {
+                encode_fastokens_ordinary(tokenizer, text)
+                    .map_err(|error| tokenizer_error!("encoding failed: {}", error.as_report()))
+            }
+        }
+    }
+
     fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
         match &self.backend {
             Backend::Hf(t) => t
@@ -200,12 +299,19 @@ impl Tokenizer for HuggingFaceTokenizer {
 
 #[cfg(test)]
 mod tests {
+    use std::path::{Path, PathBuf};
+
+    use serde_json::{Value, json};
     use tempfile::tempdir;
     use tokenizers::models::bpe::BPE;
+    use tokenizers::pre_tokenizers::byte_level::ByteLevel;
     use tokenizers::{AddedToken, Tokenizer as HfTokenizer};
 
     use super::{HuggingFaceTokenizer, Tokenizer};
 
+    const REGULAR_TOKEN: &str = "<|regular|>";
+    const SPECIAL_TOKEN: &str = "<|special|>";
+
     fn tiny_bpe_tokenizer() -> HfTokenizer {
         let vocab = [
             ("<unk>".to_string(), 0),
@@ -232,6 +338,186 @@ mod tests {
         HfTokenizer::new(model)
     }
 
+    fn ordinary_test_tokenizer_json(fused: bool, with_added_tokens: bool) -> Value {
+        let mut alphabet: Vec<char> = ByteLevel::alphabet().into_iter().collect();
+        alphabet.sort_unstable();
+        let vocab = alphabet
+            .into_iter()
+            .enumerate()
+            .map(|(id, token)| (token.to_string(), json!(id)))
+            .collect::<serde_json::Map<_, _>>();
+
+        let pre_tokenizer = if fused {
+            json!({
+                "type": "Sequence",
+                "pretokenizers": [
+                    {
+                        "type": "Split",
+                        "pattern": {"Regex": "\\S+|\\s+"},
+                        "behavior": "Isolated",
+                        "invert": false
+                    },
+                    {
+                        "type": "ByteLevel",
+                        "add_prefix_space": false,
+                        "trim_offsets": true,
+                        "use_regex": false
+                    }
+                ]
+            })
+        } else {
+            json!({
+                "type": "ByteLevel",
+                "add_prefix_space": false,
+                "trim_offsets": true,
+                "use_regex": true
+            })
+        };
+        let added_tokens = with_added_tokens.then(|| {
+            json!([
+                {
+                    "id": 256,
+                    "content": REGULAR_TOKEN,
+                    "single_word": false,
+                    "lstrip": false,
+                    "rstrip": false,
+                    "normalized": true,
+                    "special": false
+                },
+                {
+                    "id": 257,
+                    "content": SPECIAL_TOKEN,
+                    "single_word": false,
+                    "lstrip": false,
+                    "rstrip": false,
+                    "normalized": false,
+                    "special": true
+                }
+            ])
+        });
+
+        json!({
+            "version": "1.0",
+            "truncation": {
+                "direction": "Right",
+                "max_length": 24,
+                "strategy": "LongestFirst",
+                "stride": 0
+            },
+            "padding": null,
+            "added_tokens": added_tokens.unwrap_or_else(|| json!([])),
+            "normalizer": {"type": "NFC"},
+            "pre_tokenizer": pre_tokenizer,
+            "post_processor": {
+                "type": "ByteLevel",
+                "add_prefix_space": false,
+                "trim_offsets": true,
+                "use_regex": true
+            },
+            "decoder": {
+                "type": "ByteLevel",
+                "add_prefix_space": false,
+                "trim_offsets": true,
+                "use_regex": true
+            },
+            "model": {
+                "type": "BPE",
+                "dropout": null,
+                "unk_token": null,
+                "continuing_subword_prefix": null,
+                "end_of_word_suffix": null,
+                "fuse_unk": false,
+                "byte_fallback": false,
+                "ignore_merges": false,
+                "vocab": vocab,
+                "merges": []
+            }
+        })
+    }
+
+    fn write_tokenizer_json(dir: &Path, name: &str, value: &Value) -> PathBuf {
+        let path = dir.join(name);
+        std::fs::write(
+            &path,
+            serde_json::to_vec(value).expect("serialize tokenizer"),
+        )
+        .expect("write tokenizer");
+        path
+    }
+
+    fn assert_ordinary_matches_added_empty(
+        constructor: fn(&Path) -> crate::Result<HuggingFaceTokenizer>,
+        fused: bool,
+    ) {
+        let dir = tempdir().expect("create temp dir");
+        let added_path = write_tokenizer_json(
+            dir.path(),
+            "with-added.json",
+            &ordinary_test_tokenizer_json(fused, true),
+        );
+        let empty_path = write_tokenizer_json(
+            dir.path(),
+            "added-empty.json",
+            &ordinary_test_tokenizer_json(fused, false),
+        );
+        let tokenizer = constructor(&added_path).expect("load tokenizer with added tokens");
+        let added_empty = constructor(&empty_path).expect("load tokenizer with empty added tokens");
+
+        if let super::Backend::Fastokens(inner) | super::Backend::FastokensByteLevel(inner) =
+            &tokenizer.backend
+        {
+            assert_eq!(super::fastokens_fused_split(inner).is_some(), fused);
+        }
+
+        assert_eq!(
+            tokenizer.encode(REGULAR_TOKEN, false).unwrap(),
+            vec![tokenizer.token_to_id(REGULAR_TOKEN).unwrap()]
+        );
+        assert_eq!(
+            tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
+            vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
+        );
+
+        for text in [
+            "",
+            "hello",
+            "Cafe\u{301}",
+            REGULAR_TOKEN,
+            SPECIAL_TOKEN,
+            "hello <|regular|> Cafe\u{301} <|special|> tail",
+        ] {
+            assert_eq!(
+                tokenizer.encode_ordinary(text).unwrap(),
+                added_empty.encode(text, false).unwrap(),
+                "fused={fused}, text={text:?}",
+            );
+        }
+        if matches!(&tokenizer.backend, super::Backend::Hf(_)) {
+            assert_eq!(
+                tokenizer
+                    .encode_ordinary("hello <|regular|> Cafe\u{301} <|special|> tail")
+                    .unwrap()
+                    .len(),
+                24,
+                "HF post-processing must retain configured truncation",
+            );
+        }
+    }
+
+    #[test]
+    fn hf_ordinary_matches_original_encode_with_added_empty() {
+        for fused in [false, true] {
+            assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_hf, fused);
+        }
+    }
+
+    #[test]
+    fn fastokens_ordinary_matches_original_encode_with_added_empty() {
+        for fused in [false, true] {
+            assert_ordinary_matches_added_empty(HuggingFaceTokenizer::new_fastokens, fused);
+        }
+    }
+
     #[test]
     fn hf_constructor_resolves_added_token_ids() {
         let mut tokenizer = tiny_bpe_tokenizer();

rust/src/tokenizer/src/incremental.rs

@@ -199,6 +199,10 @@ mod tests {
             unreachable!()
         }
 
+        fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
+            unreachable!()
+        }
+
         fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
             let bytes = token_ids.iter().map(|id| *id as u8).collect::<Vec<_>>();
             Ok(String::from_utf8_lossy(&bytes).into_owned())
@@ -273,6 +277,10 @@ mod tests {
             unreachable!()
         }
 
+        fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
+            unreachable!()
+        }
+
         fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
             let mut text = String::new();
             for &token_id in token_ids {
@@ -410,6 +418,10 @@ mod tests {
             unreachable!()
         }
 
+        fn encode_ordinary(&self, _text: &str) -> Result<Vec<u32>> {
+            unreachable!()
+        }
+
         fn decode(&self, token_ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
             match token_ids {
                 [1] => Ok("abc".into()),

rust/src/tokenizer/src/lib.rs

@@ -25,6 +25,10 @@ pub trait Tokenizer: Send + Sync {
     /// Encode one prompt string into token IDs.
     fn encode(&self, text: &str, add_special_tokens: bool) -> Result<Vec<u32>>;
 
+    /// Equivalent to `encode(text, false)`, except that every added,
+    /// special, and control-token matcher is bypassed.
+    fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>>;
+
     /// Decode one token sequence into text.
     fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String>;
 

rust/src/tokenizer/src/tekken.rs

@@ -35,6 +35,12 @@ impl Tokenizer for TekkenTokenizer {
             .map_err(|error| tokenizer_error!("encoding failed: {error}"))
     }
 
+    fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
+        self.inner
+            .encode(text, false, false)
+            .map_err(|error| tokenizer_error!("encoding failed: {error}"))
+    }
+
     fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
         let policy = if skip_special_tokens {
             tekken::SpecialTokenPolicy::Ignore
@@ -67,3 +73,51 @@ impl Tokenizer for TekkenTokenizer {
         self.inner.is_special_token(token_id)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use base64::Engine as _;
+    use tekken::config::TokenizerVersion;
+    use tekken::{SpecialTokenInfo, TokenInfo};
+
+    use super::*;
+
+    fn test_tokenizer() -> TekkenTokenizer {
+        let vocab = (0_u8..=255)
+            .map(|byte| TokenInfo {
+                rank: byte as usize,
+                token_bytes: base64::engine::general_purpose::STANDARD.encode([byte]),
+                token_str: None,
+            })
+            .collect();
+        let special_tokens = vec![SpecialTokenInfo {
+            rank: 0,
+            token_str: "<control>".to_string(),
+            is_control: true,
+        }];
+        let inner = Tekkenizer::new(
+            vocab,
+            &special_tokens,
+            r"(?s).",
+            257,
+            1,
+            TokenizerVersion::V3,
+            None,
+        )
+        .expect("build Tekken tokenizer");
+        TekkenTokenizer { inner }
+    }
+
+    #[test]
+    fn ordinary_matches_tekkens_empty_special_encoding() {
+        let tokenizer = test_tokenizer();
+        let text = "user <control> text";
+        let control_id = tokenizer.token_to_id("<control>").unwrap();
+        let ordinary_ids = tokenizer.encode_ordinary(text).unwrap();
+
+        assert_eq!(control_id, 0);
+        assert_eq!(ordinary_ids, tokenizer.encode(text, false).unwrap());
+        assert!(!ordinary_ids.contains(&control_id));
+        assert_eq!(tokenizer.decode(&ordinary_ids, false).unwrap(), text);
+    }
+}

rust/src/tokenizer/src/test_utils.rs

@@ -208,6 +208,10 @@ impl Tokenizer for TestTokenizer {
         Ok(ids)
     }
 
+    fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
+        Ok(text.as_bytes().iter().copied().map(u32::from).collect())
+    }
+
     fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
         let mut output = String::new();
         let mut pending_bytes = Vec::new();
@@ -374,6 +378,32 @@ mod tests {
         assert!(!tokenizer.is_special_id(0xF002));
     }
 
+    #[test]
+    fn ordinary_encoding_bypasses_all_configured_tokens() {
+        let tokenizer = TestTokenizer::new()
+            .with_bos_token("<bos>", 256)
+            .with_special_token("<control>", 257)
+            .with_regular_token("<visible>", 258);
+        let ordinary_text = "user <control> and <visible>";
+
+        assert_eq!(tokenizer.encode("<control>", false).unwrap(), vec![257]);
+        assert_eq!(tokenizer.encode("<visible>", false).unwrap(), vec![258]);
+        assert_eq!(
+            tokenizer.encode_ordinary(ordinary_text).unwrap(),
+            ordinary_text.as_bytes().iter().copied().map(u32::from).collect::<Vec<_>>()
+        );
+
+        let mut segmented = tokenizer.encode("<control>", false).unwrap();
+        segmented.extend(tokenizer.encode_ordinary(ordinary_text).unwrap());
+        segmented.extend(tokenizer.encode("<visible>", false).unwrap());
+        assert_eq!(segmented.first(), Some(&257));
+        assert_eq!(segmented.last(), Some(&258));
+        assert_eq!(
+            tokenizer.decode(&segmented, false).unwrap(),
+            format!("<control>{ordinary_text}<visible>")
+        );
+    }
+
     #[test]
     #[should_panic(expected = "configured test token id 255 overlaps byte fallback range 0..=255")]
     fn configured_token_id_must_stay_outside_byte_range() {

rust/src/tokenizer/src/tiktoken.rs

@@ -462,6 +462,13 @@ impl Tokenizer for TiktokenTokenizer {
         })
     }
 
+    fn encode_ordinary(&self, text: &str) -> Result<Vec<u32>> {
+        Ok(match &self.backend {
+            Backend::Riptoken(backend) => backend.inner.encode_ordinary(text),
+            Backend::TiktokenRs(backend) => backend.inner.encode_ordinary(text),
+        })
+    }
+
     fn decode(&self, token_ids: &[u32], skip_special_tokens: bool) -> Result<String> {
         // Filter passes:
         //
@@ -752,6 +759,39 @@ mod tests {
         }
     }
 
+    #[test]
+    fn tiktoken_ordinary_bypasses_every_registered_added_token() {
+        let dir = tempfile::tempdir().expect("create temp dir");
+        let bpe_path = write_synthetic_bpe_file(dir.path());
+        fs::write(
+            dir.path().join("tokenizer_config.json"),
+            r#"{
+                "added_tokens_decoder": {
+                    "257": { "content": "<|im_end|>", "special": true },
+                    "258": { "content": "<|tool_call_begin|>", "special": false }
+                }
+            }"#,
+        )
+        .expect("write tokenizer_config.json");
+        fs::write(dir.path().join("config.json"), r#"{"vocab_size": 260}"#)
+            .expect("write config.json");
+
+        let input = "<|im_end|><|tool_call_begin|><|reserved_token_259|>";
+        let expected: Vec<u32> = input.as_bytes().iter().copied().map(u32::from).collect();
+        for backend in explicit_backends(&bpe_path) {
+            assert_eq!(backend.encode("<|im_end|>", false).unwrap(), vec![257]);
+            assert_eq!(
+                backend.encode("<|tool_call_begin|>", false).unwrap(),
+                vec![258]
+            );
+            assert_eq!(
+                backend.encode("<|reserved_token_259|>", false).unwrap(),
+                vec![259]
+            );
+            assert_eq!(backend.encode_ordinary(input).unwrap(), expected);
+        }
+    }
+
     /// `vocab_size` may live under `text_config` for composite (e.g.
     /// multimodal) configs.
     #[test]

rust/src/tracing/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "vllm-tracing"
+version.workspace = true
+edition.workspace = true
+description = "Shared tracing subscriber and log formatting for vLLM Rust binaries"
+license.workspace = true
+
+[dependencies]
+time.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+
+[lints]
+workspace = true

rust/src/tracing/src/lib.rs

@@ -1,6 +1,8 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+//! Shared tracing subscriber and log formatting for vLLM Rust binaries.
+
 use std::{env, fmt, process};
 
 use time::UtcOffset;
@@ -26,8 +28,8 @@ const RESET: &str = "\x1b[0m";
 const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
     format_description!("[month]-[day] [hour]:[minute]:[second]");
 
-/// Install the process-wide vLLM-style tracing subscriber for the CLI binary.
-pub(crate) fn init_tracing(process_label: &str) {
+/// Install the process-wide vLLM-style tracing subscriber.
+pub fn init_tracing(process_label: &str) {
     let filter = build_targets_filter(
         env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
         env::var("RUST_LOG").ok().as_deref(),

tests/compile/passes/distributed/test_fusion_all_reduce.py

@@ -272,12 +272,10 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
         token_num=16,
         eps=1e-6,
         dtype: torch.dtype = torch.bfloat16,
-        use_triton_quant: bool = False,
     ):
         super().__init__()
         self.hidden_size = hidden_size
         self.eps = eps
-        self.use_triton_quant = use_triton_quant
         assert hidden_size % self.quant_group_size == 0, (
             f"hidden_size ({hidden_size}) must be a multiple of "
             f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
@@ -289,10 +287,6 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
         ]
 
     def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
-        if self.use_triton_quant:
-            return torch.ops.vllm.triton_per_token_group_quant_fp8(
-                rms, self.quant_group_size
-            )
         return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
             rms, self.quant_group_size
         )
@@ -339,11 +333,7 @@ class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
     def ops_in_model_before(self):
         return [
             torch.ops.vllm.all_reduce.default,
-            (
-                torch.ops.vllm.triton_per_token_group_quant_fp8.default
-                if self.use_triton_quant
-                else torch.ops.vllm.rocm_aiter_group_fp8_quant.default
-            ),
+            torch.ops.vllm.rocm_aiter_group_fp8_quant.default,
         ]
 
     def ops_in_model_after(self):
@@ -646,7 +636,6 @@ def all_reduce_fusion_pass_on_test_model(
 
 
 @multi_gpu_test(num_gpus=2)
-@pytest.mark.parametrize("use_triton_quant", [True, False])
 @pytest.mark.parametrize("batch_size", [8])
 @pytest.mark.parametrize("seq_len", [8])
 @pytest.mark.parametrize("hidden_size", [128])
@@ -663,7 +652,6 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
     hidden_size: int,
     dtype: torch.dtype,
     enable_rms_norm_custom_op: bool,
-    use_triton_quant: bool,
     monkeypatch: pytest.MonkeyPatch,
 ):
     """Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
@@ -676,9 +664,9 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
     * ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
       single ``rms`` consumer)
     * ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
-      residual, DSv3.2 indexer fan-out; parametrized over both
-      ``triton_per_token_group_quant_fp8`` and ``rocm_aiter_group_fp8_quant``
-      producers).
+      residual, DSv3.2 indexer fan-out; parametrized over
+      ``rocm_aiter_group_fp8_quant``
+      producer).
     """
     with monkeypatch.context() as m:
         m.setenv("VLLM_ROCM_USE_AITER", "1")
@@ -703,7 +691,6 @@ def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
                 hidden_size,
                 dtype,
                 enable_rms_norm_custom_op,
-                use_triton_quant,
                 monkeypatch,
             ),
             nprocs=nprocs,
@@ -721,7 +708,6 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
     hidden_size: int,
     dtype: torch.dtype,
     enable_rms_norm_custom_op: bool,
-    use_triton_quant: bool,
     monkeypatch: pytest.MonkeyPatch,
 ):
     set_random_seed(0)
@@ -749,10 +735,7 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
     custom_ops = []
     if enable_rms_norm_custom_op:
         custom_ops.append("+rms_norm")
-    # ``triton_per_token_group_quant_fp8`` is emitted by ``QuantFP8.forward_hip``
-    # only when QuantFP8 is enabled as a custom op (and ``use_triton=True`` at
-    # the call site). The patterns in this PR are robust to both Triton and
-    # rocm_aiter forms; we always enable +quant_fp8 so the matcher's example
+    # We always enable +quant_fp8 so the matcher's example
     # trace finds the same form the test model uses.
     custom_ops.append("+quant_fp8")
 
@@ -783,9 +766,7 @@ def rocm_aiter_group_quant_fusion_pass_on_test_model(
         )
 
         token_num = batch_size * seq_len
-        model = test_model_cls(
-            hidden_size, token_num, dtype=dtype, use_triton_quant=use_triton_quant
-        )
+        model = test_model_cls(hidden_size, token_num, dtype=dtype)
 
         hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
 

tests/compile/passes/test_fusion.py

@@ -195,8 +195,6 @@ class TestModel(torch.nn.Module):
             # Blockwise path
             if self.use_aiter_fusion and self.use_aiter_quant_op:
                 return [rocm_aiter_ops.get_group_quant_op()]
-            if self.use_aiter_fusion:
-                return [torch.ops.vllm.triton_per_token_group_quant_fp8.default]
         else:
             if self.use_aiter_quant_op:
                 return [rocm_aiter_ops.get_per_token_quant_op()]

tests/compile/passes/test_silu_mul_quant_fusion.py

@@ -158,13 +158,6 @@ class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
             input_dtype=dtype,
         )
 
-        if not current_platform.is_fp8_fnuz():
-            kernel = self.w8a8_block_fp8_linear.kernel
-            orig_quant = kernel.quant_fp8
-            kernel.quant_fp8 = lambda *a, use_triton=False, **kw: orig_quant(
-                *a, use_triton=True, **kw
-            )
-
         self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
 
     def forward(self, x):
@@ -175,9 +168,7 @@ class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
     def ops_in_model_before(self):
         return [
             SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
-            rocm_aiter_ops.get_group_quant_op()
-            if current_platform.is_fp8_fnuz()
-            else torch.ops.vllm.triton_per_token_group_quant_fp8.default,
+            rocm_aiter_ops.get_group_quant_op(),
         ]
 
     def ops_in_model_after(self):

tests/compile/test_dynamic_shapes_compilation.py

@@ -1,7 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-import gc
 import tempfile
 from contextlib import contextmanager
 
@@ -9,8 +8,7 @@ import pytest
 import torch
 
 from tests.models.utils import check_logprobs_close
-from tests.utils import wait_for_rocm_memory_to_settle
-from vllm import LLM, SamplingParams
+from vllm import SamplingParams
 from vllm.compilation.decorators import support_torch_compile
 from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
 from vllm.config.compilation import (
@@ -49,6 +47,7 @@ def get_test_models():
 @pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
 def test_dynamic_shapes_compilation(
     monkeypatch,
+    vllm_runner,
     model_name,
     shapes_type,
     use_aot_compile,
@@ -79,9 +78,13 @@ def test_dynamic_shapes_compilation(
 
     print(f"Testing {shapes_type.name} dynamic shapes...")
 
-    # Initialize the model with specific dynamic shapes configuration
-    model = LLM(
-        model=model_name,
+    sampling_params = SamplingParams(max_tokens=5, temperature=0, logprobs=10)
+    test_prompts = [prompt, "The capital of France is"]
+
+    # VllmRunner shuts down the engine core on exit, so the eager model
+    # below never races a lingering compiled engine for GPU memory.
+    with vllm_runner(
+        model_name,
         compilation_config={
             "mode": CompilationMode.VLLM_COMPILE,
             "dynamic_shapes_config": {
@@ -90,33 +93,25 @@ def test_dynamic_shapes_compilation(
             },
         },
         max_model_len=1024,
-    )
+        enable_chunked_prefill=None,
+    ) as vllm_model:
+        compiled_outputs = []
+        for p in test_prompts:
+            output = vllm_model.llm.generate(p, sampling_params)[0].outputs[0]
+            assert len(output.text.strip()) > 0, "Compiled model produced empty output"
+            compiled_outputs.append((output.token_ids, output.text, output.logprobs))
 
-    sampling_params = SamplingParams(max_tokens=5, temperature=0, logprobs=10)
-    test_prompts = [prompt, "The capital of France is"]
-
-    compiled_outputs = []
-    for p in test_prompts:
-        output = model.generate(p, sampling_params)[0].outputs[0]
-        assert len(output.text.strip()) > 0, "Compiled model produced empty output"
-        compiled_outputs.append((output.token_ids, output.text, output.logprobs))
-
-    del model
-    gc.collect()
-    torch.accelerator.empty_cache()
-    torch.accelerator.synchronize()
-    wait_for_rocm_memory_to_settle()
-
-    eager_model = LLM(model=model_name, enforce_eager=True, max_model_len=1024)
-    eager_outputs = []
-    for p in test_prompts:
-        output = eager_model.generate(p, sampling_params)[0].outputs[0]
-        assert len(output.text.strip()) > 0, "Eager model produced empty output"
-        eager_outputs.append((output.token_ids, output.text, output.logprobs))
-    del eager_model
-    gc.collect()
-    torch.accelerator.empty_cache()
-    torch.accelerator.synchronize()
+    with vllm_runner(
+        model_name,
+        enforce_eager=True,
+        max_model_len=1024,
+        enable_chunked_prefill=None,
+    ) as vllm_model:
+        eager_outputs = []
+        for p in test_prompts:
+            output = vllm_model.llm.generate(p, sampling_params)[0].outputs[0]
+            assert len(output.text.strip()) > 0, "Eager model produced empty output"
+            eager_outputs.append((output.token_ids, output.text, output.logprobs))
 
     check_logprobs_close(
         outputs_0_lst=eager_outputs,
@@ -241,44 +236,39 @@ def test_model_specialization_with_evaluate_guards(
 
 
 @pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
-def test_piecewise_backend_empty_sym_shape_indices():
+def test_piecewise_backend_empty_sym_shape_indices(vllm_runner):
     """Test that PiecewiseBackend handles empty sym_shape_indices correctly.
 
     When all inputs have static shapes (no torch.SymInt), sym_shape_indices
     will be empty. The fix in PiecewiseBackend.__call__ handles this case
     by using the first compiled range_entry.
     """
-    gc.collect()
-    torch.accelerator.empty_cache()
-    torch.accelerator.synchronize()
-
     # Use small max_model_len and max_num_batched_tokens to encourage
     # static shape compilation with empty sym_shape_indices
-    llm = LLM(
-        model="Qwen/Qwen3-0.6B",
+    with vllm_runner(
+        "Qwen/Qwen3-0.6B",
         max_model_len=512,
         max_num_batched_tokens=1,
+        enable_chunked_prefill=None,
         compilation_config={
             "mode": CompilationMode.VLLM_COMPILE,
             "dynamic_shapes_config": {
                 "type": DynamicShapesType.BACKED.value,
             },
         },
-    )
+    ) as vllm_model:
+        sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=10)
 
-    sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=10)
+        # Generate with static shape inputs
+        output = vllm_model.llm.generate(
+            "Hello, my name is", sampling_params=sampling_params
+        )
+        result = output[0].outputs[0].text
+        assert len(result) > 0, "Should generate non-empty output"
 
-    # Generate with static shape inputs
-    output = llm.generate("Hello, my name is", sampling_params=sampling_params)
-    result = output[0].outputs[0].text
-    assert len(result) > 0, "Should generate non-empty output"
-
-    # Generate again to verify compilation works with empty sym_shape_indices
-    output = llm.generate("The capital of France is", sampling_params=sampling_params)
-    result = output[0].outputs[0].text
-    assert len(result) > 0, "Should generate non-empty output on second run"
-
-    del llm
-    gc.collect()
-    torch.accelerator.empty_cache()
-    torch.accelerator.synchronize()
+        # Generate again to verify compilation works with empty sym_shape_indices
+        output = vllm_model.llm.generate(
+            "The capital of France is", sampling_params=sampling_params
+        )
+        result = output[0].outputs[0].text
+        assert len(result) > 0, "Should generate non-empty output on second run"

tests/distributed/test_elastic_ep.py

@@ -3,7 +3,9 @@
 
 import os
 import subprocess
+import threading
 import time
+from concurrent.futures import ThreadPoolExecutor
 
 import pytest
 import requests
@@ -40,6 +42,106 @@ def _send_scale_command(server: RemoteOpenAIServer, new_dp_size: int) -> bool:
         return False
 
 
+def _traffic_loop(
+    server: RemoteOpenAIServer,
+    dp_rank: int | None,
+    ready: threading.Barrier,
+    stop: threading.Event,
+    finished: threading.Event,
+    is_probe: bool = False,
+) -> list[tuple[float, float, int | None]]:
+    url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
+    payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
+    headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
+    request_payload = None if is_probe else payload
+    responses = []
+    is_ready = False
+    while not stop.is_set():
+        request_start = time.perf_counter()
+        try:
+            response = requests.post(
+                url, json=request_payload, headers=headers, timeout=120
+            )
+            status_code = response.status_code
+        except requests.exceptions.RequestException:
+            status_code = None
+        responses.append((request_start, time.perf_counter(), status_code))
+        if status_code == 200:
+            if not is_ready:
+                ready.wait(timeout=120)
+                is_ready = True
+            if finished.is_set():
+                return responses
+        time.sleep(0.05)
+    return responses
+
+
+def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
+    rejected = [end for _, end, status in responses if status == 503]
+    if not rejected:
+        return 0
+    recovered = next(
+        end for _, end, status in responses if status == 200 and end > rejected[-1]
+    )
+    return recovered - rejected[0]
+
+
+def _scale_with_traffic(
+    server: RemoteOpenAIServer,
+    source_dp_size: int,
+    new_dp_size: int,
+    traffic_mode: str,
+) -> None:
+    traffic_clients: list[int | None] = []
+    if traffic_mode == "light":
+        traffic_clients = [0]
+    elif traffic_mode == "heavy":
+        traffic_clients = [None] * source_dp_size
+    clients = [(None, True)] + [(rank, False) for rank in traffic_clients]
+    ready = threading.Barrier(len(clients) + 1)
+    stop = threading.Event()
+    finished = threading.Event()
+
+    with ThreadPoolExecutor(max_workers=len(clients)) as executor:
+        futures = [
+            executor.submit(
+                _traffic_loop, server, rank, ready, stop, finished, is_probe
+            )
+            for rank, is_probe in clients
+        ]
+        try:
+            ready.wait(timeout=120)
+            start_time = time.perf_counter()
+            assert _send_scale_command(server, new_dp_size)
+            scale_seconds = time.perf_counter() - start_time
+            finished.set()
+            probe_result, *results = [future.result(timeout=120) for future in futures]
+        finally:
+            stop.set()
+
+    bad_statuses = {
+        status
+        for responses in [probe_result, *results]
+        for _, _, status in responses
+        if status not in (200, 503)
+    }
+    assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
+    probe_503 = [start for start, _, status in probe_result if status == 503]
+    assert probe_503, "Scaling probe did not observe commit"
+    assert not results or any(
+        status == 200 and start_time <= request_start and request_end < probe_503[0]
+        for responses in results
+        for request_start, request_end, status in responses
+    ), "No request completed successfully during preparation"
+
+    print(
+        f"[Elastic EP timing][{source_dp_size}->{new_dp_size}]"
+        f"[traffic={traffic_mode}] "
+        f"scale_seconds={scale_seconds:.3f} "
+        f"downtime_seconds={_downtime(probe_result):.3f}"
+    )
+
+
 def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
     assert server.port is not None
     result = evaluate_gsm8k(
@@ -59,7 +161,7 @@ def _run_gsm8k_eval(server: RemoteOpenAIServer, stage: str) -> float:
     return accuracy
 
 
-def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
+def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str]:
     args = [
         "--trust-remote-code",
         "--tensor-parallel-size",
@@ -78,57 +180,65 @@ def _base_serve_args(use_async_eplb: bool = False) -> list[str]:
         "--eplb-config.num_redundant_experts",
         "0",
         "--eplb-config.use_async",
-        "true" if use_async_eplb else "false",
+        "true",
         "--eplb-config.step_interval",
-        "10",
+        "300",
         "--eplb-config.window_size",
         "5",
         "--data-parallel-backend",
         "ray",
         "--data-parallel-size",
-        "2",
+        str(dp_size),
         "--api-server-count",
         "1",
+        "--disable-access-log-for-endpoints",
+        "/is_scaling_elastic_ep",
     ]
 
     leader_address = os.environ.get("LEADER_ADDRESS")
     if leader_address:
         args.extend(["--data-parallel-address", leader_address])
+    if enforce_eager:
+        args.append("--enforce-eager")
 
     return args
 
 
 @pytest.mark.parametrize(
-    "use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
+    ("enforce_eager", "traffic_mode"),
+    [
+        pytest.param(True, "none", id="enforce_eager_none"),
+        pytest.param(True, "light", id="enforce_eager_light"),
+        pytest.param(True, "heavy", id="enforce_eager_heavy"),
+        pytest.param(False, "heavy", id="cuda_graphs_heavy"),
+    ],
 )
 @multi_gpu_test(num_gpus=4)
-def test_elastic_ep_scaling(use_async_eplb: bool):
-    if use_async_eplb:
-        from vllm.distributed.eplb.eplb_communicator import has_nixl
+def test_elastic_ep_scaling(enforce_eager: bool, traffic_mode: str):
+    from vllm.distributed.eplb.eplb_communicator import has_nixl
 
-        if not has_nixl():
-            pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
+    if not has_nixl():
+        pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
 
-    vllm_serve_args = _base_serve_args(use_async_eplb)
+    initial_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_INITIAL_DP", "2"))
+    target_dp_size = int(os.getenv("VLLM_TEST_ELASTIC_EP_TARGET_DP", "4"))
+    assert target_dp_size > initial_dp_size
+    vllm_serve_args = _base_serve_args(initial_dp_size, enforce_eager)
 
     with RemoteOpenAIServer(
         MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
     ) as server:
-        initial_accuracy = _run_gsm8k_eval(server, "Initial (2 GPUs)")
-
-        assert _send_scale_command(server, 4)
-        time.sleep(10)
-        scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (4 GPUs)")
+        initial_accuracy = _run_gsm8k_eval(server, "Initial")
 
+        _scale_with_traffic(server, initial_dp_size, target_dp_size, traffic_mode)
+        scale_up_accuracy = _run_gsm8k_eval(server, "After scale up")
         assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
             f"Scale up accuracy {scale_up_accuracy:.3f} dropped more than "
             f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
         )
 
-        assert _send_scale_command(server, 2)
-        time.sleep(5)
-        scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
-
+        _scale_with_traffic(server, target_dp_size, initial_dp_size, traffic_mode)
+        scale_down_accuracy = _run_gsm8k_eval(server, "After scale down")
         assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (
             f"Scale down accuracy {scale_down_accuracy:.3f} dropped more than "
             f"{ACCURACY_TOL} below initial accuracy {initial_accuracy:.3f}"
@@ -147,24 +257,20 @@ def test_elastic_ep_scaling(use_async_eplb: bool):
         print(f"  Tolerance:  {ACCURACY_TOL:.3f}")
 
 
-@pytest.mark.parametrize(
-    "use_async_eplb", [False, True], ids=["sync_eplb", "async_eplb"]
-)
 @multi_gpu_test(num_gpus=4)
-def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
+def test_elastic_ep_scaling_uneven():
     """Test scale up with uneven worker distribution.
 
     This tests the case where num_new_workers % old_dp_size != 0,
     specifically 2 -> 3 where remainder = 1 % 2 = 1.
     This exercises the remainder handling in sender-receiver pairing.
     """
-    if use_async_eplb:
-        from vllm.distributed.eplb.eplb_communicator import has_nixl
+    from vllm.distributed.eplb.eplb_communicator import has_nixl
 
-        if not has_nixl():
-            pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
+    if not has_nixl():
+        pytest.skip("Async EPLB with elastic EP requires NIXL (not installed)")
 
-    vllm_serve_args = _base_serve_args(use_async_eplb)
+    vllm_serve_args = _base_serve_args()
 
     with RemoteOpenAIServer(
         MODEL_NAME, vllm_serve_args, env_dict={}, max_wait_seconds=1200
@@ -174,7 +280,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
         # Scale 2 -> 3: This has remainder = 1 % 2 = 1
         # Tests uneven sender-receiver pairing
         assert _send_scale_command(server, 3)
-        time.sleep(10)
         scale_up_accuracy = _run_gsm8k_eval(server, "After scale up (3 GPUs)")
 
         assert scale_up_accuracy >= initial_accuracy - ACCURACY_TOL, (
@@ -184,7 +289,6 @@ def test_elastic_ep_scaling_uneven(use_async_eplb: bool):
 
         # Scale back down to 2
         assert _send_scale_command(server, 2)
-        time.sleep(5)
         scale_down_accuracy = _run_gsm8k_eval(server, "After scale down (2 GPUs)")
 
         assert scale_down_accuracy >= initial_accuracy - ACCURACY_TOL, (

tests/distributed/test_shm_broadcast.py

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+import io
+import pickle
 import random
 import threading
 import time
@@ -10,12 +12,15 @@ from unittest import mock
 import multiprocess as mp
 import numpy as np
 import pytest
+import torch
 import torch.distributed as dist
 
 from vllm.distributed.device_communicators import shm_broadcast
 from vllm.distributed.device_communicators.shm_broadcast import (
     MessageQueue,
     ShmRingBuffer,
+    _rebuild_tensor,
+    _reduce_tensor,
     check_shm_free_space,
 )
 from vllm.distributed.utils import StatelessProcessGroup
@@ -354,6 +359,150 @@ def test_message_queue_busy_to_idle():
     distributed_run(worker_fn_test_busy_to_idle, 4)
 
 
+@worker_fn_wrapper
+def worker_fn_tensor_broadcast():
+    rank = dist.get_rank()
+    writer_rank = 0
+    message_queue = MessageQueue.create_from_process_group(
+        dist.group.WORLD, 8 * 1024 * 1024, 4, writer_rank
+    )
+
+    # Both ranks construct the identical reference payload.
+    torch.manual_seed(42)
+    payload = {
+        # 2MiB: rides the shm ring as an out-of-band buffer (the receiving
+        # side must copy out of the reusable ring chunk).
+        "mid": torch.randn(1024, 512),
+        # 16MiB > max_chunk_bytes: overflows to the zmq socket (the
+        # receiving side aliases the zmq.Frame zero-copy).
+        "big": torch.randn(4096, 2048, dtype=torch.bfloat16),
+        "nested": ["plain", 123, {"inner": torch.arange(5)}],
+    }
+
+    if rank == writer_rank:
+        with mock.patch(
+            "vllm.distributed.device_communicators.shm_broadcast._reduce_tensor",
+            wraps=_reduce_tensor,
+        ) as wrapped_reduce:
+            message_queue.enqueue(payload)
+        assert wrapped_reduce.call_count == 3
+        # Cycle the ring (max_chunks=4) several times over so that aliased
+        # ring chunks would be overwritten.
+        for i in range(16):
+            message_queue.enqueue({"junk": torch.full((1024, 512), float(i))})
+    else:
+        received = message_queue.dequeue(timeout=30)
+        for key in ("mid", "big"):
+            assert torch.equal(received[key], payload[key]), key
+            assert received[key].dtype == payload[key].dtype, key
+        assert torch.equal(received["nested"][2]["inner"], torch.arange(5))
+
+        snapshot = received["mid"].clone()
+        for i in range(16):
+            junk = message_queue.dequeue(timeout=30)
+            assert torch.equal(junk["junk"], torch.full((1024, 512), float(i)))
+        # Tensors received via the shm ring must not alias chunk memory
+        # that the writer has reused for subsequent messages.
+        assert torch.equal(received["mid"], snapshot)
+        # Rebuilt tensors must be writable, like regular tensors.
+        received["mid"] += 1.0
+        received["big"][0, 0] = 1.0
+
+    dist.barrier()
+    print(f"tensor broadcast passed the test! Rank {rank}")
+
+
+def test_tensor_broadcast():
+    distributed_run(worker_fn_tensor_broadcast, 2)
+
+
+def _dumps_oob(obj) -> tuple[bytes, list]:
+    """Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
+    dispatch table + out-of-band buffers >= 1MiB."""
+    buffers = []
+
+    def callback(buf: pickle.PickleBuffer) -> bool:
+        raw = buf.raw()
+        if raw.nbytes < 1024 * 1024:
+            return True
+        buffers.append(raw)
+        return False
+
+    bio = io.BytesIO()
+    pickler = pickle.Pickler(
+        bio, protocol=pickle.HIGHEST_PROTOCOL, buffer_callback=callback
+    )
+    pickler.dispatch_table = {torch.Tensor: _reduce_tensor}
+    pickler.dump(obj)
+    return bio.getvalue(), buffers
+
+
+@pytest.mark.parametrize(
+    "case",
+    [
+        "small",
+        "mid",
+        "bf16",
+        "fp8",
+        "empty",
+        "scalar",
+        "noncontig",
+        "requires_grad",
+        "conj",
+        "param",
+    ],
+)
+def test_tensor_pickle_roundtrip(case: str):
+    tensor = {
+        # Inlined in-band (< 1MiB) and out-of-band (>= 1MiB) buffers.
+        "small": lambda: torch.randn(100, 10),
+        "mid": lambda: torch.randn(1024, 512),
+        # Dtypes numpy doesn't recognize.
+        "bf16": lambda: torch.randn(512, 512, dtype=torch.bfloat16),
+        "fp8": lambda: torch.randn(32, 32).to(torch.float8_e4m3fn),
+        # Shape edge cases.
+        "empty": lambda: torch.empty(0, 8),
+        "scalar": lambda: torch.tensor(3.14),
+        "noncontig": lambda: torch.randn(64, 64).t(),
+        # These fall back to torch's default reducer.
+        "requires_grad": lambda: torch.randn(8, 8, requires_grad=True),
+        "conj": lambda: torch.randn(4, dtype=torch.complex64).conj(),
+        "param": lambda: torch.nn.Parameter(torch.randn(4), requires_grad=False),
+    }[case]()
+
+    data, buffers = _dumps_oob({"tensor": tensor, "meta": list(range(10))})
+    received = pickle.loads(data, buffers=buffers)["tensor"]
+
+    assert received.shape == tensor.shape
+    assert received.dtype == tensor.dtype
+    if tensor.dtype == torch.float8_e4m3fn:
+        assert torch.equal(received.view(torch.uint8), tensor.view(torch.uint8))
+    else:
+        assert torch.equal(received, tensor)
+    assert received.requires_grad == tensor.requires_grad
+    assert isinstance(received, type(tensor))
+    if tensor.numel() and not tensor.requires_grad:
+        # Rebuilt tensors must be writable, like regular tensors.
+        received.view(-1)[0] = 1.0
+
+
+@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
+def test_reduce_tensor_fallback(case: str):
+    """Tensors the zero-copy reducer can't safely alias must fall back to
+    torch's default reduction."""
+    if case == "cuda":
+        if not torch.cuda.is_available():
+            pytest.skip("requires CUDA")
+        tensor = torch.randn(4, device="cuda")
+    elif case == "requires_grad":
+        tensor = torch.randn(8, requires_grad=True)
+    else:
+        tensor = torch.randn(4, dtype=torch.complex64).conj()
+
+    reduced = _reduce_tensor(tensor)
+    assert reduced[0] is not _rebuild_tensor
+
+
 @pytest.mark.parametrize("should_warn", [False, True])
 def test_reader_timeout_caps_indefinite_waits(should_warn):
     with (

tests/distributed/test_weight_transfer.py

@@ -1247,7 +1247,7 @@ class RecordingClient:
         self.order.append("update")
         self.last_update_info = update_info
 
-    def finish_weight_update(self) -> None:
+    def finish_weight_update(self, weight_version: str | None = None) -> None:
         self.order.append("finish")
 
 
@@ -1303,6 +1303,10 @@ class TestTrainerClients:
         assert isinstance(update_req, WeightTransferUpdateRequest)
         assert update_req.update_info == {"names": ["w"]}
 
+        client.finish_weight_update("step-42")
+        handle.finish_weight_update.remote.assert_called_once_with()
+        handle.update_weight_version.remote.assert_called_once_with("step-42")
+
     def test_http_client_pickles_ipc_handles_for_json(self, monkeypatch):
         """HTTP update_weights must encode raw ipc_handles as a base64 pickle."""
         captured = {}
@@ -1334,6 +1338,9 @@ class TestTrainerClients:
         client.update_weights(update_info)
         assert captured["json"]["update_info"] == update_info
 
+        client.finish_weight_update("step-42")
+        assert captured["json"] == {"weight_version": "step-42"}
+
 
 class TestModuleSource:
     """`ModuleSource` metadata vs. materialized iteration (dense, no GPU)."""

tests/engine/test_short_mm_context.py

@@ -3,6 +3,8 @@
 
 import pytest
 
+from vllm.exceptions import VLLMValidationError
+
 from ..conftest import IMAGE_ASSETS
 
 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts(
@@ -19,7 +21,9 @@ models = ["llava-hf/llava-1.5-7b-hf"]
 def test_context_length_too_short(vllm_runner, image_assets, model):
     images = [asset.pil_image for asset in image_assets]
 
-    with pytest.raises(ValueError, match="longer than the maximum model length"):
+    with pytest.raises(
+        VLLMValidationError, match="longer than the maximum model length"
+    ):
         vllm_model = vllm_runner(
             model,
             # LLaVA has a feature size of 576

tests/entrypoints/llm/test_chat.py

@@ -6,6 +6,7 @@ import pytest
 
 from vllm import LLM
 from vllm.distributed import cleanup_dist_env_and_memory
+from vllm.exceptions import VLLMValidationError
 from vllm.sampling_params import SamplingParams
 
 
@@ -157,7 +158,7 @@ def test_chat_batch_failure_cleanup(llm_for_failure_test):
     batch_2 = [valid_msg, valid_msg]
     sampling_params = SamplingParams(temperature=0, max_tokens=10)
 
-    with pytest.raises(ValueError, match="maximum context length is"):
+    with pytest.raises(VLLMValidationError, match="maximum context length is"):
         llm.chat(batch_1, sampling_params=sampling_params)
     assert llm.llm_engine.get_num_unfinished_requests() == 0
 

tests/entrypoints/llm/test_prompt_validation.py

@@ -5,17 +5,18 @@ import pytest
 import torch
 
 from vllm import LLM
+from vllm.exceptions import VLLMValidationError
 
 
 def test_empty_prompt():
     llm = LLM(model="openai-community/gpt2", enforce_eager=True)
-    with pytest.raises(ValueError, match="decoder prompt cannot be empty"):
+    with pytest.raises(VLLMValidationError, match="decoder prompt cannot be empty"):
         llm.generate([""])
 
 
 def test_out_of_vocab_token():
     llm = LLM(model="openai-community/gpt2", enforce_eager=True)
-    with pytest.raises(ValueError, match="out of vocabulary"):
+    with pytest.raises(VLLMValidationError, match="out of vocabulary"):
         llm.generate({"prompt_token_ids": [999999]})
 
 

tests/entrypoints/multimodal/llm/test_mm_embeds_only.py

@@ -6,6 +6,7 @@ import pytest
 from tests.entrypoints.multimodal.conftest import managed_llm
 from vllm import LLM, SamplingParams
 from vllm.assets.image import ImageAsset
+from vllm.exceptions import VLLMValidationError
 
 MODEL = "llava-hf/llava-1.5-7b-hf"
 PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
@@ -42,7 +43,7 @@ def test_generate_with_embedding(llm: LLM):
 def test_raw_image_rejected(llm: LLM):
     """Raw image input is still rejected when limit=0."""
     raw_image = ImageAsset("stop_sign").pil_image
-    with pytest.raises(ValueError, match=r"At most 0 image\(s\)"):
+    with pytest.raises(VLLMValidationError, match=r"At most 0 image\(s\)"):
         llm.generate(
             {"prompt": PROMPT, "multi_modal_data": {"image": raw_image}},
             sampling_params=SamplingParams(max_tokens=16),

tests/entrypoints/multimodal/openai/chat_completion/test_audio.py

@@ -10,6 +10,7 @@ import pytest_asyncio
 from tests.utils import RemoteOpenAIServer
 from vllm.assets.audio import AudioAsset
 from vllm.multimodal.utils import encode_audio_base64, encode_audio_url, fetch_audio
+from vllm.platforms import current_platform
 
 MODEL_NAME = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
 TEST_AUDIO_URLS = [
@@ -18,6 +19,10 @@ TEST_AUDIO_URLS = [
 ]
 MAXIMUM_AUDIOS = 2
 
+# Disable prefix caching on ROCm to reduce non-determinism in
+# streaming-vs-non-streaming comparisons.
+_ROCM_ARGS = ["--no-enable-prefix-caching"] if current_platform.is_rocm() else []
+
 
 @pytest.fixture(scope="module")
 def server():
@@ -32,6 +37,7 @@ def server():
         "--trust-remote-code",
         "--limit-mm-per-prompt",
         json.dumps({"audio": MAXIMUM_AUDIOS}),
+        *_ROCM_ARGS,
     ]
 
     with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:

tests/entrypoints/openai/chat_completion/test_chat.py

@@ -18,6 +18,7 @@ from tests.utils import RemoteOpenAIServer
 from vllm.entrypoints.openai.chat_completion.protocol import (
     ChatCompletionRequest,
 )
+from vllm.exceptions import VLLMValidationError
 from vllm.sampling_params import SamplingParams
 
 # any model with a chat template should work here
@@ -1074,7 +1075,7 @@ def test_chat_completion_request_n_parameter_exceeds_default_limit(
         max_tokens=10,
     )
 
-    with pytest.raises(ValueError, match="n must be at most"):
+    with pytest.raises(VLLMValidationError, match="n must be at most"):
         request.to_sampling_params(
             max_tokens=10,
             default_sampling_params={},
@@ -1136,7 +1137,7 @@ def test_chat_completion_request_n_parameter_custom_limit(
         max_tokens=10,
     )
 
-    with pytest.raises(ValueError, match="n must be at most 128"):
+    with pytest.raises(VLLMValidationError, match="n must be at most 128"):
         request_over.to_sampling_params(
             max_tokens=10,
             default_sampling_params={},
@@ -1160,7 +1161,7 @@ def test_chat_completion_request_n_parameter_massive_value(
         max_tokens=1,
     )
 
-    with pytest.raises(ValueError, match="n must be at most"):
+    with pytest.raises(VLLMValidationError, match="n must be at most"):
         request.to_sampling_params(
             max_tokens=1,
             default_sampling_params={},

tests/entrypoints/openai/chat_completion/test_chat_error.py

@@ -6,7 +6,6 @@ from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
-from pydantic import ValidationError
 
 from vllm.config.multimodal import MultiModalConfig
 from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -18,6 +17,7 @@ from vllm.entrypoints.openai.engine.protocol import GenerationError
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.scale_out.render.serving import ServingRender
+from vllm.exceptions import VLLMValidationError
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers.hf import HfRenderer
 from vllm.renderers.online_renderer import OnlineRenderer
@@ -479,7 +479,7 @@ def test_json_schema_response_format_missing_schema():
 def test_structural_tag_response_format_invalid(format_value):
     """Malformed structural tags should be rejected during request validation."""
     with pytest.raises(
-        ValidationError,
+        VLLMValidationError,
         match="Invalid response_format structural_tag",
     ):
         ChatCompletionRequest(
@@ -493,7 +493,7 @@ def test_structural_tag_response_format_invalid(format_value):
 def test_batch_structural_tag_response_format_invalid(format_value):
     """Batch chat should reject malformed structural tags at request parsing."""
     with pytest.raises(
-        ValidationError,
+        VLLMValidationError,
         match="Invalid response_format structural_tag",
     ):
         BatchChatCompletionRequest(
@@ -507,7 +507,7 @@ def test_batch_structural_tag_response_format_invalid(format_value):
 def test_structured_outputs_structural_tag_invalid(structural_tag):
     """Malformed direct structured_outputs structural tags should be rejected."""
     with pytest.raises(
-        ValidationError,
+        VLLMValidationError,
         match="Invalid structured_outputs structural_tag",
     ):
         ChatCompletionRequest(
@@ -521,7 +521,7 @@ def test_structured_outputs_structural_tag_invalid(structural_tag):
 def test_non_numeric_logprobs_rejected(field_name):
     """A non-numeric logprobs value must be a clean 400 validation error, not a
     TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
-    with pytest.raises(ValidationError, match=f"`{field_name}` must be an integer"):
+    with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
         ChatCompletionRequest(
             model=MODEL_NAME,
             messages=[{"role": "user", "content": "hello"}],

tests/entrypoints/openai/chat_completion/test_logprob_token_ids.py

@@ -14,11 +14,11 @@ digit-token vocab id).
 import math
 
 import pytest
-from pydantic import ValidationError
 
 from tests.utils import RemoteOpenAIServer
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
+from vllm.exceptions import VLLMValidationError
 
 MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
 
@@ -87,7 +87,7 @@ def test_completion_request_decouples_top_k_from_explicit_token_ids():
 
 
 def test_completion_rejects_explicit_token_ids_without_generated_tokens():
-    with pytest.raises(ValidationError, match="no output tokens are generated"):
+    with pytest.raises(VLLMValidationError, match="no output tokens are generated"):
         CompletionRequest(
             model=MODEL_NAME,
             prompt="Hello",
@@ -99,7 +99,7 @@ def test_completion_rejects_explicit_token_ids_without_generated_tokens():
 
 
 def test_requests_reject_explicit_token_ids_with_beam_search():
-    with pytest.raises(ValidationError, match="not supported with beam search"):
+    with pytest.raises(VLLMValidationError, match="not supported with beam search"):
         ChatCompletionRequest(
             model=MODEL_NAME,
             messages=[{"role": "user", "content": "Hello"}],
@@ -108,7 +108,7 @@ def test_requests_reject_explicit_token_ids_with_beam_search():
             use_beam_search=True,
         )
 
-    with pytest.raises(ValidationError, match="not supported with beam search"):
+    with pytest.raises(VLLMValidationError, match="not supported with beam search"):
         CompletionRequest(
             model=MODEL_NAME,
             prompt="Hello",

tests/entrypoints/openai/chat_completion/test_thinking_token_budget_validation.py

@@ -2,15 +2,15 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import pytest
-from pydantic import ValidationError
 
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
+from vllm.exceptions import VLLMValidationError
 
 
 @pytest.mark.parametrize("raw_value", [-2, 0.6, 10.5])
 def test_chat_completion_request_rejects_invalid_thinking_token_budget(raw_value):
-    with pytest.raises(ValidationError, match="thinking_token_budget"):
+    with pytest.raises(VLLMValidationError, match="thinking_token_budget"):
         ChatCompletionRequest.model_validate(
             {
                 "model": "qwen",
@@ -44,7 +44,7 @@ def test_chat_completion_request_accepts_minus_one_as_unlimited():
 
 @pytest.mark.parametrize("raw_value", [0.6, 3.14, -2])
 def test_completion_request_rejects_invalid_thinking_token_budget(raw_value):
-    with pytest.raises(ValidationError, match="thinking_token_budget"):
+    with pytest.raises(VLLMValidationError, match="thinking_token_budget"):
         CompletionRequest.model_validate(
             {
                 "model": "qwen",

tests/entrypoints/openai/completion/test_completion_error.py

@@ -6,7 +6,6 @@ from typing import Any
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
-from pydantic import ValidationError
 
 from vllm.config.multimodal import MultiModalConfig
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
@@ -18,6 +17,7 @@ from vllm.entrypoints.openai.engine.protocol import (
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.scale_out.render.serving import ServingRender
+from vllm.exceptions import VLLMValidationError
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers.hf import HfRenderer
 from vllm.renderers.online_renderer import OnlineRenderer
@@ -430,7 +430,7 @@ def test_json_schema_response_format_missing_schema():
 def test_structural_tag_response_format_invalid(format_value):
     """Malformed structural tags should be rejected during request validation."""
     with pytest.raises(
-        ValidationError,
+        VLLMValidationError,
         match="Invalid response_format structural_tag",
     ):
         CompletionRequest(
@@ -445,7 +445,7 @@ def test_structural_tag_response_format_invalid(format_value):
 def test_structured_outputs_structural_tag_invalid(structural_tag):
     """Malformed direct structured_outputs structural tags should be rejected."""
     with pytest.raises(
-        ValidationError,
+        VLLMValidationError,
         match="Invalid structured_outputs structural_tag",
     ):
         CompletionRequest(
@@ -616,7 +616,7 @@ class TestCompletionPromptListLimit:
 def test_non_numeric_logprobs_rejected(field_name):
     """A non-numeric logprobs value must be a clean 400 validation error, not a
     TypeError from the mode='before' comparison (which surfaces as HTTP 500)."""
-    with pytest.raises(ValidationError, match=f"`{field_name}` must be an integer"):
+    with pytest.raises(VLLMValidationError, match=f"`{field_name}` must be an integer"):
         CompletionRequest(
             model=MODEL_NAME,
             prompt="Test prompt",

tests/entrypoints/openai/completion/test_prompt_validation.py

@@ -13,6 +13,7 @@ import torch
 
 from tests.utils import RemoteOpenAIServer
 from vllm.config import ModelConfig
+from vllm.exceptions import VLLMValidationError
 from vllm.renderers.embed_utils import safe_load_prompt_embeds
 
 
@@ -111,5 +112,5 @@ def test_disable_prompt_embeds(dtype: torch.dtype, seq_len: int, hidden_size: in
     buffer.seek(0)
     encoded_tensor = pybase64.b64encode(buffer.getvalue())
 
-    with pytest.raises(ValueError, match="--enable-prompt-embeds"):
+    with pytest.raises(VLLMValidationError, match="--enable-prompt-embeds"):
         safe_load_prompt_embeds(model_config, encoded_tensor)

tests/entrypoints/openai/responses/test_sampling_params.py

@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.responses.protocol import (
     ResponsesRequest,
     ResponseTextConfig,
 )
+from vllm.exceptions import VLLMValidationError
 from vllm.sampling_params import StructuredOutputsParams
 
 
@@ -163,7 +164,7 @@ class TestResponsesRequestSamplingParams:
             text=text_config,
         )
 
-        with pytest.raises(ValueError) as exc_info:
+        with pytest.raises(VLLMValidationError) as exc_info:
             request.to_sampling_params(default_max_tokens=1000)
 
         assert "Cannot specify both structured_outputs and text.format" in str(

tests/entrypoints/openai/test_dp_supervisor.py

@@ -201,6 +201,7 @@ def test_run_vllm_dp_server_uses_rust_frontend_when_enabled(monkeypatch):
     monkeypatch.setattr(dp_sup.os, "setpgrp", lambda: None)
     monkeypatch.setattr(dp_sup, "set_process_title", lambda *_args: None)
     monkeypatch.setattr(dp_sup, "decorate_logs", lambda *_args: None)
+    monkeypatch.setattr(dp_sup.envs, "VLLM_USE_RUST_FRONTEND", True, raising=False)
     monkeypatch.setattr(
         dp_sup.envs,
         "VLLM_RUST_FRONTEND_PATH",

tests/entrypoints/openai/test_openai_schema.py

@@ -148,6 +148,7 @@ def test_openapi_stateless(case: schemathesis.Case):
         "/start_draft_weight_update",
         "/update_weights",
         "/finish_weight_update",
+        "/update_weight_version",
     ):
         return
 

tests/entrypoints/serve/instrumentator/test_http_status_metrics.py

@@ -8,54 +8,95 @@ PrometheusInstrumentatorMiddleware before being caught by ServerErrorMiddleware.
 """
 
 from argparse import Namespace
-from http import HTTPStatus
 
 import httpx
 import pytest
-from fastapi import FastAPI, HTTPException, Request
-from fastapi.exceptions import RequestValidationError
-from fastapi.responses import JSONResponse
+from fastapi import HTTPException
 from prometheus_client import CollectorRegistry
-from prometheus_fastapi_instrumentator import Instrumentator
 
-from vllm.entrypoints.serve.utils.server_utils import exception_handler
-from vllm.exceptions import VLLMNotFoundError, VLLMValidationError
+from vllm.entrypoints.openai.api_server import build_app
+from vllm.exceptions import (
+    VLLMNotFoundError,
+    VLLMServerError,
+    VLLMValidationError,
+)
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
+def should_do_global_cleanup_after_test() -> bool:
+    # This suite never initializes distributed/accelerator state.
+    return False
+
+
+def _build_args() -> Namespace:
+    """Minimal args for ``build_app``; avoids ``make_arg_parser`` device probing."""
+    return Namespace(
+        disable_fastapi_docs=True,
+        enable_offline_docs=False,
+        root_path=None,
+        allowed_origins=["*"],
+        allow_credentials=False,
+        allowed_methods=["*"],
+        allowed_headers=["*"],
+        api_key=None,
+        enable_request_id_headers=False,
+        enable_fault_tolerance=False,
+        middleware=[],
+        log_error_stack=False,
+    )
+
+
+@pytest.fixture(scope="module")
 def registry():
-    """Create a fresh Prometheus registry for each test."""
+    """Shared Prometheus registry for the module-scoped app."""
     return CollectorRegistry()
 
 
-@pytest.fixture
+@pytest.fixture(scope="module")
 def app(registry):
-    """Create a minimal FastAPI app that mirrors vLLM's exception handler
-    and Prometheus middleware setup."""
+    """Build the real vLLM FastAPI app once and attach probe routes that raise.
 
-    app = FastAPI()
+    Patch the name used by ``attach_router`` (imported into the instrumentator
+    metrics module), not ``vllm.v1.metrics.prometheus`` alone — that binding is
+    captured at import time.
+    """
+    import vllm.entrypoints.serve.instrumentator.metrics as metrics_mod
 
-    # Mock app state that exception_handler needs
-    app.state.args = Namespace(log_error_stack=False)
+    original = metrics_mod.get_prometheus_registry
+    metrics_mod.get_prometheus_registry = lambda: registry
+    try:
+        app = build_app(_build_args(), supported_tasks=())
+    finally:
+        metrics_mod.get_prometheus_registry = original
 
-    # Register exception handlers exactly as vLLM does in build_app()
-    app.exception_handler(HTTPException)(_http_exception_handler)
-    app.exception_handler(RequestValidationError)(_validation_exception_handler)
-    app.exception_handler(ValueError)(exception_handler)
-    app.exception_handler(TypeError)(exception_handler)
-    app.exception_handler(OverflowError)(exception_handler)
-    app.exception_handler(NotImplementedError)(exception_handler)
-    app.exception_handler(VLLMValidationError)(exception_handler)
-    app.exception_handler(VLLMNotFoundError)(exception_handler)
-    app.exception_handler(Exception)(exception_handler)
+    @app.get("/raise_http_exception_400")
+    async def raise_http_exception_400():
+        raise HTTPException(status_code=400, detail="bad request")
 
-    # Instrument with Prometheus (same as vLLM's attach_router)
-    Instrumentator(
-        excluded_handlers=["/metrics"],
-        registry=registry,
-    ).add().instrument(app)
+    @app.get("/raise_http_exception_404")
+    async def raise_http_exception_404():
+        raise HTTPException(status_code=404, detail="not found")
+
+    @app.get("/raise_request_validation_error")
+    async def raise_request_validation_error(n: int):
+        # Invalid ``n`` triggers FastAPI's RequestValidationError.
+        return {"n": n}
+
+    @app.get("/raise_vllm_validation_error")
+    async def raise_vllm_validation_error():
+        raise VLLMValidationError("bad parameter", parameter="temperature")
+
+    @app.get("/raise_vllm_not_found_error")
+    async def raise_vllm_not_found_error():
+        raise VLLMNotFoundError("model not found")
+
+    @app.get("/raise_vllm_server_error")
+    async def raise_vllm_server_error():
+        # Bare VLLMServerError goes through vllm_error_handler → 500.
+        # EngineGenerateError / EngineDeadError are not used here: they call
+        # terminate_if_errored and need engine/server state.
+        raise VLLMServerError("internal server failure")
 
-    # Test routes that raise different exception types
     @app.get("/raise_value_error")
     async def raise_value_error():
         raise ValueError("invalid input value")
@@ -72,22 +113,6 @@ def app(registry):
     async def raise_not_implemented_error():
         raise NotImplementedError("feature not supported")
 
-    @app.get("/raise_vllm_validation_error")
-    async def raise_vllm_validation_error():
-        raise VLLMValidationError("bad parameter", parameter="temperature")
-
-    @app.get("/raise_vllm_not_found_error")
-    async def raise_vllm_not_found_error():
-        raise VLLMNotFoundError("model not found")
-
-    @app.get("/raise_http_exception_400")
-    async def raise_http_exception_400():
-        raise HTTPException(status_code=400, detail="bad request")
-
-    @app.get("/raise_http_exception_404")
-    async def raise_http_exception_404():
-        raise HTTPException(status_code=404, detail="not found")
-
     @app.get("/raise_runtime_error")
     async def raise_runtime_error():
         raise RuntimeError("unexpected server error")
@@ -99,14 +124,6 @@ def app(registry):
     return app
 
 
-async def _http_exception_handler(req: Request, exc: HTTPException):
-    return JSONResponse({"error": exc.detail}, status_code=exc.status_code)
-
-
-async def _validation_exception_handler(req: Request, exc: RequestValidationError):
-    return JSONResponse({"error": str(exc)}, status_code=HTTPStatus.BAD_REQUEST)
-
-
 def _get_http_requests_total(registry, method: str, handler: str):
     """Extract the http_requests_total metric values grouped by status.
 
@@ -128,31 +145,31 @@ def _get_http_requests_total(registry, method: str, handler: str):
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    "endpoint,expected_status_group,expected_http_code",
+    "endpoint,expected_status_group,expected_http_code,request_kwargs",
     [
-        # These should record as 4xx in Prometheus
-        ("/raise_value_error", "4xx", 400),
-        ("/raise_type_error", "4xx", 400),
-        ("/raise_overflow_error", "4xx", 400),
-        ("/raise_vllm_validation_error", "4xx", 400),
-        ("/raise_vllm_not_found_error", "4xx", 404),
-        ("/raise_http_exception_400", "4xx", 400),
-        ("/raise_http_exception_404", "4xx", 404),
-        # NotImplementedError returns 501 which is still 5xx group
-        ("/raise_not_implemented_error", "5xx", 501),
-        # These should record as 5xx in Prometheus (genuine server errors)
-        ("/raise_runtime_error", "5xx", 500),
-        # Successful requests should record as 2xx
-        ("/success", "2xx", 200),
+        ("/raise_http_exception_400", "4xx", 400, {}),
+        ("/raise_http_exception_404", "4xx", 404, {}),
+        ("/raise_request_validation_error", "4xx", 400, {"params": {"n": "x"}}),
+        ("/raise_vllm_validation_error", "4xx", 400, {}),
+        ("/raise_vllm_not_found_error", "4xx", 404, {}),
+        ("/raise_vllm_server_error", "5xx", 500, {}),
+        ("/raise_value_error", "4xx", 400, {}),
+        ("/raise_type_error", "4xx", 400, {}),
+        ("/raise_overflow_error", "4xx", 400, {}),
+        ("/raise_not_implemented_error", "5xx", 501, {}),
+        ("/raise_runtime_error", "5xx", 500, {}),
+        ("/success", "2xx", 200, {}),
     ],
     ids=[
+        "HTTPException(400)->4xx",
+        "HTTPException(404)->4xx",
+        "RequestValidationError->4xx",
+        "VLLMValidationError->4xx",
+        "VLLMNotFoundError->4xx",
+        "VLLMServerError->5xx",
         "ValueError->4xx",
         "TypeError->4xx",
         "OverflowError->4xx",
-        "VLLMValidationError->4xx",
-        "VLLMNotFoundError->4xx",
-        "HTTPException(400)->4xx",
-        "HTTPException(404)->4xx",
         "NotImplementedError->5xx",
         "RuntimeError->5xx",
         "success->2xx",
@@ -164,6 +181,7 @@ async def test_http_requests_total_records_correct_status(
     endpoint,
     expected_status_group,
     expected_http_code,
+    request_kwargs,
 ):
     """Verify that http_requests_total records the correct status group.
 
@@ -177,7 +195,7 @@ async def test_http_requests_total_records_correct_status(
     async with httpx.AsyncClient(
         transport=transport, base_url="http://testserver"
     ) as client:
-        response = await client.get(endpoint)
+        response = await client.get(endpoint, **request_kwargs)
 
     # Verify the HTTP response code returned to the client is correct
     assert response.status_code == expected_http_code, (

tests/entrypoints/unit_tests/test_chat_utils.py

@@ -21,6 +21,7 @@ from vllm.entrypoints.chat_utils import (
     parse_chat_messages,
     parse_chat_messages_async,
 )
+from vllm.exceptions import VLLMValidationError
 from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
 from vllm.multimodal.utils import (
     encode_audio_url,
@@ -1504,7 +1505,7 @@ def test_parse_chat_messages_rejects_too_many_images_in_one_message(
             "ignore",
             message="coroutine 'async_get_and_parse_image' was never awaited",
         )
-        with pytest.raises(ValueError, match="At most"):
+        with pytest.raises(VLLMValidationError, match="At most"):
             parse_chat_messages(
                 [
                     {
@@ -1540,7 +1541,7 @@ def test_parse_chat_messages_rejects_too_many_images_across_messages(
             "ignore",
             message="coroutine 'async_get_and_parse_image' was never awaited",
         )
-        with pytest.raises(ValueError, match="At most"):
+        with pytest.raises(VLLMValidationError, match="At most"):
             parse_chat_messages(
                 [
                     {

tests/entrypoints/weight_transfer/test_weight_transfer_llm.py

@@ -234,6 +234,7 @@ def test_update_weights_calls_engine():
             assert shapes == test_shapes
 
         llm.finish_weight_update()
+        assert llm.get_weight_version() == "default"
 
 
 @create_new_process_for_each_test()
@@ -259,6 +260,8 @@ def test_full_weight_transfer_flow():
             weight_transfer_config=WeightTransferConfig(backend="nccl"),
         )
 
+        assert llm.get_weight_version() == "default"
+
         # Step 1: Initialize weight transfer engine
         llm.init_weight_transfer_engine(
             WeightTransferInitRequest(init_info={"test_param": "flow_test"})
@@ -278,8 +281,15 @@ def test_full_weight_transfer_flow():
             )
         )
 
+        assert llm.get_weight_version() == "default"
+
         # Step 4: Finish weight update
-        llm.finish_weight_update()
+        llm.finish_weight_update("step-42")
+
+        assert llm.get_weight_version() == "step-42"
+
+        llm.update_weight_version("manual-version")
+        assert llm.get_weight_version() == "manual-version"
 
         # Verify the full flow completed
         def check_flow(self):

tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-baseline.yaml

@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+model_name: openai/gpt-oss-20b
+metric_threshold: 0.568
+reasoning_effort: low

tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-triton-attn.yaml

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+model_name: openai/gpt-oss-20b
+metric_threshold: 0.568
+reasoning_effort: low
+server_args: "--attention-backend TRITON_ATTN"

tests/evals/gpt_oss/configs/models-xpu.txt

@@ -0,0 +1,3 @@
+# Intel XPU model configurations for GPQA evaluation
+gpt-oss-20b-xpu-baseline.yaml
+gpt-oss-20b-xpu-triton-attn.yaml