Bump actions/setup-python from 6.1.0 to 6.3.0

Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.1.0 to 6.3.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/83679a892e2d95755f2dac6acb0bfd1e9ac5d548...ece7cb06caefa5fff74198d8649806c4678c61a1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>
Bump actions/checkout from 6.0.1 to 7.0.0 (#33057 )
2026-06-30 12:18:49 +00:00 · 2026-06-30 13:16:20 +01:00 · 2026-06-30 13:01:03 +01:00 · 2026-06-30 11:20:33 +00:00 · 2026-06-30 18:17:43 +08:00 · 2026-06-30 09:57:27 +00:00
309 changed files with 26383 additions and 4752 deletions
@@ -23,4 +23,5 @@ steps:
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
+      pytest -v -s basic_correctness/test_cpu_offload.py &&
      pytest -v -s basic_correctness/test_mem.py::test_end_to_end'
@@ -128,10 +128,10 @@ steps:
      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
      'cd tests &&
      export VLLM_WORKER_MULTIPROC_METHOD=spawn &&
-      (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) &&
      pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" &&
      pytest -v -s lora/test_transformers_model.py &&
      pytest -v -s lora/test_chatglm3_tp.py &&
+      pytest -v -s lora/test_llama_tp.py::test_llama_lora &&
      pytest -s -v lora/test_minicpmv_tp.py'

 - label: LoRA Multimodal
@@ -0,0 +1,27 @@
+group: Models - Distributed
+depends_on: 
+  - image-build-xpu
+steps:
+- label: Distributed Model Tests (2 GPUs)
+  key: distributed-model-tests-2-gpus
+  timeout_in_minutes: 50
+  device: intel_gpu
+  agent_tags:
+    label: production
+    gpu: 2+
+    mem: 24+
+  no_plugin: true
+  working_dir: "."
+  env:
+    REGISTRY: "public.ecr.aws/q9t5s3a7"
+    REPO: "vllm-ci-test-repo"
+    VLLM_TEST_DEVICE: "xpu"
+  source_file_dependencies:
+  - vllm/model_executor/model_loader/sharded_state_loader.py
+  - vllm/model_executor/models/
+  - tests/model_executor/model_loader/test_sharded_state_loader.py
+  commands:
+    - >-
+      bash .buildkite/scripts/hardware_ci/run-intel-test.sh
+      'cd tests &&
+      pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"'
@@ -125,7 +125,5 @@ steps:
      pip install open-clip-torch --no-deps &&
      cd tests &&
      pytest -v -s models/multimodal/processing/test_tensor_schema.py
-      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4]"
-      --deselect "tests/models/multimodal/processing/test_tensor_schema.py::test_model_tensor_schema[Qwen/Qwen2.5-Omni-7B-AWQ]"
      --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB'
  parallelism: 4
@@ -845,10 +845,12 @@ steps:
  source_file_dependencies:
  - vllm/
  - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out

 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
  timeout_in_minutes: 180
@@ -1196,6 +1198,27 @@ steps:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt

+- label: ROCm LM Eval Large Models (8 GPUs) # TBD
+  timeout_in_minutes: 180
+  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
+  agent_pool: mi300_8
+  optional: true
+  num_gpus: 8
+  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
+  source_file_dependencies:
+  - vllm/model_executor/models/
+  - vllm/model_executor/model_loader/
+  - vllm/model_executor/layers/quantization/
+  - vllm/v1/attention/backends/
+  - vllm/v1/attention/selector.py
+  - vllm/model_executor/layers/layernorm.py
+  - csrc/
+  - vllm/_aiter_ops.py
+  - vllm/platforms/rocm.py
+  commands:
+  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
+
 #---------------------------------------------------------  mi300 · examples  ----------------------------------------------------------#

 - label: Examples # TBD
@@ -2390,27 +2413,6 @@ steps:
  - export VLLM_USE_DEEP_GEMM=0
  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4

- label: ROCm LM Eval Large Models (8 GPUs) # TBD
-  timeout_in_minutes: 180
-  mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
-  agent_pool: mi325_8
-  optional: true
-  num_gpus: 8
-  working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
-  source_file_dependencies:
-  - vllm/model_executor/models/
-  - vllm/model_executor/model_loader/
-  - vllm/model_executor/layers/quantization/
-  - vllm/v1/attention/backends/
-  - vllm/v1/attention/selector.py
-  - vllm/model_executor/layers/layernorm.py
-  - csrc/
-  - vllm/_aiter_ops.py
-  - vllm/platforms/rocm.py
-  commands:
-  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8
-
 #-----------------------------------------------------  mi325 · models / language  -----------------------------------------------------#

 - label: Language Models Test (Extended Generation) # TBD
@@ -2559,10 +2561,12 @@ steps:
  source_file_dependencies:
  - vllm/
  - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out

 - label: Entrypoints Integration (API Server OpenAI - Part 1) # TBD
  timeout_in_minutes: 180
@@ -29,6 +29,8 @@ steps:
  mirror:
    amd:
      device: mi325_1
+      # TODO(akaratza): Test after Torch >= 2.12 bump
+      soft_fail: true
      depends_on:
      - image-build-amd

@@ -40,10 +42,12 @@ steps:
  source_file_dependencies:
  - vllm/
  - tests/entrypoints/serve
+  - tests/entrypoints/scale_out
  commands:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  - pytest -v -s entrypoints/serve --ignore=entrypoints/serve/dev/rpc
  - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
+  - pytest -v -s entrypoints/scale_out
  mirror:
    amd:
      device: mi325_1
@@ -46,7 +46,7 @@ steps:
  - vllm/v1/engine/
  - tests/utils.py
  # - tests/entrypoints/serve/dev/rpc/test_collective_rpc.py
-  - tests/entrypoints/serve/disagg/test_serving_tokens.py
+  - tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py
  - tests/entrypoints/serve/instrumentator/test_basic.py
  - tests/entrypoints/serve/instrumentator/test_metrics.py
  # - tests/entrypoints/serve/dev/test_sleep.py
@@ -55,7 +55,7 @@ steps:
  - export VLLM_WORKER_MULTIPROC_METHOD=spawn
  # - pytest -v -s entrypoints/serve/dev/rpc/test_collective_rpc.py
  - pytest -v -s entrypoints/serve/instrumentator/test_basic.py -k "not show_version and not server_load"
-  - pytest -v -s entrypoints/serve/disagg/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
+  - pytest -v -s entrypoints/scale_out/token_in_token_out/test_serving_tokens.py -k "not stream and not lora and not test_generate_logprobs and not stop_string_workflow"
  - pytest -v -s entrypoints/serve/instrumentator/test_metrics.py -k "text and not show and not run_batch and not test_metrics_counts and not test_metrics_exist"
  # - pytest -v -s entrypoints/serve/dev/test_sleep.py

@@ -94,6 +94,8 @@ steps:
    amd:
      device: mi325_1
      timeout_in_minutes: 65
+      # TODO(akaratza): Test after Torch >= 2.12 bump
+      soft_fail: true
      depends_on:
      - image-build-amd
      source_file_dependencies:
@@ -327,7 +327,7 @@ jobs:
                message: 'CC {users} for ROCm-related issue',
              },
              mistral: {
-                users: ['patrickvonplaten', 'juliendenize', 'andylolu2'],
+                users: ['patrickvonplaten', 'juliendenize', 'andylolu2', 'NickLucche'],
                message: 'CC {users} for Mistral-related issue',
              },
              // Add more label -> user mappings here
@@ -27,7 +27,7 @@ jobs:
    timeout-minutes: 30

    steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
        with:
@@ -48,8 +48,8 @@ jobs:
    if: always() && (needs.pre-run-check.result == 'success' || needs.pre-run-check.result == 'skipped')
    runs-on: [self-hosted, linux, x64, vllm-runners]
    steps:
-    - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
-    - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0
+    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+    - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
      with:
        python-version: "3.12"
    # Provide shellcheck on PATH so tools/pre_commit/shellcheck.sh skips its
@@ -131,6 +131,19 @@ repos:
        --python-version, "3.12",
      ]
      files: ^requirements/(common|xpu|test/xpu)\.(in|txt)$
+    - id: pip-compile
+      alias: pip-compile-cpu
+      name: pip-compile-cpu
+      args: [
+        requirements/test/cuda.in,
+        -o, requirements/test/cpu.txt,
+        --index-strategy, unsafe-best-match,
+        --torch-backend, cpu,
+        --python-platform, x86_64-manylinux_2_28,
+        --python-version, "3.12",
+      ]
+      files: ^requirements/(common|cpu|test/(cuda|cpu))\.(in|txt)$
+      exclude: ^requirements/test/cuda\.txt$
    - id: pip-compile
      alias: pip-compile-docs
      name: pip-compile-docs
@@ -140,6 +140,21 @@ if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
    WITH_SOABI)
 endif()

+#
+# fs_io extension (pure CXX; must stay above the non-CUDA device branch
+# so CPU builds define the target before the early return).
+# GIL-releasing filesystem helpers for FileSystemTierManager.
+#
+if(Python_VERSION VERSION_GREATER_EQUAL "3.11")
+  define_extension_target(
+    fs_io_C
+    DESTINATION vllm
+    LANGUAGE CXX
+    SOURCES csrc/fs_io.cpp
+    USE_SABI 3.11
+    WITH_SOABI)
+endif()
+
 #
 # Forward the non-CUDA device extensions to external CMake scripts.
 #
@@ -0,0 +1,69 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+#include <Python.h>
+
+#include <unistd.h>
+
+#include <vector>
+
+extern "C" {
+
+static void _batch_lookup(const std::vector<const char*>& paths,
+                          std::vector<int>& exists_flags) {
+  for (size_t i = 0; i < paths.size(); i++) {
+    exists_flags[i] = (access(paths[i], F_OK) == 0) ? 1 : 0;
+  }
+}
+
+/// @brief Check file existence for a batch of paths.
+/// @param paths list[str] – absolute paths to check.
+/// @return list[bool] – True if the corresponding path exists, False otherwise.
+/// @note Releases the GIL for the entire batch. File existence via access(2).
+static PyObject* batch_lookup(PyObject* /*self*/, PyObject* args) {
+  PyObject* path_list;
+  if (!PyArg_ParseTuple(args, "O!", &PyList_Type, &path_list)) {
+    return nullptr;
+  }
+
+  const Py_ssize_t n = PyList_Size(path_list);
+  std::vector<const char*> paths(n);
+  for (Py_ssize_t i = 0; i < n; i++) {
+    paths[i] = PyUnicode_AsUTF8AndSize(PyList_GetItem(path_list, i), nullptr);
+    if (paths[i] == nullptr) {
+      return nullptr;
+    }
+  }
+
+  std::vector<int> exists_flags(n);
+  {
+    Py_BEGIN_ALLOW_THREADS _batch_lookup(paths, exists_flags);
+    Py_END_ALLOW_THREADS
+  }
+
+  PyObject* result = PyList_New(n);
+  if (result == nullptr) {
+    return nullptr;
+  }
+  for (Py_ssize_t i = 0; i < n; i++) {
+    PyList_SetItem(result, i, PyBool_FromLong(exists_flags[i]));
+  }
+  return result;
+}
+
+static PyMethodDef fs_io_C_methods[] = {
+    {"batch_lookup", batch_lookup, METH_VARARGS,
+     "batch_lookup(paths: list[str]) -> list[bool]\n"
+     "\n"
+     "Check file existence for a batch of paths."},
+    {nullptr, nullptr, 0, nullptr},
+};
+
+static struct PyModuleDef fs_io_C_module = {
+    PyModuleDef_HEAD_INIT, "fs_io_C", "Filesystem helpers for KV offload", -1,
+    fs_io_C_methods,
+};
+
+PyMODINIT_FUNC PyInit_fs_io_C(void) { return PyModule_Create(&fs_io_C_module); }
+
+}  // extern "C"
@@ -793,7 +793,7 @@ RUN --mount=type=cache,target=/opt/uv/cache \
 # Install FlashInfer JIT cache (requires CUDA-version-specific index URL)
 # https://docs.flashinfer.ai/installation.html
 # From versions.json: .flashinfer.version
-ARG FLASHINFER_VERSION=0.6.12
+ARG FLASHINFER_VERSION=0.6.13
 RUN --mount=type=cache,target=/opt/uv/cache \
    uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \
        --index-url https://flashinfer.ai/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.')
@@ -193,26 +193,16 @@ FROM base AS vllm-test-deps

 WORKDIR /vllm-workspace

-# Copy test requirements
-COPY requirements/test/cuda.in requirements/test/cpu.in
+# Test requirements are compiled from requirements/test/cuda.in into
+# requirements/test/cpu.txt by the pip-compile-cpu pre-commit hook, which
+# resolves CPU wheels via uv's --torch-backend cpu.
+COPY requirements/test/cpu.txt requirements/test/cpu.txt

-RUN \
-    sed -i '/mamba_ssm/d' requirements/test/cpu.in && \
-    remove_packages_not_supported_on_aarch64() { \
-    case "$(uname -m)" in \
-    aarch64|arm64) \
-    sed -i '/decord/d' requirements/test/cpu.in; \
-    sed -i '/terratorch/d' requirements/test/cpu.in; \
-    ;; \
-    esac; \
-    }; \
-    remove_packages_not_supported_on_aarch64 && \
-    sed -i 's/^torch==.*/torch==2.11.0/g' requirements/test/cpu.in && \
-    sed -i 's/torchaudio.*/torchaudio/g' requirements/test/cpu.in && \
-    sed -i 's/torchvision.*/torchvision/g' requirements/test/cpu.in && \
-    # Related issue: https://github.com/vllm-project/vllm/pull/38800#issuecomment-4228314305
-    sed -i 's/^sentence-transformers.*/sentence-transformers==5.3.0/g' requirements/test/cpu.in && \
-    uv pip compile requirements/test/cpu.in -o requirements/test/cpu.txt --index-strategy unsafe-best-match --torch-backend cpu
+# cpu.txt is compiled for x86_64, so platform markers are resolved away. Drop
+# packages unavailable on aarch64 (decord, terratorch) for arm builds.
+RUN case "$(uname -m)" in \
+    aarch64|arm64) sed -i '/^decord==/d; /^terratorch==/d' requirements/test/cpu.txt ;; \
+    esac

 RUN --mount=type=cache,target=/root/.cache/uv \
    uv pip install -r requirements/test/cpu.txt
@@ -257,13 +257,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2.


 # build flashinfer for torch nightly from source around 10 mins
-# release version: v0.6.12
+# release version: v0.6.13
 # todo(elainewy): cache flashinfer build result for faster build
 ENV CCACHE_DIR=/root/.cache/ccache
 RUN --mount=type=cache,target=/root/.cache/ccache \
    --mount=type=cache,target=/root/.cache/uv \
    echo "git clone flashinfer..." \
-    && git clone --depth 1 --branch v0.6.12 --recursive https://github.com/flashinfer-ai/flashinfer.git \
+    && git clone --depth 1 --branch v0.6.13 --recursive https://github.com/flashinfer-ai/flashinfer.git \
    && cd flashinfer \
    && git submodule update --init --recursive \
    && echo "finish git clone flashinfer..." \
@@ -68,7 +68,7 @@
      "default": "true"
    },
    "FLASHINFER_VERSION": {
-      "default": "0.6.12"
+      "default": "0.6.13"
    },
    "GDRCOPY_CUDA_VERSION": {
      "default": "12.8"
@@ -167,6 +167,7 @@ Priority is **1 = highest** (tried first).
 | `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |
 | `FLASH_ATTN_DIFFKV` | | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ✅ | Decoder | Any |
 | `FLEX_ATTENTION` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder Only | Any |
+| `HPC_ATTN` | | fp16, bf16 | `auto`, `fp8_e4m3` | 64 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | ≥9.0 |
 | `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ✅ | ✅ | ❌ | ❌ | Decoder | N/A |
 | `ROCM_AITER_UNIFIED_ATTN` | | bf16 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |
 | `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |
@@ -220,7 +221,7 @@ MLA decode backends are selected using the standard
 | Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. |
 | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |
 | `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
-| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x |
+| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |
 | `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x |
 | `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x |
 | `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |
@@ -89,7 +89,7 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k
 | gpt oss triton | standard | N/A | N/A | <sup>5</sup> | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],</br>[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] |
 | marlin | standard,</br>batched | <sup>3</sup> / N/A | <sup>3</sup> / N/A | silu,</br>swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.fused_marlin_moe],</br>[`MarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.MarlinExperts],</br>[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.BatchedMarlinExperts] |
 | trtllm | standard | mxfp4,</br>nvfp4 | G(16),G(32) | <sup>5</sup> | N | Y | [`TrtLlmMxfp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsMonolithic],</br>[`TrtLlmMxfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsModular],</br>[`TrtLlmNvFp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsMonolithic],</br>[`TrtLlmNvfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsModular] |
-| hpc | standard | fp8 | G(128),T | silu | Y | Y | [`HPCExperts`][vllm.model_executor.layers.fused_moe.experts.hpc.HPCExperts] |
+| hpc | standard | fp8 | G(128),T | silu | Y | Y | [`HPCExperts`][vllm.model_executor.layers.fused_moe.hpc_moe.HPCExperts] |
 | rocm aiter moe | standard | mxfp4,</br>fp8 | G(32),G(128),A,T | silu, gelu,</br>swigluoai | Y | N | `rocm_aiter_fused_experts`,</br>`AiterExperts` |
 | cpu_fused_moe | standard | N/A | N/A | silu | N | N | [`CPUFusedMOE`][vllm.model_executor.layers.fused_moe.cpu_fused_moe.CPUFusedMOE] |
 | naive batched<sup>4</sup> | batched | int8,</br>fp8 | G,A,T | silu, gelu | <sup>6</sup> | Y | [`NaiveBatchedExperts`][vllm.model_executor.layers.fused_moe.experts.fused_batched_moe.NaiveBatchedExperts] |
@@ -13,5 +13,6 @@ vLLM's examples are organized into the following categories:
 - **[`rl/`](../../examples/rl)** – Reinforcement learning examples.
 - **[`deployment/`](../../examples/deployment)** – Examples for deploying vLLM in production.
 - **[`ray_serving/`](../../examples/ray_serving)** – Scalable serving using Ray.
- **[`disaggregated/`](../../examples/disaggregated)** – Examples for disaggregated serving (separate prefill and decode), including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
+- **[`disaggregated/`](../../examples/disaggregated)** – Examples for Disaggregated P/D (Prefill/Decoding) inference, including various kv cache connectors (LMCache, Mooncake, FlexKV, P2P NCCL) and failure recovery.
+- **[`scale_out/`](../../examples/scale_out)** – Examples for Token In <> Token Out API Server.
 - **[`observability/`](../../examples/observability)** – Metrics, logging, tracing (OpenTelemetry), and dashboards (Grafana, Perses).
@@ -120,6 +120,20 @@ To enable KV cache sharing between multiple vLLM instances using the same `root_
 PYTHONHASHSEED=0 vllm serve ...
 ```

+### P2P (Including P/D)
+
+The P2P tier (`type: "p2p"`) shares completed KV blocks between vLLM instances over RDMA via NIXL. Each instance binds a control socket on `host:port` and exchanges blocks directly with peers — no shared filesystem required.
+
+| Key | Required | Default | Notes |
+| --- | --- | --- | --- |
+| `type` | yes | — | Must be `p2p`. |
+| `host` | no | `0.0.0.0` | Address the control socket binds to. |
+| `port` | no | `7777` | Port for the control socket. Must be reachable from peers. |
+| `backends` | no | `["UCX"]` | NIXL transport backends. See [NixlConnector Usage Guide](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin) for available backends and selection guidance. |
+| `num_threads` | no | `4` | NIXL agent worker threads. Only used when `backends` is UCX-only; ignored when any non-UCX backend is requested. |
+
+The `backends` and `num_threads` options mirror the conditional logic used by [`NixlConnector`](nixl_connector_usage.md#selecting-a-nixl-transport-backend-plugin): when any non-UCX backend is configured, NIXL is initialised with `backends=...`; otherwise it falls back to a UCX-only agent with the configured `num_threads`. This lets the P2P tier use a different transport (e.g. `MOONCAKE`, `GDS_MT`, `LIBFABRIC`) than the main `NixlConnector` running in the same process.
+
 ## Tuning Tips

 - `cpu_bytes_to_use`: a bigger CPU tier means fewer trips to slower secondary tiers and a higher hit rate. The value is total across all workers, not per-worker. Leave headroom for the rest of the host workload.
@@ -586,7 +586,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
 | `MiDashengLMModel` | MiDashengLM | T + A<sup>+</sup> | `mispeech/midashenglm-7b` | | ✅︎ |
 | `MiMoV2OmniForCausalLM` | MiMo-V2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `XiaomiMiMo/MiMo-V2.5-Omni` | | ✅︎ |
 | `MiniCPMO` | MiniCPM-O | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>E+</sup> | `openbmb/MiniCPM-o-2_6`, etc. | ✅︎ | ✅︎ |
-| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, etc. | ✅︎ | |
+| `MiniCPMV` | MiniCPM-V | T + I<sup>E+</sup> + V<sup>E+</sup> | `openbmb/MiniCPM-V-2` (see note), `openbmb/MiniCPM-Llama3-V-2_5`, `openbmb/MiniCPM-V-2_6`, `openbmb/MiniCPM-V-4`, `openbmb/MiniCPM-V-4_5`, `openbmb/MiniCPM-V-4_6`, etc. | ✅︎ | |
 | `MiniMaxM3SparseForConditionalGeneration` | MiniMax-M3 | T + I<sup>+</sup> + V<sup>+</sup> | `MiniMaxAI/MiniMax-M3`, `MiniMaxAI/MiniMax-M3-MXFP8`, etc. | | ✅︎ |
 | `MiniMaxVL01ForConditionalGeneration` | MiniMax-VL | T + I<sup>E+</sup> | `MiniMaxAI/MiniMax-VL-01`, etc. | | ✅︎ |
 | `Mistral3ForConditionalGeneration` | Mistral3 (HF Transformers) | T + I<sup>+</sup> | `mistralai/Mistral-Small-3.1-24B-Instruct-2503`, etc. | ✅︎ | ✅︎ |
@@ -119,9 +119,9 @@ For further details on profiling vLLM, please refer to [this page](../../contrib
 - `/ping` - SageMaker health check
 - `/invocations` - SageMaker-compatible endpoint (routes to the same inference functions as `/v1` endpoints)

-## Disaggregated Everything
+## Scale-Out APIs

-### Tokens IN <> Tokens OUT
+### Tokens IN <> Tokens OUT APIs

 - `/inference/v1/generate` - Generate completions
 - `/abort_requests` - Abort in-flight requests (only when `--tokens-only` is also set)
@@ -85,6 +85,21 @@ significantly reduce the attack surface for these types of abuse.
 Also, consider setting `VLLM_MEDIA_URL_ALLOW_REDIRECTS=0` to prevent HTTP
 redirects from being followed to bypass domain restrictions.

+### 5. **Restrict Media Decode Sizes:**
+
+Compressed media files can expand into gigabytes of memory during decoding. vLLM
+enforces decode-size limits to prevent out-of-memory denial of service:
+
+| Environment Variable | Default | Description |
+| --- | --- | --- |
+| `VLLM_MAX_IMAGE_PIXELS` | `178956970` (~179M pixels) | Maximum decoded image size in pixels. Images exceeding this are rejected before raster memory is allocated. Default matches PIL's built-in 2x decompression-bomb threshold (~680 MB for RGB). |
+| `VLLM_MAX_AUDIO_CLIP_FILESIZE_MB` | `25` | Maximum filesize in MB for a single audio file. |
+| `VLLM_MAX_AUDIO_DECODE_DURATION_S` | `600` | Maximum decoded audio duration in seconds. Prevents compressed audio from expanding into gigabytes of float32 PCM. |
+
+Setting any of these to `0` disables the corresponding limit. This is **not
+recommended** for deployments exposed to untrusted users, as it removes the
+protection against resource-exhaustion attacks.
+
 ## Security and Firewalls: Protecting Exposed vLLM Systems

 While vLLM is designed to allow unsafe network services to be isolated to
@@ -10,8 +10,8 @@ torchaudio==2.11.0
 torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
 PyNvVideoCodec==2.0.4
 # FlashInfer should be updated together with the Dockerfile
-flashinfer-python==0.6.12
-flashinfer-cubin==0.6.12
+flashinfer-python==0.6.13
+flashinfer-cubin==0.6.13
 apache-tvm-ffi==0.1.9
 tilelang==0.1.9
 nvidia-cudnn-frontend>=1.19.1
@@ -1,3 +1,5 @@
+-r ../common.txt
+
 # testing
 pytest
 tensorizer==2.10.1
@@ -13,7 +15,6 @@ albumentations # required for Nemotron Parse in test_common.py
 av  # required for audio_in_video tests
 backoff # required for phi4mm test
 blobfile # required for kimi-vl test
-einops # required for MPT, qwen-vl
 httpx
 librosa # required for audio tests
 vector_quantize_pytorch # required for minicpmo_26 test
@@ -34,7 +35,6 @@ matplotlib # required for qwen-vl test
 mistral_common[image,audio] >= 1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
-opencv-python-headless >= 4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
@@ -55,11 +55,9 @@ grpcio-reflection==1.78.0

 arctic-inference == 0.1.1; platform_machine == "x86_64" # Required for suffix decoding test
 numba == 0.65.0 # Required for N-gram speculative decoding
-numpy
 runai-model-streamer[s3,gcs,azure]==0.15.7
 fastsafetensors>=0.3.2
 instanttensor>=0.1.5; platform_machine == "x86_64"
-pydantic>=2.12 # 2.11 leads to error on python 3.13
 decord==0.6.0; platform_machine == "x86_64"
 # terratorch is temporarily disabled while PyPI has the `lightning` package
 # in `quarantined` status (every published terratorch version transitively
@@ -9,6 +9,7 @@ aiohappyeyeballs==2.6.1
 aiohttp==3.13.3
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   aiohttp-cors
    #   datasets
    #   fsspec
@@ -24,17 +25,34 @@ albumentations==1.4.6
 alembic==1.16.4
    # via optuna
 annotated-doc==0.0.4
-    # via fastapi
+    # via
+    #   fastapi
+    #   typer
 annotated-types==0.7.0
    # via pydantic
-anyio==4.6.2.post1
+anthropic==0.112.0
    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+anyio==4.14.1
+    # via
+    #   anthropic
    #   httpx
+    #   mcp
+    #   openai
+    #   sse-starlette
    #   starlette
+    #   watchfiles
+apache-tvm-ffi==0.1.9
+    # via
+    #   -c requirements/cuda.txt
+    #   xgrammar
 arctic-inference==0.1.1
    # via -r requirements/test/cuda.in
 argcomplete==3.5.1
    # via datamodel-code-generator
+astor==0.8.1
+    # via depyf
 attrs==24.2.0
    # via
    #   aiohttp
@@ -59,6 +77,8 @@ bitsandbytes==0.49.2
    # via -r requirements/test/cuda.in
 black==24.10.0
    # via datamodel-code-generator
+blake3==1.0.9
+    # via -r requirements/test/../common.txt
 blobfile==3.0.0
    # via -r requirements/test/cuda.in
 bm25s==0.2.13
@@ -76,12 +96,17 @@ bounded-pool-executor==0.0.3
 buildkite-test-collector==0.1.9
    # via -r requirements/test/cuda.in
 cachetools==5.5.2
-    # via google-auth
+    # via
+    #   -r requirements/test/../common.txt
+    #   google-auth
+cbor2==6.1.2
+    # via -r requirements/test/../common.txt
 certifi==2024.8.30
    # via
    #   httpcore
    #   httpx
    #   requests
+    #   sentry-sdk
 cffi==2.0.0
    # via
    #   cryptography
@@ -98,9 +123,11 @@ click==8.1.7
    #   jiwer
    #   nltk
    #   ray
+    #   rich-toolkit
    #   schemathesis
-    #   typer
    #   uvicorn
+cloudpickle==3.1.2
+    # via -r requirements/test/../common.txt
 cohere-melody==0.9.0
    # via -r requirements/test/cuda.in
 colorama==0.4.6
@@ -111,6 +138,10 @@ colorful==0.5.6
    # via ray
 colorlog==6.10.1
    # via optuna
+compressed-tensors==0.17.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 contourpy==1.3.0
    # via matplotlib
 coverage==7.10.6
@@ -149,30 +180,49 @@ decorator==5.1.1
    # via librosa
 decord==0.6.0
    # via -r requirements/test/cuda.in
+depyf==0.20.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+detect-installer==0.1.0
+    # via fastapi-cloud-cli
 dill==0.3.8
    # via
    #   datasets
+    #   depyf
    #   evaluate
    #   lm-eval
    #   multiprocess
+diskcache==5.6.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 distlib==0.3.9
    # via virtualenv
+distro==1.9.0
+    # via
+    #   anthropic
+    #   openai
 dnspython==2.7.0
    # via email-validator
 docker==7.1.0
    # via gpt-oss
 docopt==0.6.2
    # via num2words
+docstring-parser==0.18.0
+    # via anthropic
 einops==0.8.1
    # via
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
    #   encodec
    #   vector-quantize-pytorch
    #   vocos
 einx==0.3.0
    # via vector-quantize-pytorch
 email-validator==2.2.0
-    # via pydantic
+    # via
+    #   fastapi
+    #   pydantic
 encodec==0.1.1
    # via vocos
 et-xmlfile==2.0.0
@@ -182,7 +232,17 @@ evaluate==0.4.3
 fastapi==0.136.3
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   gpt-oss
+    #   model-hosting-container-standards
+fastapi-cli==0.0.27
+    # via fastapi
+fastapi-cloud-cli==0.21.0
+    # via fastapi-cli
+fastar==0.11.0
+    # via
+    #   fastapi
+    #   fastapi-cloud-cli
 fastparquet==2024.11.0
    # via genai-perf
 fastrlock==0.8.2
@@ -194,6 +254,7 @@ fastsafetensors==0.3.2
 filelock==3.16.1
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   blobfile
    #   datasets
    #   huggingface-hub
@@ -243,7 +304,10 @@ google-crc32c==1.7.1
 google-resumable-media==2.7.2
    # via google-cloud-storage
 googleapis-common-protos==1.70.0
-    # via google-api-core
+    # via
+    #   google-api-core
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 gpt-oss==0.0.8
    # via -r requirements/test/cuda.in
 graphql-core==3.2.6
@@ -254,6 +318,7 @@ grpcio==1.78.0
    # via
    #   -r requirements/test/cuda.in
    #   grpcio-reflection
+    #   opentelemetry-exporter-otlp-proto-grpc
    #   ray
 grpcio-reflection==1.78.0
    # via -r requirements/test/cuda.in
@@ -275,12 +340,22 @@ html2text==2025.4.15
    # via gpt-oss
 httpcore==1.0.6
    # via httpx
+httptools==0.8.0
+    # via uvicorn
 httpx==0.27.2
    # via
    #   -r requirements/test/cuda.in
+    #   anthropic
+    #   fastapi
+    #   fastapi-cloud-cli
    #   huggingface-hub
+    #   mcp
+    #   model-hosting-container-standards
+    #   openai
    #   perceptron
    #   schemathesis
+httpx-sse==0.4.3
+    # via mcp
 huggingface-hub==1.10.2
    # via
    #   accelerate
@@ -314,6 +389,8 @@ idna==3.10
    #   httpx
    #   requests
    #   yarl
+ijson==3.5.0
+    # via -r requirements/test/../common.txt
 imagehash==4.3.2
    # via -r requirements/test/cuda.in
 imageio==2.37.0
@@ -326,6 +403,8 @@ iniconfig==2.0.0
    # via pytest
 instanttensor==0.1.5
    # via -r requirements/test/cuda.in
+interegular==0.3.3
+    # via lm-format-enforcer
 isodate==0.7.2
    # via azure-storage-blob
 isort==5.13.2
@@ -333,15 +412,21 @@ isort==5.13.2
 jinja2==3.1.6
    # via
    #   datamodel-code-generator
+    #   fastapi
    #   genai-perf
    #   lm-eval
    #   torch
+jiter==0.15.0
+    # via
+    #   anthropic
+    #   openai
 jiwer==3.0.5
    # via -r requirements/test/cuda.in
 jmespath==1.0.1
    # via
    #   boto3
    #   botocore
+    #   model-hosting-container-standards
 joblib==1.4.2
    # via
    #   librosa
@@ -350,7 +435,9 @@ joblib==1.4.2
 jsonschema==4.23.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   hypothesis-jsonschema
+    #   mcp
    #   mistral-common
    #   ray
 jsonschema-rs==0.46.5
@@ -365,6 +452,10 @@ kaleido==0.2.1
    # via genai-perf
 kiwisolver==1.4.7
    # via matplotlib
+lark==1.2.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 lazy-loader==0.4
    # via
    #   librosa
@@ -373,10 +464,20 @@ libnacl==2.1.0
    # via tensorizer
 librosa==0.10.2.post1
    # via -r requirements/test/cuda.in
+llguidance==1.7.6
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 llvmlite==0.47.0
    # via numba
 lm-eval==0.4.12
    # via -r requirements/test/cuda.in
+lm-format-enforcer==0.11.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+loguru==0.7.3
+    # via compressed-tensors
 lxml==5.3.0
    # via
    #   blobfile
@@ -398,12 +499,19 @@ mbstrdecoder==1.1.3
    #   dataproperty
    #   pytablewriter
    #   typepy
+mcp==1.28.1
+    # via -r requirements/test/../common.txt
 mdurl==0.1.2
    # via markdown-it-py
 mistral-common==1.11.5
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   -r requirements/test/cuda.in
+model-hosting-container-standards==0.1.16
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 more-itertools==10.5.0
    # via lm-eval
 mpmath==1.3.0
@@ -418,6 +526,8 @@ msgpack==1.1.0
    # via
    #   librosa
    #   ray
+msgspec==0.21.1
+    # via -r requirements/test/../common.txt
 mteb==2.8.3
    # via -r requirements/test/cuda.in
 multidict==6.1.0
@@ -434,6 +544,8 @@ networkx==3.2.1
    # via
    #   scikit-image
    #   torch
+ninja==1.13.0
+    # via -r requirements/test/../common.txt
 nltk==3.9.1
    # via rouge-score
 num2words==0.5.14
@@ -445,7 +557,7 @@ numba==0.65.0
    #   librosa
 numpy==2.2.6
    # via
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   albumentations
    #   bitsandbytes
@@ -489,6 +601,7 @@ numpy==2.2.6
    #   transformers
    #   tritonclient
    #   vocos
+    #   xgrammar
 nvidia-cublas==13.1.0.3
    # via
    #   cuda-toolkit
@@ -530,9 +643,14 @@ nvidia-nvtx==13.0.85
    # via cuda-toolkit
 open-clip-torch==2.32.0
    # via -r requirements/test/cuda.in
+openai==2.44.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 openai-harmony==0.0.4
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   gpt-oss
 opencensus==0.11.4
    # via ray
@@ -541,7 +659,7 @@ opencensus-context==0.1.3
 opencv-python-headless==4.13.0.90
    # via
    #   -c requirements/common.txt
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
    #   albumentations
    #   mistral-common
 openpyxl==3.1.5
@@ -549,24 +667,54 @@ openpyxl==3.1.5
 opentelemetry-api==1.35.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
    #   opentelemetry-exporter-prometheus
    #   opentelemetry-sdk
    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp==1.35.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+opentelemetry-exporter-otlp-proto-common==1.35.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.35.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.35.0
+    # via opentelemetry-exporter-otlp
 opentelemetry-exporter-prometheus==0.56b0
    # via ray
 opentelemetry-proto==1.35.0
-    # via ray
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   ray
 opentelemetry-sdk==1.35.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
    #   opentelemetry-exporter-prometheus
    #   ray
 opentelemetry-semantic-conventions==0.56b0
    # via opentelemetry-sdk
+opentelemetry-semantic-conventions-ai==0.4.13
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 optuna==3.6.1
    # via genai-perf
 orjson==3.11.5
    # via genai-perf
+outlines-core==0.2.14
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 packaging==24.2
    # via
    #   accelerate
@@ -578,6 +726,7 @@ packaging==24.2
    #   fastparquet
    #   huggingface-hub
    #   lazy-loader
+    #   lm-format-enforcer
    #   matplotlib
    #   optuna
    #   peft
@@ -597,6 +746,8 @@ pandas==2.2.3
    #   fastparquet
    #   genai-perf
    #   statsmodels
+partial-json-parser==0.2.1.1.post7
+    # via -r requirements/test/../common.txt
 pathspec==0.12.1
    # via black
 pathvalidate==3.2.1
@@ -611,6 +762,7 @@ perf-analyzer==0.1.0
    # via genai-perf
 pillow==10.4.0
    # via
+    #   -r requirements/test/../common.txt
    #   genai-perf
    #   imagehash
    #   imageio
@@ -644,8 +796,14 @@ pqdm==0.2.0
 prometheus-client==0.22.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   opentelemetry-exporter-prometheus
+    #   prometheus-fastapi-instrumentator
    #   ray
+prometheus-fastapi-instrumentator==8.0.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 propcache==0.2.0
    # via
    #   aiohttp
@@ -655,6 +813,7 @@ proto-plus==1.26.1
 protobuf==6.33.6
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   google-api-core
    #   googleapis-common-protos
    #   grpcio-reflection
@@ -664,11 +823,14 @@ protobuf==6.33.6
    #   tensorizer
 psutil==6.1.0
    # via
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   peft
    #   tensorizer
 py==1.11.0
    # via pytest-forked
+py-cpuinfo==9.0.0
+    # via -r requirements/test/../common.txt
 py-spy==0.4.0
    # via ray
 pyarrow==23.0.0
@@ -681,6 +843,8 @@ pyasn1==0.6.1
    #   rsa
 pyasn1-modules==0.4.2
    # via google-auth
+pybase64==1.4.3
+    # via -r requirements/test/../common.txt
 pycountry==24.6.1
    # via pydantic-extra-types
 pycparser==2.22
@@ -690,26 +854,43 @@ pycryptodomex==3.22.0
 pydantic==2.12.0
    # via
    #   -c requirements/common.txt
-    #   -r requirements/test/cuda.in
+    #   -r requirements/test/../common.txt
    #   albumentations
+    #   anthropic
+    #   compressed-tensors
    #   datamodel-code-generator
    #   fastapi
+    #   fastapi-cloud-cli
    #   gpt-oss
+    #   lm-format-enforcer
+    #   mcp
    #   mistral-common
+    #   model-hosting-container-standards
    #   mteb
+    #   openai
    #   openai-harmony
    #   pydantic-extra-types
+    #   pydantic-settings
    #   ray
+    #   xgrammar
 pydantic-core==2.41.1
    # via pydantic
 pydantic-extra-types==2.10.5
-    # via mistral-common
+    # via
+    #   fastapi
+    #   mistral-common
+pydantic-settings==2.14.2
+    # via
+    #   fastapi
+    #   mcp
 pygments==2.18.0
    # via
    #   pytest
    #   rich
 pyjwt==2.11.0
-    # via msal
+    # via
+    #   mcp
+    #   msal
 pyparsing==3.2.0
    # via matplotlib
 pyrate-limiter==4.4.0
@@ -751,6 +932,16 @@ python-dateutil==2.9.0.post0
    #   matplotlib
    #   pandas
    #   typepy
+python-dotenv==1.2.2
+    # via
+    #   pydantic-settings
+    #   uvicorn
+python-json-logger==4.1.0
+    # via -r requirements/test/../common.txt
+python-multipart==0.0.32
+    # via
+    #   fastapi
+    #   mcp
 python-rapidjson==1.20
    # via tritonclient
 pytrec-eval-terrier==0.5.7
@@ -763,12 +954,14 @@ pywavelets==1.9.0
    # via imagehash
 pyyaml==6.0.2
    # via
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   albumentations
    #   datamodel-code-generator
    #   datasets
    #   genai-perf
    #   huggingface-hub
+    #   lm-format-enforcer
    #   optuna
    #   peft
    #   ray
@@ -776,7 +969,12 @@ pyyaml==6.0.2
    #   schemathesis
    #   timm
    #   transformers
+    #   uvicorn
    #   vocos
+pyzmq==27.1.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 rapidfuzz==3.12.1
    # via jiwer
 ray==2.48.0
@@ -789,6 +987,7 @@ referencing==0.35.1
    #   jsonschema-specifications
 regex==2026.2.28
    # via
+    #   -r requirements/test/../common.txt
    #   nltk
    #   open-clip-torch
    #   sacrebleu
@@ -797,6 +996,7 @@ regex==2026.2.28
 requests==2.32.3
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   azure-core
    #   buildkite-test-collector
    #   datasets
@@ -809,6 +1009,7 @@ requests==2.32.3
    #   mistral-common
    #   msal
    #   mteb
+    #   opentelemetry-exporter-otlp-proto-http
    #   pooch
    #   ray
    #   responses
@@ -822,8 +1023,15 @@ rich==13.9.4
    #   genai-perf
    #   mteb
    #   perceptron
+    #   rich-toolkit
    #   schemathesis
    #   typer
+rich-toolkit==0.20.1
+    # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+rignore==0.7.6
+    # via fastapi-cloud-cli
 rouge-score==0.1.2
    # via lm-eval
 rpds-py==0.20.1
@@ -847,6 +1055,7 @@ sacrebleu==2.4.3
 safetensors==0.7.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   open-clip-torch
    #   peft
@@ -882,9 +1091,17 @@ sentence-transformers==5.2.0
    # via
    #   -r requirements/test/cuda.in
    #   mteb
+sentencepiece==0.2.1
+    # via -r requirements/test/../common.txt
+sentry-sdk==2.63.0
+    # via fastapi-cloud-cli
+setproctitle==1.3.7
+    # via -r requirements/test/../common.txt
 setuptools==77.0.3
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   model-hosting-container-standards
    #   pytablewriter
    #   torch
 shellingham==1.5.4
@@ -894,6 +1111,7 @@ shellingham==1.5.4
 six==1.16.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   junit-xml
    #   opencensus
    #   python-dateutil
@@ -902,8 +1120,9 @@ smart-open==7.1.0
    # via ray
 sniffio==1.3.1
    # via
-    #   anyio
+    #   anthropic
    #   httpx
+    #   openai
 sortedcontainers==2.4.0
    # via hypothesis
 soundfile==0.12.1
@@ -922,10 +1141,17 @@ sqlalchemy==2.0.41
    #   optuna
 sqlitedict==2.1.0
    # via lm-eval
+sse-starlette==3.4.5
+    # via mcp
 starlette==1.3.1
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   fastapi
+    #   mcp
+    #   model-hosting-container-standards
+    #   prometheus-fastapi-instrumentator
+    #   sse-starlette
    #   starlette-testclient
 starlette-testclient==0.4.1
    # via schemathesis
@@ -933,6 +1159,8 @@ statsmodels==0.14.4
    # via genai-perf
 structlog==25.4.0
    # via gpt-oss
+supervisor==4.3.0
+    # via model-hosting-container-standards
 sympy==1.13.3
    # via
    #   einx
@@ -962,6 +1190,7 @@ tifffile==2025.3.30
 tiktoken==0.12.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   gpt-oss
    #   lm-eval
    #   mistral-common
@@ -973,6 +1202,7 @@ timm==1.0.17
 tokenizers==0.22.2
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   -r requirements/test/cuda.in
    #   transformers
 torch==2.11.0+cu130
@@ -981,6 +1211,7 @@ torch==2.11.0+cu130
    #   -r requirements/test/cuda.in
    #   accelerate
    #   bitsandbytes
+    #   compressed-tensors
    #   encodec
    #   instanttensor
    #   mteb
@@ -994,6 +1225,7 @@ torch==2.11.0+cu130
    #   torchvision
    #   vector-quantize-pytorch
    #   vocos
+    #   xgrammar
 torchaudio==2.11.0+cu130
    # via
    #   -c requirements/cuda.txt
@@ -1009,6 +1241,7 @@ torchvision==0.26.0+cu130
    #   timm
 tqdm==4.67.3
    # via
+    #   -r requirements/test/../common.txt
    #   datasets
    #   evaluate
    #   huggingface-hub
@@ -1016,6 +1249,7 @@ tqdm==4.67.3
    #   mteb
    #   nltk
    #   open-clip-torch
+    #   openai
    #   optuna
    #   peft
    #   pqdm
@@ -1025,15 +1259,20 @@ tqdm==4.67.3
 transformers==5.5.3
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   -r requirements/test/cuda.in
+    #   compressed-tensors
    #   genai-perf
    #   peft
    #   sentence-transformers
    #   transformers-stream-generator
+    #   xgrammar
 transformers-stream-generator==0.0.5
    # via -r requirements/test/cuda.in
 triton==3.6.0
-    # via torch
+    # via
+    #   torch
+    #   xgrammar
 tritonclient==2.64.0
    # via -r requirements/test/cuda.in
 typepy==1.3.2
@@ -1041,8 +1280,10 @@ typepy==1.3.2
    #   dataproperty
    #   pytablewriter
    #   tabledata
-typer==0.15.2
+typer==0.26.8
    # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
    #   fastsafetensors
    #   huggingface-hub
    #   perceptron
@@ -1050,9 +1291,13 @@ typer==0.15.2
 typing-extensions==4.15.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   aiosignal
    #   albumentations
    #   alembic
+    #   anthropic
+    #   anyio
+    #   apache-tvm-ffi
    #   azure-core
    #   azure-identity
    #   azure-storage-blob
@@ -1062,9 +1307,13 @@ typing-extensions==4.15.0
    #   huggingface-hub
    #   librosa
    #   lm-eval
+    #   mcp
    #   mistral-common
    #   mteb
+    #   openai
    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
    #   opentelemetry-sdk
    #   opentelemetry-semantic-conventions
    #   pqdm
@@ -1072,17 +1321,20 @@ typing-extensions==4.15.0
    #   pydantic-core
    #   pydantic-extra-types
    #   pytest-asyncio
+    #   rich-toolkit
    #   schemathesis
    #   sentence-transformers
    #   sqlalchemy
    #   starlette
    #   torch
-    #   typer
    #   typing-inspection
+    #   xgrammar
 typing-inspection==0.4.2
    # via
    #   fastapi
+    #   mcp
    #   pydantic
+    #   pydantic-settings
 tzdata==2024.2
    # via pandas
 urllib3==2.2.3
@@ -1092,23 +1344,41 @@ urllib3==2.2.3
    #   docker
    #   requests
    #   responses
+    #   sentry-sdk
    #   tritonclient
 uvicorn==0.35.0
-    # via gpt-oss
+    # via
+    #   fastapi
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+    #   gpt-oss
+    #   mcp
+uvloop==0.22.1
+    # via uvicorn
 vector-quantize-pytorch==1.21.2
    # via -r requirements/test/cuda.in
 virtualenv==20.31.2
    # via ray
 vocos==0.1.0
    # via -r requirements/test/cuda.in
+watchfiles==1.2.0
+    # via
+    #   -r requirements/test/../common.txt
+    #   uvicorn
 wcwidth==0.2.13
    # via ftfy
+websockets==16.0
+    # via uvicorn
 werkzeug==3.1.3
    # via schemathesis
 word2number==1.1
    # via lm-eval
 wrapt==1.17.2
    # via smart-open
+xgrammar==0.2.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 xxhash==3.5.0
    # via
    #   datasets
@@ -15,7 +15,6 @@ albumentations # required for Nemotron Parse in test_common.py
 av # required for audio_in_video tests
 backoff # required for phi4mm test
 blobfile # required for kimi-vl test
-einops # required for MPT, qwen-vl
 httpx
 librosa # required for audio tests
 vector_quantize_pytorch # required for minicpmo_26 test
@@ -33,7 +32,6 @@ matplotlib # required for qwen-vl test
 mistral_common[image,audio]>=1.11.5 # required for voxtral test
 num2words # required for smolvlm test
 open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
-opencv-python-headless>=4.13.0 # required for video test
 datamodel_code_generator # required for minicpm3 test
 lm-eval[api]>=0.4.12 # required for model evaluation test
 mteb[bm25s]>=2, <3 # required for mteb test
@@ -54,11 +52,9 @@ grpcio-reflection==1.78.0

 arctic-inference==0.1.1 # Required for suffix decoding test
 numba==0.65.0 # Required for N-gram speculative decoding
-numpy
 runai-model-streamer[s3,gcs,azure]==0.15.7
 fastsafetensors>=0.3.2
 instanttensor>=0.1.5
-pydantic>=2.12 # 2.11 leads to error on python 3.13
 decord==0.6.0

 # Prithvi tests
@@ -74,6 +70,7 @@ gpt-oss>=0.0.7; python_version > '3.11'

 perceptron # required for isaac test
 kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
+cohere_melody>=0.9.0 # required for cohere command reasoning parser test

 # Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
 # Older versions are in conflict with terratorch requirements.
@@ -130,6 +130,8 @@ cloudpickle==3.1.2
    # via
    #   -r requirements/test/../common.txt
    #   tilelang
+cohere-melody==0.9.0
+    # via -r requirements/test/rocm.in
 colorama==0.4.6
    # via
    #   perceptron
@@ -205,7 +207,6 @@ docstring-parser==0.17.0
 einops==0.8.2
    # via
    #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
    #   encodec
    #   vector-quantize-pytorch
    #   vocos
@@ -561,7 +562,6 @@ numba==0.65.0
 numpy==2.2.6
    # via
    #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
    #   accelerate
    #   albumentations
    #   bitsandbytes
@@ -630,7 +630,6 @@ opencv-python-headless==4.13.0.92
    # via
    #   -c requirements/common.txt
    #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
    #   albumentations
    #   mistral-common
 openpyxl==3.1.5
@@ -834,7 +833,6 @@ pydantic==2.12.5
    # via
    #   -c requirements/common.txt
    #   -r requirements/test/../common.txt
-    #   -r requirements/test/rocm.in
    #   albumentations
    #   anthropic
    #   compressed-tensors
@@ -1,3 +1,5 @@
+-r ../common.txt
+
 # --- Test Infrastructure ---
 tblib
 pytest
@@ -11,6 +11,7 @@ aiohappyeyeballs==2.6.1
 aiohttp==3.13.4
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   fsspec
    #   gpt-oss
    #   lm-eval
@@ -24,12 +25,25 @@ annotated-doc==0.0.4
    #   typer
 annotated-types==0.7.0
    # via pydantic
+anthropic==0.112.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 anyio==4.13.0
    # via
+    #   anthropic
    #   httpx
+    #   mcp
+    #   openai
+    #   sse-starlette
    #   starlette
+    #   watchfiles
+apache-tvm-ffi==0.1.12
+    # via xgrammar
 arctic-inference==0.1.1
    # via -r requirements/test/xpu.in
+astor==0.8.1
+    # via depyf
 attrs==26.1.0
    # via
    #   aiohttp
@@ -39,6 +53,8 @@ audioread==3.0.1
    # via
    #   -r requirements/test/xpu.in
    #   librosa
+blake3==1.0.9
+    # via -r requirements/test/../common.txt
 blobfile==3.0.0
    # via -r requirements/test/xpu.in
 bm25s==0.2.13
@@ -47,13 +63,20 @@ bm25s==0.2.13
    #   mteb
 bounded-pool-executor==0.0.3
    # via pqdm
+cachetools==7.1.4
+    # via -r requirements/test/../common.txt
+cbor2==6.1.2
+    # via -r requirements/test/../common.txt
 certifi==2026.2.25
    # via
    #   httpcore
    #   httpx
    #   requests
+    #   sentry-sdk
 cffi==2.0.0
-    # via soundfile
+    # via
+    #   cryptography
+    #   soundfile
 chardet==5.2.0
    # via mbstrdecoder
 charset-normalizer==3.4.6
@@ -64,13 +87,22 @@ click==8.3.1
    # via
    #   jiwer
    #   nltk
+    #   rich-toolkit
    #   schemathesis
    #   typer
    #   uvicorn
+cloudpickle==3.1.2
+    # via -r requirements/test/../common.txt
 colorama==0.4.6
    # via sacrebleu
+compressed-tensors==0.17.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 coverage==7.13.5
    # via pytest-cov
+cryptography==49.0.0
+    # via pyjwt
 dataproperty==1.1.0
    # via
    #   pytablewriter
@@ -82,16 +114,35 @@ datasets==4.8.4
    #   mteb
 decorator==5.2.1
    # via librosa
+depyf==0.20.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+detect-installer==0.1.0
+    # via fastapi-cloud-cli
 dill==0.4.1
    # via
    #   datasets
+    #   depyf
    #   evaluate
    #   lm-eval
    #   multiprocess
+diskcache==5.6.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+distro==1.9.0
+    # via
+    #   anthropic
+    #   openai
+dnspython==2.8.0
+    # via email-validator
 docker==7.1.0
    # via gpt-oss
 docopt==0.6.2
    # via num2words
+docstring-parser==0.18.0
+    # via anthropic
 dpcpp-cpp-rt==2025.3.2
    # via
    #   onemkl-sycl-blas
@@ -100,15 +151,30 @@ dpcpp-cpp-rt==2025.3.2
    #   onemkl-sycl-rng
    #   onemkl-sycl-sparse
    #   torch
+einops==0.8.2
+    # via -r requirements/test/../common.txt
+email-validator==2.3.0
+    # via
+    #   fastapi
+    #   pydantic
 evaluate==0.4.6
    # via lm-eval
 fastapi==0.135.2
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   gpt-oss
+    #   model-hosting-container-standards
+fastapi-cli==0.0.27
+    # via fastapi
+fastapi-cloud-cli==0.21.0
+    # via fastapi-cli
+fastar==0.11.0
+    # via fastapi-cloud-cli
 filelock==3.25.2
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   blobfile
    #   datasets
    #   huggingface-hub
@@ -124,10 +190,16 @@ fsspec==2026.2.0
    #   evaluate
    #   huggingface-hub
    #   torch
+googleapis-common-protos==1.75.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 gpt-oss==0.0.8
    # via -r requirements/test/xpu.in
 graphql-core==3.2.8
    # via hypothesis-graphql
+grpcio==1.81.1
+    # via opentelemetry-exporter-otlp-proto-grpc
 h11==0.16.0
    # via
    #   httpcore
@@ -140,11 +212,21 @@ html2text==2025.4.15
    # via gpt-oss
 httpcore==1.0.9
    # via httpx
+httptools==0.8.0
+    # via uvicorn
 httpx==0.28.1
    # via
+    #   anthropic
    #   datasets
+    #   fastapi
+    #   fastapi-cloud-cli
    #   huggingface-hub
+    #   mcp
+    #   model-hosting-container-standards
+    #   openai
    #   schemathesis
+httpx-sse==0.4.3
+    # via mcp
 huggingface-hub==1.10.2
    # via
    #   accelerate
@@ -166,9 +248,12 @@ hypothesis-jsonschema==0.23.1
 idna==3.11
    # via
    #   anyio
+    #   email-validator
    #   httpx
    #   requests
    #   yarl
+ijson==3.5.0
+    # via -r requirements/test/../common.txt
 imageio==2.37.3
    # via scikit-image
 impi-rt==2021.17.2
@@ -212,13 +297,22 @@ intel-sycl-rt==2025.3.2
    #   dpcpp-cpp-rt
    #   oneccl
    #   torch
+interegular==0.3.3
+    # via lm-format-enforcer
 jinja2==3.1.6
    # via
    #   -c requirements/xpu.txt
+    #   fastapi
    #   lm-eval
    #   torch
+jiter==0.15.0
+    # via
+    #   anthropic
+    #   openai
 jiwer==4.0.0
    # via -r requirements/test/xpu.in
+jmespath==1.1.0
+    # via model-hosting-container-standards
 joblib==1.5.3
    # via
    #   librosa
@@ -227,7 +321,9 @@ joblib==1.5.3
 jsonschema==4.26.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   hypothesis-jsonschema
+    #   mcp
    #   mistral-common
    #   schemathesis
 jsonschema-rs==0.45.0
@@ -236,16 +332,30 @@ jsonschema-specifications==2025.9.1
    # via jsonschema
 junit-xml==1.9
    # via schemathesis
+lark==1.2.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 lazy-loader==0.5
    # via
    #   librosa
    #   scikit-image
 librosa==0.10.2.post1
    # via -r requirements/test/xpu.in
+llguidance==1.7.6
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 llvmlite==0.47.0
    # via numba
 lm-eval==0.4.12
    # via -r requirements/test/xpu.in
+lm-format-enforcer==0.11.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+loguru==0.7.3
+    # via compressed-tensors
 lxml==6.0.2
    # via
    #   blobfile
@@ -262,11 +372,14 @@ mbstrdecoder==1.1.4
    #   dataproperty
    #   pytablewriter
    #   typepy
+mcp==1.28.1
+    # via -r requirements/test/../common.txt
 mdurl==0.1.2
    # via markdown-it-py
 mistral-common==1.11.5
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   -r requirements/test/xpu.in
 mkl==2025.3.1
    # via
@@ -276,6 +389,10 @@ mkl==2025.3.1
    #   onemkl-sycl-rng
    #   onemkl-sycl-sparse
    #   torch
+model-hosting-container-standards==0.1.16
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 modelscope==1.35.3
    # via -r requirements/test/xpu.in
 more-itertools==10.8.0
@@ -284,6 +401,8 @@ mpmath==1.3.0
    # via sympy
 msgpack==1.1.2
    # via librosa
+msgspec==0.21.1
+    # via -r requirements/test/../common.txt
 mteb==2.12.7
    # via -r requirements/test/xpu.in
 multidict==6.7.1
@@ -298,6 +417,8 @@ networkx==3.6.1
    # via
    #   scikit-image
    #   torch
+ninja==1.13.0
+    # via -r requirements/test/../common.txt
 nltk==3.9.4
    # via rouge-score
 num2words==0.5.14
@@ -308,6 +429,7 @@ numba==0.65.0
    #   librosa
 numpy==2.2.6
    # via
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   albumentations
    #   bm25s
@@ -333,6 +455,7 @@ numpy==2.2.6
    #   tifffile
    #   torchvision
    #   transformers
+    #   xgrammar
 oneccl==2021.17.2
    # via
    #   oneccl-devel
@@ -356,15 +479,65 @@ onemkl-sycl-rng==2025.3.1
    # via torch
 onemkl-sycl-sparse==2025.3.1
    # via torch
+openai==2.44.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 openai-harmony==0.0.8
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   gpt-oss
 opencv-python-headless==4.13.0.92
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   albumentations
    #   mistral-common
+opentelemetry-api==1.43.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-exporter-otlp==1.43.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+opentelemetry-exporter-otlp-proto-common==1.43.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.43.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.43.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-proto==1.43.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.43.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-semantic-conventions-ai
+opentelemetry-semantic-conventions==0.64b0
+    # via
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions-ai
+opentelemetry-semantic-conventions-ai==0.5.1
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+outlines-core==0.2.14
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 packaging==26.0
    # via
    #   -c requirements/xpu.txt
@@ -373,6 +546,7 @@ packaging==26.0
    #   evaluate
    #   huggingface-hub
    #   lazy-loader
+    #   lm-format-enforcer
    #   modelscope
    #   pooch
    #   pytest
@@ -384,10 +558,13 @@ pandas==3.0.1
    # via
    #   datasets
    #   evaluate
+partial-json-parser==0.2.1.1.post7
+    # via -r requirements/test/../common.txt
 pathvalidate==3.3.1
    # via pytablewriter
 pillow==12.1.1
    # via
+    #   -r requirements/test/../common.txt
    #   imageio
    #   mistral-common
    #   scikit-image
@@ -410,16 +587,37 @@ portalocker==3.2.0
    # via sacrebleu
 pqdm==0.2.0
    # via -r requirements/test/xpu.in
+prometheus-client==0.25.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==8.0.2
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 propcache==0.4.1
    # via
    #   aiohttp
    #   yarl
+protobuf==7.35.1
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 psutil==7.2.2
-    # via accelerate
+    # via
+    #   -r requirements/test/../common.txt
+    #   accelerate
 py==1.11.0
    # via pytest-forked
+py-cpuinfo==9.0.0
+    # via -r requirements/test/../common.txt
 pyarrow==23.0.1
    # via datasets
+pybase64==1.4.3
+    # via -r requirements/test/../common.txt
 pycountry==26.2.16
    # via pydantic-extra-types
 pycparser==3.0
@@ -429,23 +627,41 @@ pycryptodomex==3.23.0
 pydantic==2.12.5
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   albumentations
+    #   anthropic
+    #   compressed-tensors
    #   fastapi
+    #   fastapi-cloud-cli
    #   gpt-oss
+    #   lm-format-enforcer
+    #   mcp
    #   mistral-common
+    #   model-hosting-container-standards
    #   mteb
+    #   openai
    #   openai-harmony
    #   pydantic-extra-types
+    #   pydantic-settings
+    #   xgrammar
 pydantic-core==2.41.5
    # via pydantic
 pydantic-extra-types==2.11.1
-    # via mistral-common
+    # via
+    #   fastapi
+    #   mistral-common
+pydantic-settings==2.14.2
+    # via
+    #   fastapi
+    #   mcp
 pyelftools==0.32
    # via triton-xpu
 pygments==2.20.0
    # via
    #   pytest
    #   rich
+pyjwt==2.13.0
+    # via mcp
 pyrate-limiter==4.1.0
    # via schemathesis
 pystemmer==3.0.0
@@ -480,19 +696,36 @@ python-dateutil==2.9.0.post0
    # via
    #   pandas
    #   typepy
+python-dotenv==1.2.2
+    # via
+    #   pydantic-settings
+    #   uvicorn
+python-json-logger==4.1.0
+    # via -r requirements/test/../common.txt
+python-multipart==0.0.32
+    # via
+    #   fastapi
+    #   mcp
 pytrec-eval-terrier==0.5.10
    # via mteb
 pytz==2026.1.post1
    # via typepy
 pyyaml==6.0.3
    # via
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   albumentations
    #   datasets
    #   huggingface-hub
+    #   lm-format-enforcer
    #   schemathesis
    #   timm
    #   transformers
+    #   uvicorn
+pyzmq==27.1.0
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 rapidfuzz==3.12.1
    # via
    #   -r requirements/test/xpu.in
@@ -503,6 +736,7 @@ referencing==0.37.0
    #   jsonschema-specifications
 regex==2026.3.32
    # via
+    #   -r requirements/test/../common.txt
    #   nltk
    #   sacrebleu
    #   tiktoken
@@ -510,6 +744,7 @@ regex==2026.3.32
 requests==2.33.1
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   datasets
    #   docker
    #   evaluate
@@ -518,6 +753,7 @@ requests==2.33.1
    #   mistral-common
    #   modelscope
    #   mteb
+    #   opentelemetry-exporter-otlp-proto-http
    #   pooch
    #   schemathesis
    #   starlette-testclient
@@ -525,8 +761,15 @@ requests==2.33.1
 rich==14.3.3
    # via
    #   mteb
+    #   rich-toolkit
    #   schemathesis
    #   typer
+rich-toolkit==0.20.1
+    # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+rignore==0.7.6
+    # via fastapi-cloud-cli
 rouge-score==0.1.2
    # via lm-eval
 rpds-py==0.30.0
@@ -538,6 +781,7 @@ sacrebleu==2.6.0
 safetensors==0.7.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   accelerate
    #   timm
    #   transformers
@@ -564,10 +808,18 @@ scipy==1.17.1
    #   sentence-transformers
 sentence-transformers==5.3.0
    # via mteb
+sentencepiece==0.2.1
+    # via -r requirements/test/../common.txt
+sentry-sdk==2.63.0
+    # via fastapi-cloud-cli
+setproctitle==1.3.7
+    # via -r requirements/test/../common.txt
 setuptools==80.10.2
    # via
    #   -c requirements/common.txt
    #   -c requirements/xpu.txt
+    #   -r requirements/test/../common.txt
+    #   model-hosting-container-standards
    #   modelscope
    #   pytablewriter
    #   torch
@@ -576,9 +828,14 @@ shellingham==1.5.4
 six==1.17.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   junit-xml
    #   python-dateutil
    #   rouge-score
+sniffio==1.3.1
+    # via
+    #   anthropic
+    #   openai
 sortedcontainers==2.4.0
    # via hypothesis
 soundfile==0.13.1
@@ -593,15 +850,24 @@ soxr==0.5.0.post1
    #   mistral-common
 sqlitedict==2.1.0
    # via lm-eval
+sse-starlette==3.4.5
+    # via mcp
 starlette==1.3.1
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   fastapi
+    #   mcp
+    #   model-hosting-container-standards
+    #   prometheus-fastapi-instrumentator
+    #   sse-starlette
    #   starlette-testclient
 starlette-testclient==0.4.1
    # via schemathesis
 structlog==25.5.0
    # via gpt-oss
+supervisor==4.3.0
+    # via model-hosting-container-standards
 sympy==1.14.0
    # via torch
 tabledata==1.3.4
@@ -636,6 +902,7 @@ tifffile==2026.3.3
 tiktoken==0.12.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   gpt-oss
    #   lm-eval
    #   mistral-common
@@ -644,19 +911,23 @@ timm==1.0.17
 tokenizers==0.22.2
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   transformers
 torch==2.12.0+xpu
    # via
    #   -c requirements/xpu.txt
    #   accelerate
+    #   compressed-tensors
    #   mteb
    #   sentence-transformers
    #   timm
    #   torchvision
+    #   xgrammar
 torchvision==0.27.0+xpu
    # via timm
 tqdm==4.67.3
    # via
+    #   -r requirements/test/../common.txt
    #   datasets
    #   evaluate
    #   huggingface-hub
@@ -664,13 +935,19 @@ tqdm==4.67.3
    #   modelscope
    #   mteb
    #   nltk
+    #   openai
    #   pqdm
    #   sentence-transformers
    #   transformers
 transformers==5.5.3
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
+    #   compressed-tensors
    #   sentence-transformers
+    #   xgrammar
+triton==3.7.1
+    # via xgrammar
 triton-xpu==3.7.1
    # via torch
 typepy==1.3.4
@@ -680,36 +957,53 @@ typepy==1.3.4
    #   tabledata
 typer==0.24.1
    # via
+    #   fastapi-cli
+    #   fastapi-cloud-cli
    #   huggingface-hub
    #   transformers
 typing-extensions==4.15.0
    # via
    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
    #   aiosignal
    #   albumentations
+    #   anthropic
    #   anyio
+    #   apache-tvm-ffi
    #   chz
    #   fastapi
+    #   grpcio
    #   huggingface-hub
    #   librosa
    #   lm-eval
+    #   mcp
    #   mistral-common
    #   mteb
+    #   openai
+    #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
    #   pqdm
    #   pydantic
    #   pydantic-core
    #   pydantic-extra-types
    #   pytest-asyncio
    #   referencing
+    #   rich-toolkit
    #   schemathesis
    #   sentence-transformers
    #   starlette
    #   torch
    #   typing-inspection
+    #   xgrammar
 typing-inspection==0.4.2
    # via
    #   fastapi
+    #   mcp
    #   pydantic
+    #   pydantic-settings
 umf==1.0.3
    # via
    #   intel-cmplr-lib-ur
@@ -720,12 +1014,30 @@ urllib3==2.6.3
    #   docker
    #   modelscope
    #   requests
+    #   sentry-sdk
 uvicorn==0.42.0
-    # via gpt-oss
+    # via
+    #   fastapi
+    #   fastapi-cli
+    #   fastapi-cloud-cli
+    #   gpt-oss
+    #   mcp
+uvloop==0.22.1
+    # via uvicorn
+watchfiles==1.2.0
+    # via
+    #   -r requirements/test/../common.txt
+    #   uvicorn
+websockets==16.0
+    # via uvicorn
 werkzeug==3.1.7
    # via schemathesis
 word2number==1.1
    # via lm-eval
+xgrammar==0.2.3
+    # via
+    #   -c requirements/common.txt
+    #   -r requirements/test/../common.txt
 xxhash==3.6.0
    # via
    #   datasets
@@ -272,6 +272,18 @@ version = "1.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"

+[[package]]
+name = "auto_enums"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e4487600931c9a89f8db7ffbdf3fbdd45bb7bd85e26861f659a463cd0dff966"
+dependencies = [
+ "derive_utils",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "auto_impl"
 version = "1.3.0"
@@ -938,6 +950,17 @@ dependencies = [
 "unicode-xid",
 ]

+[[package]]
+name = "derive_utils"
+version = "0.15.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "362f47930db19fe7735f527e6595e4900316b893ebf6d48ad3d31be928d57dd6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.117",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -1478,9 +1501,9 @@ dependencies = [

 [[package]]
 name = "h2"
-version = "0.4.13"
+version = "0.4.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
 dependencies = [
 "atomic-waker",
 "bytes",
@@ -1638,9 +1661,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"

 [[package]]
 name = "hyper"
-version = "1.8.1"
+version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11"
+checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498"
 dependencies = [
 "atomic-waker",
 "bytes",
@@ -1653,7 +1676,6 @@ dependencies = [
 "httpdate",
 "itoa",
 "pin-project-lite",
- "pin-utils",
 "smallvec",
 "tokio",
 "want",
@@ -2569,15 +2591,14 @@ dependencies = [

 [[package]]
 name = "openssl"
-version = "0.10.76"
+version = "0.10.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "951c002c75e16ea2c65b8c7e4d3d51d5530d8dfa7d060b4776828c88cfb18ecf"
+checksum = "77823a27f0babb03091cb9ed9ef80af3b39dbc82f97e8fa530374b7dafd87a45"
 dependencies = [
 "bitflags",
 "cfg-if",
 "foreign-types",
 "libc",
- "once_cell",
 "openssl-macros",
 "openssl-sys",
 ]
@@ -2610,9 +2631,9 @@ dependencies = [

 [[package]]
 name = "openssl-sys"
-version = "0.9.112"
+version = "0.9.117"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
+checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
 dependencies = [
 "cc",
 "libc",
@@ -2783,12 +2804,6 @@ version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"

-[[package]]
-name = "pin-utils"
-version = "0.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
-
 [[package]]
 name = "pkg-config"
 version = "0.3.32"
@@ -2988,7 +3003,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7"
 dependencies = [
 "heck",
- "itertools 0.10.5",
+ "itertools 0.14.0",
 "log",
 "multimap",
 "petgraph",
@@ -3009,7 +3024,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b"
 dependencies = [
 "anyhow",
- "itertools 0.10.5",
+ "itertools 0.14.0",
 "proc-macro2",
 "quote",
 "syn 2.0.117",
@@ -3503,9 +3518,9 @@ dependencies = [

 [[package]]
 name = "rustls-pki-types"
-version = "1.14.0"
+version = "1.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
+checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
 dependencies = [
 "zeroize",
 ]
@@ -4385,6 +4400,22 @@ dependencies = [
 "serde_json",
 ]

+[[package]]
+name = "tls-listener"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1461056cc1ef47003f7ee16e4cef3741068d4c7f6b627bfce49b7c00c120a530"
+dependencies = [
+ "axum",
+ "futures-util",
+ "openssl",
+ "pin-project-lite",
+ "thiserror 2.0.18",
+ "tokio",
+ "tokio-openssl",
+ "tracing",
+]
+
 [[package]]
 name = "tokenizers"
 version = "0.22.2"
@@ -4457,6 +4488,17 @@ dependencies = [
 "tokio",
 ]

+[[package]]
+name = "tokio-openssl"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59df6849caa43bb7567f9a36f863c447d95a11d5903c9cc334ba32576a27eadd"
+dependencies = [
+ "openssl",
+ "openssl-sys",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.26.4"
@@ -5220,6 +5262,7 @@ dependencies = [
 "anyhow",
 "async-openai",
 "asynk-strim-attr",
+ "auto_enums",
 "axum",
 "bytes",
 "clap",
@@ -5227,10 +5270,13 @@ dependencies = [
 "expect-test",
 "futures",
 "http-body",
+ "hyper",
+ "hyper-util",
 "indexmap 2.13.0",
 "itertools 0.14.0",
 "libc",
 "llm-multimodal",
+ "openssl",
 "prost",
 "prost-types",
 "rmp-serde",
@@ -5242,8 +5288,11 @@ dependencies = [
 "sha2",
 "socket2",
 "subtle",
+ "tempfile",
 "thiserror-ext",
+ "tls-listener",
 "tokio",
+ "tokio-openssl",
 "tokio-stream",
 "tokio-util",
 "tonic",
@@ -26,6 +26,7 @@ arc-swap = "1.9.0"
 async-openai = { version = "0.33.1", default-features = false, features = ["native-tls"] }
 async-trait = "0.1.89"
 asynk-strim-attr = "0.1.0"
+auto_enums = { version = "0.8.9", features = ["tokio1"] }
 axum = "0.8.8"
 base64 = "0.22.1"
 bytemuck = { version = "1.25.0", features = ["extern_crate_alloc"] }
@@ -43,6 +44,12 @@ half = { version = "2.7.1", features = ["bytemuck"] }
 hex = "0.4.3"
 hf-hub = { version = "0.5.0", default-features = false, features = ["tokio"] }
 http-body = "1.0.1"
+hyper = { version = "1.10.1", features = ["http1", "server"] }
+hyper-util = { version = "0.1.20", features = [
+    "server-graceful",
+    "service",
+    "tokio",
+] }
 indexmap = "2.13.0"
 itertools = "0.14.0"
 libc = "0.2.177"
@@ -54,6 +61,7 @@ native-tls-vendored = { package = "native-tls", version = "0.2.18", features = [
 ndarray = { version = "0.16.1", features = ["serde"] }
 openai-harmony = { package = "oss-harmony", git = "https://github.com/oss-harmony/harmony", tag = "v0.0.11", default-features = false }
 openai-protocol = "1.6.0"
+openssl = "0.10"
 parking_lot = "0.12.5"
 paste = "1.0.15"
 prometheus-client = "0.24.0"
@@ -89,6 +97,7 @@ thiserror = "2.0.16"
 thiserror-ext = "0.3.0"
 tiktoken-rs = "0.9.1"
 time = { version = "0.3.47", features = ["formatting", "local-offset", "macros"] }
+tls-listener = { version = "0.11.2", default-features = false, features = ["openssl", "tokio-net", "axum"] }
 tokenizers = "0.22.0"
 tokio = { version = "1.47.1", features = [
    "macros",
@@ -97,6 +106,7 @@ tokio = { version = "1.47.1", features = [
    "sync",
    "time",
 ] }
+tokio-openssl = "0.6"
 tokio-stream = "0.1"
 tokio-util = { version = "0.7.18", features = ["rt"] }
 tonic = "0.14.5"
@@ -25,7 +25,7 @@ use vllm_managed_engine::ManagedEngineConfig;
 use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
 use vllm_server::{
    ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
-    HttpListenerMode, ParserSelection, RendererSelection,
+    DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
 };

 use crate::cli::unsupported::UnsupportedArgs;
@@ -154,6 +154,11 @@ pub struct SharedRuntimeArgs {
    #[arg(long, default_value_t = 0)]
    #[serde(default)]
    pub shutdown_timeout: u64,
+    /// Maximum idle time (seconds) on a keep-alive HTTP connection before the
+    /// server closes it (default 5).
+    #[arg(long = "http-timeout-keep-alive", env = "VLLM_HTTP_TIMEOUT_KEEP_ALIVE")]
+    #[serde(default)]
+    pub http_timeout_keep_alive: Option<u64>,

    /// The file path to the chat template, or the template in single-line form
    /// for the specified model.
@@ -257,6 +262,34 @@ pub struct SharedRuntimeArgs {
    #[serde(default)]
    pub allow_credentials: bool,

+    /// The file path to the SSL key file. When omitted, the key is read from
+    /// `--ssl-certfile` (combined PEM).
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_keyfile: Option<String>,
+
+    /// The file path to the SSL cert file. Enables TLS when set.
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_certfile: Option<String>,
+
+    /// The CA certificates file used to verify client certificates (mTLS).
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_ca_certs: Option<String>,
+
+    /// Whether a client certificate is required: 0 = none, 1 = optional,
+    /// 2 = required (mirrors Python's `ssl.CERT_*`).
+    #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
+    #[serde(default)]
+    pub ssl_cert_reqs: i32,
+
+    /// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
+    /// When unset, the linked OpenSSL's default suites are used.
+    #[arg(long)]
+    #[serde(default)]
+    pub ssl_ciphers: Option<String>,
+
    /// Unsupported Python vLLM frontend arguments recognized but not yet
    /// implemented in Rust.
    #[educe(Debug(ignore))]
@@ -277,6 +310,13 @@ impl SharedRuntimeArgs {
        Duration::from_secs(self.shutdown_timeout)
    }

+    /// Maximum idle time on a keep-alive HTTP connection before the server
+    /// closes it.
+    pub fn keep_alive_timeout(&self) -> Duration {
+        self.http_timeout_keep_alive
+            .map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
+    }
+
    /// Apply fallback logic for API key configuration from env variables.
    fn apply_env_api_key_fallback(&mut self) {
        if self.api_key.is_empty()
@@ -301,8 +341,10 @@ impl SharedRuntimeArgs {
    ) -> Config {
        let ready_timeout = self.ready_timeout();
        let shutdown_timeout = self.shutdown_timeout();
+        let keep_alive_timeout = self.keep_alive_timeout();
        let api_server_options = self.api_server_options();
        let cors = self.cors_config();
+        let tls = self.tls_config();

        Config {
            transport_mode: TransportMode::Bootstrapped {
@@ -329,10 +371,12 @@ impl SharedRuntimeArgs {
            max_logprobs: self.max_logprobs,
            api_server_options,
            cors,
+            tls,
            api_keys: self.api_key,
            disable_log_stats: self.disable_log_stats,
            grpc_port: self.grpc_port,
            shutdown_timeout,
+            keep_alive_timeout,
        }
    }

@@ -349,8 +393,10 @@ impl SharedRuntimeArgs {
    ) -> Config {
        let ready_timeout = self.ready_timeout();
        let shutdown_timeout = self.shutdown_timeout();
+        let keep_alive_timeout = self.keep_alive_timeout();
        let api_server_options = self.api_server_options();
        let cors = self.cors_config();
+        let tls = self.tls_config();

        Config {
            transport_mode: TransportMode::HandshakeOwner {
@@ -375,10 +421,12 @@ impl SharedRuntimeArgs {
            max_logprobs: self.max_logprobs,
            api_server_options,
            cors,
+            tls,
            api_keys: self.api_key,
            disable_log_stats: self.disable_log_stats,
            grpc_port: self.grpc_port,
            shutdown_timeout,
+            keep_alive_timeout,
        }
    }

@@ -398,6 +446,23 @@ impl SharedRuntimeArgs {
            allow_credentials: self.allow_credentials,
        }
    }
+
+    /// Build the TLS config: `Some` when any `ssl_*` argument is set, else
+    /// `None` (plaintext). The combination is validated in [`Config::validate`].
+    fn tls_config(&self) -> Option<TlsConfig> {
+        let tls_requested = self.ssl_certfile.is_some()
+            || self.ssl_keyfile.is_some()
+            || self.ssl_ca_certs.is_some()
+            || self.ssl_cert_reqs != 0
+            || self.ssl_ciphers.is_some();
+        tls_requested.then(|| TlsConfig {
+            cert_file: self.ssl_certfile.clone(),
+            key_file: self.ssl_keyfile.clone(),
+            ca_certs: self.ssl_ca_certs.clone(),
+            cert_reqs: self.ssl_cert_reqs,
+            ciphers: self.ssl_ciphers.clone(),
+        })
+    }
 }

 fn default_engine_ready_timeout_secs() -> u64 {
@@ -41,6 +41,7 @@ fn serve_args_forward_python_flags_with_separator() {
                        max_logprobs: None,
                        grpc_port: None,
                        shutdown_timeout: 0,
+                        http_timeout_keep_alive: None,
                        chat_template: None,
                        default_chat_template_kwargs: None,
                        chat_template_content_format: Auto,
@@ -65,6 +66,11 @@ fn serve_args_forward_python_flags_with_separator() {
                            ],
                        ),
                        allow_credentials: false,
+                        ssl_keyfile: None,
+                        ssl_certfile: None,
+                        ssl_ca_certs: None,
+                        ssl_cert_reqs: 0,
+                        ssl_ciphers: None,
                    },
                    managed_engine: ManagedEngineArgs {
                        python: "../vllm/.venv/bin/python",
@@ -363,6 +369,140 @@ fn serve_passes_enable_prompt_tokens_details_into_config() {
    assert!(config.api_server_options.enable_prompt_tokens_details);
 }

+#[test]
+fn serve_passes_tls_into_config() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--ssl-certfile",
+        "/tmp/cert.pem",
+        "--ssl-keyfile",
+        "/tmp/key.pem",
+        "--ssl-ca-certs",
+        "/tmp/ca.pem",
+        "--ssl-cert-reqs",
+        "2",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    let tls = config.tls.expect("tls configured");
+    assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
+    assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
+    assert_eq!(tls.ca_certs.as_deref(), Some("/tmp/ca.pem"));
+    assert_eq!(tls.cert_reqs, 2);
+}
+
+#[test]
+fn serve_without_ssl_flags_has_no_tls() {
+    let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    assert!(config.tls.is_none());
+}
+
+#[test]
+fn serve_ssl_keyfile_without_certfile_fails_validation() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--ssl-keyfile",
+        "/tmp/key.pem",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    // TLS is requested (a key was given) but there is no certificate, so
+    // validation fails loud rather than silently serving plaintext.
+    assert_eq!(config.tls.as_ref().expect("tls requested").cert_file, None);
+    let err = config.validate().unwrap_err().to_string();
+    assert!(err.contains("--ssl-certfile is required"), "{err}");
+}
+
+#[test]
+fn serve_mtls_without_ca_certs_fails_validation() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--ssl-certfile",
+        "/tmp/cert.pem",
+        "--ssl-cert-reqs",
+        "2",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
+    // Client-cert verification without a CA bundle has nothing to verify
+    // against, so it fails loud at startup.
+    let err = config.validate().unwrap_err().to_string();
+    assert!(err.contains("--ssl-ca-certs is required"), "{err}");
+}
+
+#[test]
+fn frontend_args_json_passes_tls_into_config() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "frontend",
+        "--listen-fd",
+        "3",
+        "--input-address",
+        "ipc:///tmp/input.sock",
+        "--output-address",
+        "ipc:///tmp/output.sock",
+        "--args-json",
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_keyfile":"/tmp/key.pem"}"#,
+    ])
+    .unwrap();
+
+    let Command::Frontend(args) = cli.command else {
+        panic!("expected frontend args");
+    };
+    let config = args.into_config();
+    let tls = config.tls.expect("tls configured");
+    assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
+    assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
+}
+
+#[test]
+fn frontend_args_json_rejects_out_of_range_cert_reqs() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "frontend",
+        "--listen-fd",
+        "3",
+        "--input-address",
+        "ipc:///tmp/input.sock",
+        "--output-address",
+        "ipc:///tmp/output.sock",
+        "--args-json",
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_cert_reqs":5}"#,
+    ])
+    .unwrap();
+
+    let Command::Frontend(args) = cli.command else {
+        panic!("expected frontend args");
+    };
+    // The JSON path bypasses clap's range check, so validate() is the only guard.
+    let config = args.into_config();
+    let err = config.validate().unwrap_err().to_string();
+    assert!(err.contains("--ssl-cert-reqs"), "{err}");
+}
+
 #[test]
 fn frontend_args_json_passes_enable_request_id_headers_into_config() {
    let cli = Cli::try_parse_from([
@@ -481,13 +621,13 @@ fn serve_args_reject_unsupported_flag_arg() {
        "vllm-rs",
        "serve",
        "Qwen/Qwen3-0.6B",
-        "--ssl-keyfile",
-        "/tmp/key.pem",
+        "--root-path",
+        "/prefix",
    ])
    .unwrap_err();

    expect![[r#"
-        error: invalid value '/tmp/key.pem' for '--ssl-keyfile <SSL_KEYFILE>': argument is not implemented in Rust frontend yet
+        error: invalid value '/prefix' for '--root-path <ROOT_PATH>': argument is not implemented in Rust frontend yet

        Remove this unsupported argument to continue.

@@ -562,6 +702,7 @@ fn frontend_args_accept_json() {
                        max_logprobs: None,
                        grpc_port: None,
                        shutdown_timeout: 0,
+                        http_timeout_keep_alive: None,
                        chat_template: None,
                        default_chat_template_kwargs: None,
                        chat_template_content_format: Auto,
@@ -586,6 +727,11 @@ fn frontend_args_accept_json() {
                            ],
                        ),
                        allow_credentials: false,
+                        ssl_keyfile: None,
+                        ssl_certfile: None,
+                        ssl_ca_certs: None,
+                        ssl_cert_reqs: 0,
+                        ssl_ciphers: None,
                    },
                },
            ),
@@ -798,14 +944,14 @@ fn frontend_args_json_rejects_unsupported_fields() {
        "--output-address",
        "ipc:///tmp/output.sock",
        "--args-json",
-        r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}"#,
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}"#,
    ])
    .unwrap_err();

    expect![[r#"
-        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>': 
+        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}' for '--args-json <JSON>': 
        The following arguments are not implemented in Rust frontend yet:
-        - ssl_keyfile
+        - root_path

        Remove these arguments to continue.

@@ -825,16 +971,16 @@ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
        "--output-address",
        "ipc:///tmp/output.sock",
        "--args-json",
-        r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}"#,
+        r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}"#,
    ])
    .unwrap_err();

    let actual = error.to_string().replace(": \n", ":\n");
    expect![[r#"
-        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","ssl_keyfile":"/tmp/key.pem"}' for '--args-json <JSON>':
+        error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}' for '--args-json <JSON>':
        The following arguments are not implemented in Rust frontend yet:
        - response_role
-        - ssl_keyfile
+        - root_path

        Remove these arguments to continue.

@@ -1077,6 +1223,7 @@ fn serve_args_accept_handshake_aliases() {
                        max_logprobs: None,
                        grpc_port: None,
                        shutdown_timeout: 0,
+                        http_timeout_keep_alive: None,
                        chat_template: None,
                        default_chat_template_kwargs: None,
                        chat_template_content_format: Auto,
@@ -1101,6 +1248,11 @@ fn serve_args_accept_handshake_aliases() {
                            ],
                        ),
                        allow_credentials: false,
+                        ssl_keyfile: None,
+                        ssl_certfile: None,
+                        ssl_ca_certs: None,
+                        ssl_cert_reqs: 0,
+                        ssl_ciphers: None,
                    },
                    managed_engine: ManagedEngineArgs {
                        python: "python3",
@@ -1234,10 +1386,12 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
                ],
                allow_credentials: false,
            },
+            tls: None,
            api_keys: [],
            disable_log_stats: false,
            grpc_port: None,
            shutdown_timeout: 0ns,
+            keep_alive_timeout: 5s,
        }
    "#]]
    .assert_debug_eq(&Config {
@@ -1315,10 +1469,12 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
                ],
                allow_credentials: false,
            },
+            tls: None,
            api_keys: [],
            disable_log_stats: false,
            grpc_port: None,
            shutdown_timeout: 0ns,
+            keep_alive_timeout: 5s,
        }
    "#]]
    .assert_debug_eq(&config);
@@ -1414,10 +1570,12 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present
                ],
                allow_credentials: false,
            },
+            tls: None,
            api_keys: [],
            disable_log_stats: false,
            grpc_port: None,
            shutdown_timeout: 0ns,
+            keep_alive_timeout: 5s,
        }
    "#]]
    .assert_debug_eq(&config);
@@ -526,18 +526,6 @@ pub struct ServerUnsupportedArgs {
    #[arg(long)]
    pub disable_access_log_for_endpoints: Option<Noop>,

-    /// The file path to the SSL key file.
-    #[arg(long)]
-    pub ssl_keyfile: Option<Unsupported>,
-
-    /// The file path to the SSL cert file.
-    #[arg(long)]
-    pub ssl_certfile: Option<Unsupported>,
-
-    /// The CA certificates file.
-    #[arg(long)]
-    pub ssl_ca_certs: Option<Unsupported>,
-
    /// Refresh SSL Context when SSL certificate files change
    #[arg(
        long,
@@ -547,15 +535,6 @@ pub struct ServerUnsupportedArgs {
    )]
    pub enable_ssl_refresh: Option<Unsupported>,

-    /// Whether client certificate is required (see stdlib ssl module's).
-    #[arg(long)]
-    pub ssl_cert_reqs: Option<Unsupported>,
-
-    /// SSL cipher suites for HTTPS (TLS 1.2 and below only).
-    /// Example: 'ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305'
-    #[arg(long)]
-    pub ssl_ciphers: Option<Unsupported>,
-
    /// FastAPI root_path when app is behind a path based routing proxy.
    #[arg(long)]
    pub root_path: Option<Unsupported>,
@@ -100,6 +100,7 @@ impl EngineRoutingState {
 pub struct RequestRegistry {
    closed: bool,
    requests: HashMap<String, TrackedRequest>,
+    active_lora_requests: usize,
    routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
 }

@@ -108,6 +109,7 @@ impl RequestRegistry {
        Self {
            closed: false,
            requests: HashMap::default(),
+            active_lora_requests: 0,
            routing_per_engine: engines
                .iter()
                .map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
@@ -133,15 +135,19 @@ impl RequestRegistry {

        let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
        let (tx, rx) = mpsc::unbounded_channel();
+        let lora = lora_name.map(|adapter_name| LoraRequestState {
+            adapter_name,
+            phase: LoraPhase::Waiting,
+        });
+        if lora.is_some() {
+            self.active_lora_requests += 1;
+        }
        self.requests.insert(
            request_id,
            TrackedRequest {
                sender: tx,
                engine_id: engine_id.clone(),
-                lora: lora_name.map(|adapter_name| LoraRequestState {
-                    adapter_name,
-                    phase: LoraPhase::Waiting,
-                }),
+                lora,
            },
        );

@@ -230,6 +236,10 @@ impl RequestRegistry {
    /// Snapshot the adapter names of tracked LoRA requests as
    /// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
    pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
+        if self.active_lora_requests == 0 {
+            return (BTreeSet::new(), BTreeSet::new());
+        }
+
        let mut running = BTreeSet::new();
        let mut waiting = BTreeSet::new();
        for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
@@ -283,6 +293,7 @@ impl RequestRegistry {
        }

        self.closed = true;
+        self.active_lora_requests = 0;
        std::mem::take(&mut self.requests)
            .into_values()
            .map(|tracked| tracked.sender)
@@ -322,6 +333,9 @@ impl RequestRegistry {
    #[must_use]
    pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
        let tracked = self.requests.remove(request_id)?;
+        if tracked.lora.is_some() {
+            self.active_lora_requests -= 1;
+        }
        self.routing_per_engine
            .get_mut(&tracked.engine_id)
            .expect("request registry must track all known engines")
@@ -359,6 +373,11 @@ impl RequestRegistry {
    pub fn is_closed(&self) -> bool {
        self.closed
    }
+
+    #[cfg(test)]
+    fn active_lora_requests(&self) -> usize {
+        self.active_lora_requests
+    }
 }

 /// Internal registry for tracking active utility calls and their waiting
@@ -574,6 +593,63 @@ mod tests {
        );
    }

+    #[test]
+    fn registry_counts_only_active_lora_requests() {
+        let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
+
+        registry.register("req-plain".to_string(), None, None).unwrap();
+        assert_eq!(registry.active_lora_requests(), 0);
+        assert_eq!(
+            registry.lora_adapter_states(),
+            (adapter_names(&[]), adapter_names(&[]))
+        );
+
+        registry
+            .register(
+                "req-lora-a".to_string(),
+                Some("adapter-a".to_string()),
+                None,
+            )
+            .unwrap();
+        registry
+            .register(
+                "req-lora-b".to_string(),
+                Some("adapter-b".to_string()),
+                None,
+            )
+            .unwrap();
+        assert_eq!(registry.active_lora_requests(), 2);
+
+        drop(registry.remove("req-plain"));
+        assert_eq!(registry.active_lora_requests(), 2);
+
+        drop(registry.finish_many(&["req-lora-a".to_string()]));
+        assert_eq!(registry.active_lora_requests(), 1);
+
+        drop(registry.abort_many(&["req-lora-b".to_string()], 0.0));
+        assert_eq!(registry.active_lora_requests(), 0);
+        assert_eq!(
+            registry.lora_adapter_states(),
+            (adapter_names(&[]), adapter_names(&[]))
+        );
+    }
+
+    #[test]
+    fn registry_clears_lora_count_on_close() {
+        let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
+        registry
+            .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
+            .unwrap();
+
+        assert_eq!(registry.active_lora_requests(), 1);
+        drop(registry.close());
+        assert_eq!(registry.active_lora_requests(), 0);
+        assert_eq!(
+            registry.lora_adapter_states(),
+            (adapter_names(&[]), adapter_names(&[]))
+        );
+    }
+
    #[test]
    fn registry_drops_lora_tracking_on_abort() {
        let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
@@ -20,6 +20,27 @@ pub(crate) struct CoordinatorStateSnapshot {
    pub engines_running: bool,
 }

+impl CoordinatorStateSnapshot {
+    /// Resume the engines for a `FirstRequest` and return the wave to broadcast
+    /// and the engine to exclude from the wakeup.
+    ///
+    /// The request may have been stamped with a `request_wave` older than
+    /// `current_wave` if a `WaveComplete` advanced it after the command was
+    /// enqueued. Such a request still needs serving, so the current wave is
+    /// broadcast to every engine (`exclude = None`); the wave is never rewound.
+    /// A non-stale request excludes the engine that already received it. Mirrors
+    /// the Python coordinator's front-end path.
+    pub(crate) fn start_wave_for_first_request(
+        &mut self,
+        request_wave: u32,
+        target_engine_index: u32,
+    ) -> (u32, Option<u32>) {
+        self.engines_running = true;
+        let exclude = (request_wave >= self.current_wave).then_some(target_engine_index);
+        (self.current_wave, exclude)
+    }
+}
+
 /// Shared in-process coordinator state.
 pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;

@@ -27,9 +27,10 @@ use crate::protocol::{
 struct StartDpWaveMessage {
    /// DP wave number that all engines should start processing.
    wave: u32,
-    /// Engine index that already received the triggering request and should not
-    /// receive an extra wakeup notification.
-    exclude_engine_index: u32,
+    /// Engine index that already received the triggering request and so does not
+    /// need an extra wakeup. `None` wakes every engine (used when the triggering
+    /// request was for a stale wave).
+    exclude_engine_index: Option<u32>,
 }

 /// Background half of the in-process coordinator.
@@ -57,7 +58,11 @@ impl InProcCoordinatorRunner {
    }

    /// Broadcast Python-compatible `START_DP_WAVE` to all connected engines.
-    async fn broadcast_start_wave(&mut self, wave: u32, exclude_engine_index: u32) -> Result<()> {
+    async fn broadcast_start_wave(
+        &mut self,
+        wave: u32,
+        exclude_engine_index: Option<u32>,
+    ) -> Result<()> {
        let payload = encode_msgpack(&StartDpWaveMessage {
            wave,
            exclude_engine_index,
@@ -86,13 +91,17 @@ impl InProcCoordinatorRunner {
                        engine_id: target_engine_id.to_vec(),
                    }
                })?;
-                self.state.lock().current_wave = wave;
+                let (current_wave, exclude) = {
+                    let mut state = self.state.lock();
+                    state.start_wave_for_first_request(wave, target_engine_index)
+                };
                debug!(
-                    wave,
-                    exclude_engine_index = target_engine_index,
+                    current_wave,
+                    request_wave = wave,
+                    ?exclude,
                    "starting DP wave after first request while engines were paused"
                );
-                self.broadcast_start_wave(wave, target_engine_index).await?;
+                self.broadcast_start_wave(current_wave, exclude).await?;
            }
        }
        Ok(())
@@ -150,7 +159,7 @@ impl InProcCoordinatorRunner {
                            exclude_engine_index = engine_index,
                            "starting DP wave after stale-wave notification from engine"
                        );
-                        self.broadcast_start_wave(wave, engine_index).await?;
+                        self.broadcast_start_wave(wave, Some(engine_index)).await?;
                    }
                }
            },
@@ -202,3 +211,48 @@ impl InProcCoordinatorRunner {
        inner.close_registries(Arc::new(error));
    }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::coordinator::handle::CoordinatorStateSnapshot;
+
+    /// A `FirstRequest` for the current wave starts that wave and excludes the
+    /// engine that already received the triggering request.
+    #[test]
+    fn first_request_for_current_wave_excludes_target() {
+        let mut state = CoordinatorStateSnapshot {
+            current_wave: 3,
+            engines_running: false,
+        };
+
+        let (wave, exclude) = state.start_wave_for_first_request(3, 2);
+
+        assert_eq!(wave, 3);
+        assert_eq!(exclude, Some(2));
+        assert!(state.engines_running);
+        assert_eq!(state.current_wave, 3);
+    }
+
+    /// A `FirstRequest` whose wave was superseded by a racing `WaveComplete`
+    /// (`request_wave < current_wave`) must still start the request's wave: it
+    /// broadcasts the current wave and wakes every engine (`exclude = None`)
+    /// rather than rewinding the wave or dropping the request.
+    #[test]
+    fn stale_first_request_starts_current_wave_for_all_engines() {
+        let mut state = CoordinatorStateSnapshot {
+            current_wave: 4,
+            engines_running: false,
+        };
+
+        // Request stamped with wave 3 while the coordinator already advanced to 4.
+        let (wave, exclude) = state.start_wave_for_first_request(3, 2);
+
+        assert_eq!(
+            wave, 4,
+            "must broadcast the current wave, not the stale one"
+        );
+        assert_eq!(exclude, None, "a stale request must wake every engine");
+        assert!(state.engines_running);
+        assert_eq!(state.current_wave, 4, "wave must not be rewound");
+    }
+}
@@ -1285,18 +1285,24 @@ async fn dropping_multiple_live_streams_aborts_all_in_a_burst() {
                )
                .await;

-                let abort =
-                    timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
-                assert_eq!(abort[0].as_ref(), &[0x01]);
-                let ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
+                // Aborts may coalesce into one burst or split across several.
+                let mut aborted = BTreeSet::new();
+                while aborted.len() < 3 {
+                    let abort =
+                        timeout(Duration::from_secs(1), recv_engine_message(dealer)).await.unwrap();
+                    assert_eq!(abort[0].as_ref(), &[0x01]);
+                    let ids: Vec<String> = rmp_serde::from_slice(&abort[1]).unwrap();
+                    aborted.extend(ids);
+                }
                assert_eq!(
-                    ids,
-                    vec![
+                    aborted,
+                    BTreeSet::from([
                        "req-1".to_string(),
                        "req-2".to_string(),
                        "req-3".to_string()
-                    ]
+                    ])
                );
+                // No spurious extra aborts.
                assert!(
                    timeout(Duration::from_millis(100), recv_engine_message(dealer)).await.is_err()
                );
@@ -154,7 +154,7 @@ mod tests {
                "flag": true,
                "payload": { "nested": true },
                "items": [1, 2],
-                "empty": null,
+                "empty": "null",
            })
        );
    }
@@ -166,7 +166,13 @@ impl JsonParamType {

        // Typically, these types are already handled by checking the "type" field, but
        // we can also infer them from their characteristic fields if "type" is missing.
-        if schema.contains_key("enum") {
+        if let Some(values) = schema.get("enum").and_then(Value::as_array) {
+            // Enum values are treated as strings, except that a `null` member
+            // makes the parameter nullable (mirrors Python's enum type
+            // inference), so a literal "null" coerces to JSON null.
+            if values.iter().any(Value::is_null) {
+                return Some(Self::one_of(vec![Self::String, Self::Null]));
+            }
            return Some(Self::String);
        }
        if schema.contains_key("items") {
@@ -277,9 +283,12 @@ impl JsonParamType {

 /// Convert one parameter input to a normalized JSON value.
 fn convert_with_optional_schema(param_type: Option<&JsonParamType>, input: &ParamInput) -> Value {
-    // For literal `null`, always convert to JSON null value.
+    // Coerce the literal text `null` to JSON null, except for `string`-typed
+    // params, where it must stay the string "null": a model emitting the literal
+    // text "null" for a string field means the string, not a missing value.
    if let ParamInput::Text(value) = input
        && value.eq_ignore_ascii_case("null")
+        && param_type != Some(&JsonParamType::String)
    {
        return Value::Null;
    }
@@ -685,21 +694,43 @@ mod tests {
    }

    #[test]
-    fn convert_params_preserves_null_for_known_param() {
-        let schemas = ToolSchemas::from_tools(&[test_tool(
-            "convert",
-            json!({
-                "type": "object",
-                "properties": {
-                    "value": { "type": "string" }
-                }
-            }),
-        )]);
+    fn string_param_preserves_literal_null_text() {
+        // A `string`-typed param whose value is the literal text "null"/"NULL"
+        // must stay a string (the original case is preserved), rather than being
+        // coerced to JSON null. Non-string types keep coercing "null" to null.
+        let params = ToolSchema::from_schema(&json!({
+            "type": "object",
+            "properties": {
+                "name": { "type": "string" },
+                "count": { "type": "integer" },
+                "anything": {}
+            }
+        }));

-        let converted = schemas
-            .convert_params_with_schema("convert", vec![("value".to_string(), "NULL".to_string())]);
+        assert_eq!(params.convert("name", text("null")), json!("null"));
+        assert_eq!(params.convert("name", text("NULL")), json!("NULL"));
+        // Non-string and schema-less params are unchanged: "null" -> null.
+        assert_eq!(params.convert("count", text("null")), json!(null));
+        assert_eq!(params.convert("anything", text("null")), json!(null));
+    }

-        assert_eq!(converted.get("value"), Some(&json!(null)));
+    #[test]
+    fn nullable_enum_param_coerces_literal_null() {
+        // An enum that includes `null` admits a null value, so a literal "null"
+        // must coerce to JSON null (matching Python's `extract_types_from_schema`,
+        // which infers `null` from the enum values), while a non-null enum keeps
+        // "null" as a string.
+        let params = ToolSchema::from_schema(&json!({
+            "type": "object",
+            "properties": {
+                "mode": { "enum": [null, "auto"] },
+                "color": { "enum": ["red", "green"] }
+            }
+        }));
+
+        assert_eq!(params.convert("mode", text("null")), json!(null));
+        assert_eq!(params.convert("mode", text("auto")), json!("auto"));
+        assert_eq!(params.convert("color", text("null")), json!("null"));
    }

    #[test]
@@ -841,7 +872,7 @@ mod tests {
                "user_id": 42,
                "urgent": true,
                "note": "Please leave at front desk.",
-                "nil": null,
+                "nil": "NULL",
                "shipping": {
                    "city": "Singapore",
                    "zip": 18956
@@ -7,14 +7,18 @@ license.workspace = true
 [dependencies]
 anyhow.workspace = true
 asynk-strim-attr.workspace = true
+auto_enums.workspace = true
 axum.workspace = true
 educe.workspace = true
 futures.workspace = true
 http-body.workspace = true
+hyper.workspace = true
+hyper-util.workspace = true
 indexmap.workspace = true
 itertools.workspace = true
 libc.workspace = true
 llm-multimodal.workspace = true
+openssl.workspace = true
 prost.workspace = true
 prost-types.workspace = true
 rmpv.workspace = true
@@ -25,7 +29,9 @@ sha2.workspace = true
 socket2.workspace = true
 subtle.workspace = true
 thiserror-ext.workspace = true
+tls-listener.workspace = true
 tokio.workspace = true
+tokio-openssl.workspace = true
 tokio-stream.workspace = true
 tokio-util.workspace = true
 tonic.workspace = true
@@ -54,6 +60,7 @@ clap.workspace = true
 expect-test.workspace = true
 rmp-serde.workspace = true
 serial_test.workspace = true
+tempfile.workspace = true
 tower.workspace = true
 vllm-engine-core-client = { workspace = true, features = ["test-util"] }
 zeromq.workspace = true
@@ -71,10 +71,12 @@ async fn main() -> Result<()> {
        max_logprobs: None,
        api_server_options: ApiServerOptions::default(),
        cors: CorsConfig::default(),
+        tls: None,
        api_keys: Vec::new(),
        disable_log_stats: false,
        grpc_port: None,
        shutdown_timeout: Duration::ZERO,
+        keep_alive_timeout: Duration::from_secs(5),
    };

    let bind_address = format!("127.0.0.1:{port}");
@@ -10,6 +10,10 @@ use serde_json::Value;
 use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
 use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode};

+/// Default keep-alive idle timeout (seconds); also the head-read bound
+/// when keep-alive is disabled (`0`).
+pub const DEFAULT_KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
+
 /// How the HTTP server obtains its listening socket.
 #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
 pub enum HttpListenerMode {
@@ -99,6 +103,54 @@ impl CorsConfig {
    }
 }

+/// TLS settings mirroring Python's uvicorn `ssl_*` arguments.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct TlsConfig {
+    /// PEM certificate chain file. Required when TLS is configured; may also
+    /// hold the private key (combined PEM) when `key_file` is unset.
+    pub cert_file: Option<String>,
+    /// PEM private key file. When `None`, the key is read from `cert_file`
+    /// (combined PEM).
+    pub key_file: Option<String>,
+    /// PEM CA bundle used to verify client certificates (mTLS). Required when
+    /// `cert_reqs` is non-zero.
+    pub ca_certs: Option<String>,
+    /// Client-certificate requirement, mirroring Python's `ssl.CERT_*`:
+    /// 0 = none, 1 = optional, 2 = required.
+    pub cert_reqs: i32,
+    /// OpenSSL cipher string for TLS 1.2 and below, mirroring Python's
+    /// `ssl.set_ciphers`. `None` keeps the forward-secret AEAD default.
+    pub ciphers: Option<String>,
+}
+
+impl TlsConfig {
+    /// Structurally validate the TLS arguments; the cert/key material is parsed
+    /// later, when the OpenSSL context is built.
+    pub fn validate(&self) -> Result<()> {
+        if self.cert_file.is_none() {
+            bail!(
+                "--ssl-certfile is required to enable TLS; \
+                 --ssl-keyfile/--ssl-ca-certs/--ssl-cert-reqs/--ssl-ciphers \
+                 cannot be used without it"
+            );
+        }
+        if !matches!(self.cert_reqs, 0..=2) {
+            bail!(
+                "--ssl-cert-reqs must be 0 (none), 1 (optional), or 2 (required), got {}",
+                self.cert_reqs
+            );
+        }
+        if self.cert_reqs != 0 && self.ca_certs.is_none() {
+            bail!(
+                "--ssl-ca-certs is required when --ssl-cert-reqs is {} \
+                 (client certificate verification)",
+                self.cert_reqs
+            );
+        }
+        Ok(())
+    }
+}
+
 /// Normalized runtime configuration for the minimal OpenAI-compatible server.
 #[derive(Educe, Clone, PartialEq, Eq, Serialize)]
 #[educe(Debug)]
@@ -138,6 +190,9 @@ pub struct Config {
    pub api_server_options: ApiServerOptions,
    /// CORS settings applied to every HTTP response.
    pub cors: CorsConfig,
+    /// TLS settings. `None` serves plaintext HTTP; `Some` terminates TLS at the
+    /// listener.
+    pub tls: Option<TlsConfig>,
    /// API keys accepted as bearer tokens for guarded routes.
    #[serde(skip_serializing)]
    #[educe(Debug(method(fmt_redacted_api_keys)))]
@@ -150,6 +205,9 @@ pub struct Config {
    pub grpc_port: Option<u16>,
    /// Maximum time to wait for active HTTP/gRPC requests to drain on shutdown.
    pub shutdown_timeout: Duration,
+    /// Maximum idle time on a keep-alive HTTP connection before the server
+    /// closes it (`VLLM_HTTP_TIMEOUT_KEEP_ALIVE`, default 5s).
+    pub keep_alive_timeout: Duration,
 }

 impl Config {
@@ -158,6 +216,9 @@ impl Config {
    pub fn validate(&self) -> Result<()> {
        vllm_chat::validate_parser_overrides(&self.tool_call_parser, &self.reasoning_parser)?;
        self.cors.validate()?;
+        if let Some(tls) = &self.tls {
+            tls.validate()?;
+        }
        if let Some(max_logprobs) = self.max_logprobs
            && max_logprobs < -1
        {
@@ -4,16 +4,21 @@ mod convert;

 use std::pin::Pin;
 use std::sync::Arc;
+use std::task::{Context, Poll};

-use futures::{Stream, StreamExt as _};
+use futures::{Stream, StreamExt as _, stream};
 use thiserror_ext::AsReport as _;
+use tokio::io::{AsyncRead, AsyncWrite, ReadBuf};
 use tokio::sync::mpsc;
+use tokio_openssl::SslStream;
 use tokio_stream::wrappers::ReceiverStream;
+use tonic::transport::server::{Connected, TcpConnectInfo};
 use tonic::{Request, Response, Status};
 use tracing::info;
 use vllm_text::{DecodedTextEvent, TextOutputStreamExt as _};

 use self::convert::ResponseOpts;
+use crate::listener::{Listener, ListenerIo};
 use crate::state::AppState;

 /// Generated protobuf/gRPC types for the `vllm` package.
@@ -26,6 +31,78 @@ pub use pb::generate_server::GenerateServer;
 #[cfg(test)]
 mod tests;

+/// Newtype over `tokio-openssl`'s `SslStream` so we can implement tonic's
+/// [`Connected`] on it (the orphan rule blocks doing so on the foreign type).
+pub(crate) struct GrpcTlsStream {
+    inner: SslStream<ListenerIo>,
+}
+
+impl GrpcTlsStream {
+    pub(crate) fn new(inner: SslStream<ListenerIo>) -> Self {
+        Self { inner }
+    }
+}
+
+impl AsyncRead for GrpcTlsStream {
+    fn poll_read(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &mut ReadBuf<'_>,
+    ) -> Poll<std::io::Result<()>> {
+        Pin::new(&mut self.get_mut().inner).poll_read(cx, buf)
+    }
+}
+
+impl AsyncWrite for GrpcTlsStream {
+    fn poll_write(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+        buf: &[u8],
+    ) -> Poll<std::io::Result<usize>> {
+        Pin::new(&mut self.get_mut().inner).poll_write(cx, buf)
+    }
+
+    fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+        Pin::new(&mut self.get_mut().inner).poll_flush(cx)
+    }
+
+    fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
+        Pin::new(&mut self.get_mut().inner).poll_shutdown(cx)
+    }
+}
+
+impl Connected for GrpcTlsStream {
+    type ConnectInfo = TcpConnectInfo;
+
+    fn connect_info(&self) -> TcpConnectInfo {
+        self.inner.get_ref().connect_info()
+    }
+}
+
+/// Adapt the shared server listener into tonic's incoming stream shape.
+pub(crate) fn incoming(listener: Listener) -> impl Stream<Item = std::io::Result<ListenerIo>> {
+    stream::unfold(listener, |mut listener| async move {
+        let (io, _) = axum::serve::Listener::accept(&mut listener).await;
+        Some((Ok(io), listener))
+    })
+}
+
+/// Wrap the gRPC listener so each accepted connection completes a TLS handshake
+/// before tonic serves it.
+pub(crate) fn tls_incoming(
+    listener: Listener,
+    context: openssl::ssl::SslContext,
+    handshake_timeout: std::time::Duration,
+) -> impl Stream<Item = std::io::Result<GrpcTlsStream>> {
+    tls_listener::builder(context)
+        .handshake_timeout(handshake_timeout)
+        .listen(listener)
+        .map(|res| {
+            res.map(|(inner, _addr)| GrpcTlsStream::new(inner))
+                .map_err(std::io::Error::other)
+        })
+}
+
 /// gRPC Generate service implementation backed by the shared application state.
 pub struct GenerateServiceImpl {
    state: Arc<AppState>,
@@ -1,11 +1,19 @@
 use std::future::Future;
+use std::io;
 use std::pin::Pin;
 use std::sync::Arc;
 use std::task::{Context, Poll};
+use std::time::Duration;

 use futures::StreamExt as _;
+use hyper_util::rt::TokioIo;
+use openssl::ssl::{SslConnector, SslFiletype, SslMethod};
 use serial_test::serial;
-use tonic::transport::Server as TonicServer;
+use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+use tokio::net::TcpStream;
+use tokio_openssl::SslStream;
+use tonic::transport::{Channel, Endpoint, Server as TonicServer, Uri};
+use tower::service_fn;
 use vllm_chat::{
    ChatBackend, ChatLlm, ChatRenderer, ChatRequest, ChatTextBackend, DefaultChatOutputProcessor,
    DynChatOutputProcessor, DynChatRenderer, NewChatOutputProcessorOptions, RenderedPrompt,
@@ -22,8 +30,11 @@ use zeromq::prelude::{SocketRecv, SocketSend};
 use zeromq::{DealerSocket, PushSocket, ZmqMessage};

 use super::pb::generate_client::GenerateClient;
-use super::{GenerateServer, GenerateServiceImpl, pb};
+use super::{GenerateServer, GenerateServiceImpl, incoming, pb, tls_incoming};
+use crate::listener::Listener;
 use crate::state::AppState;
+use crate::tls;
+use crate::tls_tests::{TestCerts, server_tls};

 // ========================================================================================
 // Helpers (mirrors the patterns in routes/tests.rs)
@@ -211,17 +222,12 @@ impl ChatRenderer for FakeTextBackend {
    }
 }

-/// Spin up a gRPC server backed by a mock engine that serves a single request
-/// with the given output specs. Returns the client, the gRPC server task, and
-/// the mock engine task.
-async fn grpc_test_server(
+/// Build the gRPC service + mock engine that serves a single request with the
+/// given output specs. Shared by the plaintext and TLS server fixtures.
+async fn setup_grpc_service(
    engine_id: impl Into<EngineId>,
    output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
-) -> (
-    GenerateClient<tonic::transport::Channel>,
-    tokio::task::JoinHandle<()>,
-    MockEngineTask,
-) {
+) -> (GenerateServer<GenerateServiceImpl>, MockEngineTask) {
    let ipc = IpcNamespace::new().expect("create ipc namespace");
    let handshake_address = ipc.handshake_endpoint();
    let engine_id = engine_id.into();
@@ -259,14 +265,29 @@ async fn grpc_test_server(
        Arc::new(FakeTextBackend) as Arc<dyn ChatTextBackend>,
    );
    let state = Arc::new(AppState::new(vec!["test-model".to_string()], chat));
-    let svc = GenerateServer::new(GenerateServiceImpl::new(state));
+    (
+        GenerateServer::new(GenerateServiceImpl::new(state)),
+        engine_task,
+    )
+}
+
+/// Spin up a plaintext gRPC server backed by a mock engine. Returns the client,
+/// the gRPC server task, and the mock engine task.
+async fn grpc_test_server(
+    engine_id: impl Into<EngineId>,
+    output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
+) -> (
+    GenerateClient<tonic::transport::Channel>,
+    tokio::task::JoinHandle<()>,
+    MockEngineTask,
+) {
+    let (svc, engine_task) = setup_grpc_service(engine_id, output_specs).await;

-    // Bind to an OS-assigned port.
    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
    let addr = listener.local_addr().expect("local addr");

    let server_task = tokio::spawn(async move {
-        let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener);
+        let incoming = incoming(Listener::Tcp(listener));
        TonicServer::builder()
            .add_service(svc)
            .serve_with_incoming(incoming)
@@ -274,7 +295,6 @@ async fn grpc_test_server(
            .expect("grpc server");
    });

-    // Connect the client.
    let grpc_client = GenerateClient::connect(format!("http://{addr}"))
        .await
        .expect("connect grpc client");
@@ -282,6 +302,158 @@ async fn grpc_test_server(
    (grpc_client, server_task, engine_task)
 }

+/// Spin up a TLS gRPC server (server cert from `certs`, `cert_reqs` mTLS mode).
+/// Returns the address, the server task, and the mock engine task.
+async fn grpc_tls_test_server(
+    engine_id: impl Into<EngineId>,
+    output_specs: Vec<(Vec<u32>, Option<EngineCoreFinishReason>)>,
+    certs: &TestCerts,
+    cert_reqs: i32,
+) -> (String, tokio::task::JoinHandle<()>, MockEngineTask) {
+    let (svc, engine_task) = setup_grpc_service(engine_id, output_specs).await;
+    let context = tls::build_grpc_server_config(&server_tls(certs, cert_reqs))
+        .expect("build grpc tls config");
+
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
+    let addr = listener.local_addr().expect("local addr").to_string();
+
+    let server_task = tokio::spawn(async move {
+        let incoming = tls_incoming(Listener::Tcp(listener), context, tls::TLS_HANDSHAKE_TIMEOUT);
+        TonicServer::builder()
+            .add_service(svc)
+            .serve_with_incoming(incoming)
+            .await
+            .expect("grpc tls server");
+    });
+
+    (addr, server_task, engine_task)
+}
+
+/// Build a tonic `Generate` client over a tokio-openssl connector, optionally
+/// with a client identity for mTLS. Hand-rolled because tonic 0.14 ships no
+/// OpenSSL transport.
+async fn grpc_tls_client(
+    certs: &TestCerts,
+    addr: &str,
+    identity: Option<&str>,
+) -> Result<GenerateClient<Channel>, tonic::transport::Error> {
+    let ca = certs.path("ca.pem");
+    let identity = identity.map(|name| {
+        (
+            certs.path(&format!("{name}.pem")),
+            certs.path(&format!("{name}.key")),
+        )
+    });
+    let target = addr.to_string();
+
+    let connector = service_fn(move |_: Uri| {
+        let ca = ca.clone();
+        let identity = identity.clone();
+        let target = target.clone();
+        async move {
+            let tcp = TcpStream::connect(&target).await?;
+            let mut builder =
+                SslConnector::builder(SslMethod::tls_client()).map_err(io::Error::other)?;
+            builder.set_ca_file(&ca).map_err(io::Error::other)?;
+            if let Some((cert, key)) = &identity {
+                builder.set_certificate_chain_file(cert).map_err(io::Error::other)?;
+                builder.set_private_key_file(key, SslFiletype::PEM).map_err(io::Error::other)?;
+            }
+            let mut config = builder.build().configure().map_err(io::Error::other)?;
+            config.set_verify_hostname(false);
+            config.set_alpn_protos(b"\x02h2").map_err(io::Error::other)?;
+            let ssl = config.into_ssl("127.0.0.1").map_err(io::Error::other)?;
+            let mut stream = SslStream::new(ssl, tcp).map_err(io::Error::other)?;
+            Pin::new(&mut stream).connect().await.map_err(io::Error::other)?;
+            Ok::<_, io::Error>(TokioIo::new(stream))
+        }
+    });
+
+    let channel = Endpoint::from_shared(format!("https://{addr}"))
+        .expect("grpc endpoint")
+        .connect_with_connector(connector)
+        .await?;
+    Ok(GenerateClient::new(channel))
+}
+
+/// Complete a raw TLS handshake against the gRPC port (offering ALPN `h2`) for
+/// the ALPN-negotiation assertion.
+async fn grpc_tls_handshake(
+    certs: &TestCerts,
+    addr: &str,
+) -> io::Result<Pin<Box<SslStream<TcpStream>>>> {
+    let tcp = TcpStream::connect(addr).await?;
+    let mut builder = SslConnector::builder(SslMethod::tls_client()).map_err(io::Error::other)?;
+    builder.set_ca_file(certs.path("ca.pem")).map_err(io::Error::other)?;
+    let mut config = builder.build().configure().map_err(io::Error::other)?;
+    config.set_verify_hostname(false);
+    config.set_alpn_protos(b"\x02h2").map_err(io::Error::other)?;
+    let ssl = config.into_ssl("127.0.0.1").map_err(io::Error::other)?;
+    let mut stream = Box::pin(SslStream::new(ssl, tcp).map_err(io::Error::other)?);
+    stream.as_mut().connect().await.map_err(io::Error::other)?;
+    Ok(stream)
+}
+
+/// Spin up a plaintext gRPC server, optionally with HTTP/2 keepalive set to
+/// `keepalive` for both the PING interval and the unanswered-PING timeout.
+async fn grpc_server_with_keepalive(
+    engine_id: impl Into<EngineId>,
+    keepalive: Option<Duration>,
+) -> (String, tokio::task::JoinHandle<()>, MockEngineTask) {
+    let (svc, engine_task) = setup_grpc_service(engine_id, default_stream_output_specs()).await;
+
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("bind grpc listener");
+    let addr = listener.local_addr().expect("local addr").to_string();
+
+    let mut builder = TonicServer::builder();
+    if let Some(interval) = keepalive {
+        builder = builder
+            .http2_keepalive_interval(Some(interval))
+            .http2_keepalive_timeout(Some(interval));
+    }
+
+    let server_task = tokio::spawn(async move {
+        let incoming = incoming(Listener::Tcp(listener));
+        builder
+            .add_service(svc)
+            .serve_with_incoming(incoming)
+            .await
+            .expect("grpc server");
+    });
+
+    (addr, server_task, engine_task)
+}
+
+/// Establish an HTTP/2 connection (preface + SETTINGS exchange) then go silent,
+/// ACKing the server's SETTINGS but never its keepalive PINGs. Returns whether
+/// the SERVER closes the connection within `wait`. A minimal hand-rolled h2 peer
+/// because a real client auto-ACKs PINGs and so can never be kept-alive-evicted.
+async fn h2_unresponsive_peer_closed_within(addr: &str, wait: Duration) -> bool {
+    let mut tcp = TcpStream::connect(addr).await.expect("connect");
+    tcp.write_all(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n").await.expect("preface");
+    tcp.write_all(&[0, 0, 0, 0x4, 0, 0, 0, 0, 0]).await.expect("client settings");
+
+    let closed = tokio::time::timeout(wait, async {
+        let mut header = [0u8; 9];
+        while tcp.read_exact(&mut header).await.is_ok() {
+            let len = u32::from_be_bytes([0, header[0], header[1], header[2]]) as usize;
+            let frame_type = header[3];
+            let flags = header[4];
+            let mut payload = vec![0u8; len];
+            if tcp.read_exact(&mut payload).await.is_err() {
+                return;
+            }
+            // ACK the server's SETTINGS so the only thing left unanswered is PINGs.
+            if frame_type == 0x4 && flags & 0x1 == 0 {
+                let _ = tcp.write_all(&[0, 0, 0, 0x4, 0x1, 0, 0, 0, 0]).await;
+            }
+        }
+    })
+    .await;
+
+    closed.is_ok()
+}
+
 // ========================================================================================
 // Tests
 // ========================================================================================
@@ -720,3 +892,173 @@ async fn unary_generate_output_text_defaults_to_true() {
    engine_task.await.expect("mock engine task");
    server_task.abort();
 }
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn grpc_generate_succeeds_over_tls() {
+    let certs = TestCerts::generate();
+    let (addr, server_task, engine_task) = grpc_tls_test_server(
+        b"engine-grpc-tls-unary",
+        default_stream_output_specs(),
+        &certs,
+        0,
+    )
+    .await;
+
+    let mut client = grpc_tls_client(&certs, &addr, None).await.expect("tls client");
+    let response = client
+        .generate(pb::GenerateRequest {
+            request_id: "test-tls-unary".to_string(),
+            model: "test-model".to_string(),
+            prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
+            stopping: Some(pb::StoppingCriteria {
+                max_new_tokens: 10,
+                ..Default::default()
+            }),
+            response: Some(pb::ResponseOptions {
+                output_text: Some(true),
+                ..Default::default()
+            }),
+            ..Default::default()
+        })
+        .await
+        .expect("unary generate over tls")
+        .into_inner();
+
+    assert_eq!(response.outputs.expect("outputs present").text, "hi");
+
+    engine_task.await.expect("mock engine task");
+    server_task.abort();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn grpc_tls_negotiates_h2_alpn() {
+    let certs = TestCerts::generate();
+    let (addr, server_task, _engine_task) = grpc_tls_test_server(
+        b"engine-grpc-tls-alpn",
+        default_stream_output_specs(),
+        &certs,
+        0,
+    )
+    .await;
+
+    let stream = grpc_tls_handshake(&certs, &addr).await.expect("handshake");
+    assert_eq!(
+        stream.ssl().selected_alpn_protocol(),
+        Some(&b"h2"[..]),
+        "server must negotiate h2 ALPN"
+    );
+
+    server_task.abort();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn grpc_mtls_required_rejects_client_without_certificate() {
+    let certs = TestCerts::generate();
+    let (addr, server_task, _engine_task) = grpc_tls_test_server(
+        b"engine-grpc-tls-mtls-reject",
+        default_stream_output_specs(),
+        &certs,
+        2,
+    )
+    .await;
+
+    // With TLS 1.3 the missing-client-cert rejection surfaces on first use, not
+    // at the handshake, so drive an RPC and assert the call fails.
+    let outcome = match grpc_tls_client(&certs, &addr, None).await {
+        Err(_) => Err(()),
+        Ok(mut client) => client
+            .generate(pb::GenerateRequest {
+                request_id: "test-tls-mtls-reject".to_string(),
+                model: "test-model".to_string(),
+                prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
+                stopping: Some(pb::StoppingCriteria {
+                    max_new_tokens: 10,
+                    ..Default::default()
+                }),
+                ..Default::default()
+            })
+            .await
+            .map(|_| ())
+            .map_err(|_| ()),
+    };
+    assert!(
+        outcome.is_err(),
+        "mTLS-required gRPC must reject a client without a certificate"
+    );
+
+    server_task.abort();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn grpc_mtls_required_accepts_valid_client_certificate() {
+    let certs = TestCerts::generate();
+    let (addr, server_task, engine_task) = grpc_tls_test_server(
+        b"engine-grpc-tls-mtls-accept",
+        default_stream_output_specs(),
+        &certs,
+        2,
+    )
+    .await;
+
+    let mut client = grpc_tls_client(&certs, &addr, Some("client")).await.expect("mtls client");
+    let response = client
+        .generate(pb::GenerateRequest {
+            request_id: "test-tls-mtls".to_string(),
+            model: "test-model".to_string(),
+            prompt: Some(pb::generate_request::Prompt::Text("hello".to_string())),
+            stopping: Some(pb::StoppingCriteria {
+                max_new_tokens: 10,
+                ..Default::default()
+            }),
+            response: Some(pb::ResponseOptions {
+                output_text: Some(true),
+                ..Default::default()
+            }),
+            ..Default::default()
+        })
+        .await
+        .expect("mtls generate over tls")
+        .into_inner();
+
+    assert_eq!(response.outputs.expect("outputs present").text, "hi");
+
+    engine_task.await.expect("mock engine task");
+    server_task.abort();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn grpc_keepalive_closes_unresponsive_connection() {
+    let (addr, server_task, _engine_task) =
+        grpc_server_with_keepalive(b"engine-grpc-keepalive", Some(Duration::from_millis(150)))
+            .await;
+
+    let closed = h2_unresponsive_peer_closed_within(&addr, Duration::from_secs(5)).await;
+    assert!(
+        closed,
+        "keepalive must close a peer that stops answering PINGs"
+    );
+
+    server_task.abort();
+}
+
+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn grpc_without_keepalive_keeps_unresponsive_connection_open() {
+    // Without keepalive the same unresponsive peer is NOT
+    // closed, proving the close above is attributable to keepalive.
+    let (addr, server_task, _engine_task) =
+        grpc_server_with_keepalive(b"engine-grpc-no-keepalive", None).await;
+
+    let closed = h2_unresponsive_peer_closed_within(&addr, Duration::from_secs(1)).await;
+    assert!(
+        !closed,
+        "without keepalive an idle h2 connection must stay open"
+    );
+
+    server_task.abort();
+}
@@ -10,20 +10,34 @@ mod routes;
 mod runtime;
 mod server_info;
 mod state;
+mod tls;
+#[cfg(test)]
+mod tls_tests;
 mod utils;

+use std::future::Future;
 use std::sync::{Arc, OnceLock};
+use std::time::Duration;

 use anyhow::{Context as _, Result};
 use axum::Router;
-use axum::serve::ListenerExt as _;
-pub use config::{ApiServerOptions, Config, CoordinatorMode, CorsConfig, HttpListenerMode};
+use axum::body::Body;
+use axum::http::Request;
+pub use config::{
+    ApiServerOptions, Config, CoordinatorMode, CorsConfig, DEFAULT_KEEP_ALIVE_TIMEOUT,
+    HttpListenerMode, TlsConfig,
+};
+use futures::FutureExt as _;
+use hyper::body::Incoming;
+use hyper::server::conn::http1;
+use hyper_util::rt::{TokioIo, TokioTimer};
+use hyper_util::server::graceful::GracefulShutdown;
+use hyper_util::service::TowerToHyperService;
 use tokio::net::TcpListener;
 use tokio::time::{Instant, sleep_until};
-use tokio_stream::wrappers::TcpListenerStream;
-use tokio_util::either::Either;
 use tokio_util::sync::CancellationToken;
 use tonic::transport::Server as TonicServer;
+use tower::ServiceExt as _;
 use tracing::{info, trace, warn};
 use vllm_chat::{ChatLlm, LoadModelBackendsOptions, load_model_backends};
 pub use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection};
@@ -36,6 +50,13 @@ use crate::routes::build_router;
 use crate::server_info::ServerInfoSnapshot;
 use crate::state::AppState;

+/// How often the server PINGs an idle gRPC connection to reap a dead peer;
+/// tonic enables no keepalive by default. 2h matches the gRPC-core default.
+const GRPC_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(7200);
+/// How long the server waits for a keepalive PING reply before dropping the gRPC
+/// connection. 20s matches the gRPC-core default.
+const GRPC_KEEPALIVE_TIMEOUT: Duration = Duration::from_secs(20);
+
 /// Resolve the public model names accepted by the frontend.
 fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Vec<String> {
    if served_model_name.is_empty() {
@@ -45,6 +66,17 @@ fn effective_served_model_names(model: &str, served_model_name: &[String]) -> Ve
    }
 }

+/// Choose the gRPC listener host. It follows the HTTP TCP host when there is
+/// one; otherwise (unix socket or inherited fd) it defaults to IPv4 loopback
+/// rather than all interfaces, so the side-car is never accidentally
+/// network-exposed.
+fn grpc_bind_host(listener_mode: &HttpListenerMode) -> &str {
+    match listener_mode {
+        HttpListenerMode::BindTcp { host, .. } => host.as_str(),
+        HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "127.0.0.1",
+    }
+}
+
 /// Build the shared application state for one configured model and one engine
 /// client.
 async fn build_state(config: &Config) -> Result<Arc<AppState>> {
@@ -130,6 +162,15 @@ where
 {
    config.validate().context("invalid OpenAI frontend configuration")?;

+    // Build the TLS server config once, up front, so a bad cert/key fails fast
+    // before the (potentially long) engine handshake.
+    let tls_config = config
+        .tls
+        .as_ref()
+        .map(tls::build_server_config)
+        .transpose()
+        .context("invalid TLS configuration")?;
+
    // Also check shutdown during the (potentially long) startup handshake.
    let state = tokio::select! {
        result = build_state(&config) => result?,
@@ -144,40 +185,39 @@ where

    // Optionally bind the gRPC Generate server on a separate port. Bind
    // synchronously here so bind errors (port in use, permission denied, ...)
-    // surface before we start serving, rather than being deferred until
-    // shutdown. The gRPC listener follows the same host as the HTTP listener so
-    // that enabling --grpc-port does not accidentally expose the service on all
-    // interfaces when HTTP is intentionally local-only.
+    // surface before serving rather than being deferred until shutdown.
    let grpc_setup = if let Some(grpc_port) = config.grpc_port {
-        let grpc_host = match &config.listener_mode {
-            HttpListenerMode::BindTcp { host, .. } => host.as_str(),
-            HttpListenerMode::BindUnix { .. } | HttpListenerMode::InheritedFd { .. } => "0.0.0.0",
-        };
+        let grpc_host = grpc_bind_host(&config.listener_mode);
        let grpc_listener = TcpListener::bind((grpc_host, grpc_port))
            .await
            .with_context(|| format!("failed to bind gRPC listener on {grpc_host}:{grpc_port}"))?;
        let addr = grpc_listener.local_addr()?;
+        let grpc_listener = Listener::Tcp(grpc_listener);
+        // gRPC reuses the HTTP TLS config (same SslContext) plus ALPN h2.
+        let grpc_tls = config
+            .tls
+            .as_ref()
+            .map(tls::build_grpc_server_config)
+            .transpose()
+            .context("invalid gRPC TLS configuration")?;
        let svc = grpc::GenerateServer::new(grpc::GenerateServiceImpl::new(state.clone()));
        let svc = TonicServer::builder()
+            .http2_keepalive_interval(Some(GRPC_KEEPALIVE_INTERVAL))
+            .http2_keepalive_timeout(Some(GRPC_KEEPALIVE_TIMEOUT))
            .layer(middleware::request_runtime_layer(state.clone()))
            .add_service(svc);
-        info!(%addr, "starting gRPC server");
-        Some((grpc_listener, svc))
+        info!(%addr, tls = grpc_tls.is_some(), "starting gRPC server");
+        Some((grpc_listener, svc, grpc_tls))
    } else {
        None
    };

-    info!(%bind_address, %model, "starting OpenAI server");
-
-    // Set TCP_NODELAY on accepted connections to reduce latency.
-    // By `tap_io` we will do this on every accepted connection.
-    let listener = listener.tap_io(|io| {
-        if let Either::Left(tcp_stream) = io
-            && let Err(err) = tcp_stream.set_nodelay(true)
-        {
-            trace!(error = %err, "failed to enable TCP_NODELAY on accepted HTTP connection");
-        }
-    });
+    let scheme = if tls_config.is_some() {
+        "https"
+    } else {
+        "http"
+    };
+    info!(%bind_address, %scheme, %model, "starting OpenAI server");

    // Run HTTP and gRPC concurrently under a child token of the caller's shutdown
    // token. Caller cancellation propagates into both protocols; if either
@@ -208,17 +248,27 @@ where
        }
    });

+    // 0 disables keep-alive but still bounds the head read (default), so a
+    // silent client cannot hold the connection open.
+    let keep_alive_timeout = config.keep_alive_timeout;
+    let timeouts = ConnectionTimeouts {
+        handshake: tls::TLS_HANDSHAKE_TIMEOUT,
+        header_read: if keep_alive_timeout.is_zero() {
+            DEFAULT_KEEP_ALIVE_TIMEOUT
+        } else {
+            keep_alive_timeout
+        },
+        keep_alive_enabled: !keep_alive_timeout.is_zero(),
+    };
+
    let http_fut = {
        let shutdown = server_shutdown.child_token();
        let server_shutdown = server_shutdown.clone();
        let force_shutdown = force_shutdown.clone();
        async move {
-            let server =
-                axum::serve(listener, app).with_graceful_shutdown(shutdown.cancelled_owned());
-
            let result = tokio::select! {
-                result = server => {
-                    result.context("HTTP server failed")
+                result = serve_listener(listener, tls_config, app, shutdown.cancelled_owned(), timeouts) => {
+                    result
                }
                _ = force_shutdown.cancelled() => {
                    warn!("HTTP graceful shutdown deadline elapsed; aborting server");
@@ -236,16 +286,24 @@ where
        let server_shutdown = server_shutdown.clone();
        let force_shutdown = force_shutdown.clone();
        async move {
-            let Some((grpc_listener, svc)) = grpc_setup else {
+            let Some((grpc_listener, svc, grpc_tls)) = grpc_setup else {
                // No gRPC configured: just wait for shutdown so we do not race the
                // join! by resolving early and tripping the cancellation token.
                shutdown.cancelled().await;
                return Ok(());
            };
-            let server = svc.serve_with_incoming_shutdown(
-                TcpListenerStream::new(grpc_listener),
-                shutdown.cancelled_owned(),
-            );
+            // Box to unify the TLS and plaintext arms' different stream types.
+            let server = match grpc_tls {
+                Some(context) => {
+                    let incoming =
+                        grpc::tls_incoming(grpc_listener, context, tls::TLS_HANDSHAKE_TIMEOUT);
+                    svc.serve_with_incoming_shutdown(incoming, shutdown.cancelled_owned()).boxed()
+                }
+                None => {
+                    let incoming = grpc::incoming(grpc_listener);
+                    svc.serve_with_incoming_shutdown(incoming, shutdown.cancelled_owned()).boxed()
+                }
+            };

            let result = tokio::select! {
                result = server => {
@@ -272,6 +330,99 @@ where
    state.shutdown(shutdown_deadline).await
 }

+/// Per-connection timeouts applied while serving HTTP/HTTPS.
+#[derive(Clone, Copy)]
+pub(crate) struct ConnectionTimeouts {
+    /// Max time for a client to complete the TLS handshake (TLS path only).
+    pub(crate) handshake: Duration,
+    /// HTTP/1 header-read timeout (bounds idle keep-alive and the head read).
+    pub(crate) header_read: Duration,
+    /// Whether HTTP/1 keep-alive is enabled; `false` closes after each response.
+    pub(crate) keep_alive_enabled: bool,
+}
+
+/// Apply optional TLS termination and per-connection HTTP timeouts, then serve
+/// `app`. Shared by [`serve_with_router_extension`] and the TLS tests.
+async fn serve_listener(
+    listener: Listener,
+    tls: Option<openssl::ssl::SslContext>,
+    app: Router,
+    shutdown: impl Future<Output = ()> + Send + 'static,
+    timeouts: ConnectionTimeouts,
+) -> Result<()> {
+    match tls {
+        Some(context) => {
+            // tls-listener terminates TLS (handshake + timeout); serve_connections
+            // owns the HTTP keep-alive/idle bound that axum::serve cannot express.
+            // Failed handshakes (incl. timeouts) log at ERROR via tls-listener.
+            let listener = tls_listener::builder(context)
+                .handshake_timeout(timeouts.handshake)
+                .listen(listener);
+            serve_connections(
+                listener,
+                app,
+                shutdown,
+                timeouts.header_read,
+                timeouts.keep_alive_enabled,
+            )
+            .await
+            .context("HTTPS server failed")
+        }
+        None => serve_connections(
+            listener,
+            app,
+            shutdown,
+            timeouts.header_read,
+            timeouts.keep_alive_enabled,
+        )
+        .await
+        .context("HTTP server failed"),
+    }
+}
+
+/// Serve `app` per connection (HTTP/1) with a keep-alive idle timeout and
+/// graceful drain. Hand-rolled on hyper because [`axum::serve()`] takes no config.
+async fn serve_connections<L>(
+    mut listener: L,
+    app: Router,
+    shutdown: impl Future<Output = ()> + Send,
+    header_read: Duration,
+    keep_alive_enabled: bool,
+) -> Result<()>
+where
+    L: axum::serve::Listener,
+{
+    let graceful = GracefulShutdown::new();
+    let mut shutdown = std::pin::pin!(shutdown);
+    loop {
+        let (io, _addr) = tokio::select! {
+            conn = listener.accept() => conn,
+            () = &mut shutdown => break,
+        };
+
+        let service = TowerToHyperService::new(
+            app.clone().map_request(|req: Request<Incoming>| req.map(Body::new)),
+        );
+        let mut builder = http1::Builder::new();
+        builder.timer(TokioTimer::new()).header_read_timeout(header_read);
+        if !keep_alive_enabled {
+            builder.keep_alive(false);
+        }
+        let connection = builder.serve_connection(TokioIo::new(io), service);
+        let connection = graceful.watch(connection);
+
+        tokio::spawn(async move {
+            if let Err(err) = connection.await {
+                trace!(error = %err, "failed to serve connection");
+            }
+        });
+    }
+
+    drop(listener);
+    graceful.shutdown().await;
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
    use super::*;
@@ -293,4 +444,23 @@ mod tests {
            served_names
        );
    }
+
+    #[test]
+    fn grpc_bind_host_follows_http_tcp_host() {
+        let mode = HttpListenerMode::BindTcp {
+            host: "0.0.0.0".to_string(),
+            port: 8000,
+        };
+        assert_eq!(grpc_bind_host(&mode), "0.0.0.0");
+    }
+
+    #[test]
+    fn grpc_bind_host_defaults_to_loopback_without_tcp_host() {
+        let unix = HttpListenerMode::BindUnix {
+            path: "/tmp/vllm.sock".to_string(),
+        };
+        let inherited = HttpListenerMode::InheritedFd { fd: 3 };
+        assert_eq!(grpc_bind_host(&unix), "127.0.0.1");
+        assert_eq!(grpc_bind_host(&inherited), "127.0.0.1");
+    }
 }
@@ -1,28 +1,49 @@
-//! Unified HTTP listener wrapper for the Rust frontend.
+//! Unified listener wrapper for the Rust frontend.
 //!
 //! This module hides the difference between TCP and Unix-domain listeners so
 //! the rest of the server can bind or inherit one socket and pass it to
 //! `axum::serve(...)` through a single type.

 use std::io::Result;
-use std::net::TcpListener as StdTcpListener;
+use std::net::{SocketAddr, TcpListener as StdTcpListener};
 use std::os::fd::{FromRawFd, IntoRawFd, OwnedFd};
 use std::os::unix::net::UnixListener as StdUnixListener;
+use std::pin::Pin;
+use std::task::{Context, Poll, ready};

+use auto_enums::enum_derive;
 use socket2::Socket;
+use tls_listener::{AsyncAccept, AsyncListener};
 use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream};
-use tokio_util::either::Either;
+use tonic::transport::server::{Connected, TcpConnectInfo};
+use tracing::trace;

 use crate::HttpListenerMode;

-/// Runtime listener type used by the OpenAI-compatible HTTP server, which is
-/// either a TCP listener or a Unix-domain listener.
+/// Runtime listener type used by the OpenAI-compatible HTTP or gRPC server,
+/// which is either a TCP listener or a Unix-domain listener.
 #[derive(Debug)]
 pub enum Listener {
    Tcp(TcpListener),
    Unix(UnixListener),
 }

+/// Runtime listener I/O type which is either a TCP stream or a Unix-domain stream.
+#[derive(Debug)]
+#[enum_derive(tokio1::AsyncRead, tokio1::AsyncWrite)]
+pub enum ListenerIo {
+    Tcp(TcpStream),
+    Unix(UnixStream),
+}
+
+/// Runtime listener address type which is either a TCP address or a Unix-domain address.
+#[derive(Debug)]
+#[allow(dead_code)]
+pub enum ListenerAddr {
+    Tcp(SocketAddr),
+    Unix(tokio::net::unix::SocketAddr),
+}
+
 impl Listener {
    /// Bind or adopt the listener described by the frontend configuration.
    ///
@@ -70,34 +91,95 @@ impl Listener {
            Ok(Self::Tcp(TcpListener::from_std(std_listener)?))
        }
    }
+
+    fn listener_addr(&self) -> Result<ListenerAddr> {
+        match self {
+            Self::Tcp(listener) => listener.local_addr().map(ListenerAddr::Tcp),
+            Self::Unix(listener) => listener.local_addr().map(ListenerAddr::Unix),
+        }
+    }
+}
+
+impl Connected for ListenerIo {
+    type ConnectInfo = TcpConnectInfo;
+
+    fn connect_info(&self) -> TcpConnectInfo {
+        match self {
+            Self::Tcp(stream) => stream.connect_info(),
+            Self::Unix(_) => TcpConnectInfo {
+                local_addr: None,
+                remote_addr: None,
+            },
+        }
+    }
+}
+
+/// Attempt to set `TCP_NODELAY` on the accepted TCP stream.
+fn enable_tcp_nodelay(stream: TcpStream) -> TcpStream {
+    if let Err(err) = stream.set_nodelay(true) {
+        trace!(error = %err, "failed to enable TCP_NODELAY on accepted TCP connection");
+    }
+    stream
 }

 /// Allow the unified listener to plug directly into `axum::serve(...)`.
 impl axum::serve::Listener for Listener {
-    type Addr = Either<std::net::SocketAddr, tokio::net::unix::SocketAddr>;
-    type Io = Either<TcpStream, UnixStream>;
+    type Addr = ListenerAddr;
+    type Io = ListenerIo;

    async fn accept(&mut self) -> (Self::Io, Self::Addr) {
        match self {
            Self::Tcp(listener) => {
-                let (io, addr) = listener.accept().await;
-                (Either::Left(io), Either::Left(addr))
+                let (io, addr) = axum::serve::Listener::accept(listener).await;
+                (
+                    ListenerIo::Tcp(enable_tcp_nodelay(io)),
+                    ListenerAddr::Tcp(addr),
+                )
            }
            Self::Unix(listener) => {
-                let (io, addr) = listener.accept().await;
-                (Either::Right(io), Either::Right(addr))
+                let (io, addr) = axum::serve::Listener::accept(listener).await;
+                (ListenerIo::Unix(io), ListenerAddr::Unix(addr))
            }
        }
    }

    fn local_addr(&self) -> Result<Self::Addr> {
-        match self {
-            Self::Tcp(listener) => listener.local_addr().map(Either::Left),
-            Self::Unix(listener) => listener.local_addr().map(Either::Right),
+        self.listener_addr()
+    }
+}
+
+/// Allow the unified listener to be adaptable to `tls_listener`.
+impl AsyncAccept for Listener {
+    type Connection = ListenerIo;
+    type Address = ListenerAddr;
+    type Error = std::io::Error;
+
+    fn poll_accept(
+        self: Pin<&mut Self>,
+        cx: &mut Context<'_>,
+    ) -> Poll<Result<(Self::Connection, Self::Address)>> {
+        match self.get_mut() {
+            Self::Tcp(listener) => {
+                let (io, addr) = ready!(listener.poll_accept(cx))?;
+                Poll::Ready(Ok((
+                    ListenerIo::Tcp(enable_tcp_nodelay(io)),
+                    ListenerAddr::Tcp(addr),
+                )))
+            }
+            Self::Unix(listener) => {
+                let (io, addr) = ready!(listener.poll_accept(cx))?;
+                Poll::Ready(Ok((ListenerIo::Unix(io), ListenerAddr::Unix(addr))))
+            }
        }
    }
 }

+impl AsyncListener for Listener {
+    fn local_addr(&self) -> Result<Self::Address> {
+        self.listener_addr()
+    }
+}
+
 #[cfg(test)]
 mod tests {
    use std::net::{Ipv4Addr, SocketAddrV4};
@@ -34,14 +34,20 @@ pub(super) fn validate_request_compat(
        );
    }

-    if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs
-        && prompt_logprobs < 0
-        && prompt_logprobs != -1
-    {
-        bail_invalid_request!(
-            param = "sampling_params",
-            "`prompt_logprobs` must be a non-negative value or -1."
-        );
+    if let Some(prompt_logprobs) = request.sampling_params.prompt_logprobs {
+        if prompt_logprobs < 0 && prompt_logprobs != -1 {
+            bail_invalid_request!(
+                param = "sampling_params",
+                "`prompt_logprobs` must be a non-negative value or -1."
+            );
+        }
+
+        if request.stream {
+            bail_invalid_request!(
+                param = "sampling_params",
+                "`prompt_logprobs` are not available when `stream=true`."
+            );
+        }
    }

    Ok(())
@@ -97,4 +103,54 @@ mod tests {
        };
        assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
    }
+
+    #[test]
+    fn validate_request_compat_rejects_streaming_prompt_logprobs() {
+        let request: GenerateRequest = serde_json::from_value(json!({
+            "model": "Qwen/Qwen1.5-0.5B-Chat",
+            "token_ids": [11, 22],
+            "stream": true,
+            "sampling_params": {
+                "prompt_logprobs": 0
+            }
+        }))
+        .expect("parse request");
+        assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
+
+        let request: GenerateRequest = serde_json::from_value(json!({
+            "model": "Qwen/Qwen1.5-0.5B-Chat",
+            "token_ids": [11, 22],
+            "stream": true,
+            "sampling_params": {
+                "prompt_logprobs": 1
+            }
+        }))
+        .expect("parse request");
+        assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
+
+        let request: GenerateRequest = serde_json::from_value(json!({
+            "model": "Qwen/Qwen1.5-0.5B-Chat",
+            "token_ids": [11, 22],
+            "stream": true,
+            "sampling_params": {
+                "prompt_logprobs": -1
+            }
+        }))
+        .expect("parse request");
+        assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
+    }
+
+    #[test]
+    fn validate_request_compat_accepts_non_stream_prompt_logprobs() {
+        let request: GenerateRequest = serde_json::from_value(json!({
+            "model": "Qwen/Qwen1.5-0.5B-Chat",
+            "token_ids": [11, 22],
+            "stream": false,
+            "sampling_params": {
+                "prompt_logprobs": 1
+            }
+        }))
+        .expect("parse request");
+        assert!(validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_ok());
+    }
 }
@@ -4129,6 +4129,45 @@ async fn raw_generate_rejects_empty_token_ids() {
    assert_eq!(json["error"]["param"], "token_ids");
 }

+#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
+#[serial]
+async fn raw_generate_rejects_streaming_prompt_logprobs() {
+    let mut app = test_app().await;
+
+    for prompt_logprobs in [0, 1] {
+        let response = app
+            .call(
+                Request::builder()
+                    .method("POST")
+                    .uri("/inference/v1/generate")
+                    .header("content-type", "application/json")
+                    .body(Body::from(
+                        json!({
+                            "model": "Qwen/Qwen1.5-0.5B-Chat",
+                            "token_ids": [11, 22],
+                            "stream": true,
+                            "sampling_params": {
+                                "prompt_logprobs": prompt_logprobs
+                            }
+                        })
+                        .to_string(),
+                    ))
+                    .expect("build request"),
+            )
+            .await
+            .expect("call app");
+
+        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
+        let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body");
+        let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json");
+        assert_eq!(json["error"]["param"], "sampling_params");
+        assert_eq!(
+            json["error"]["message"],
+            "`prompt_logprobs` are not available when `stream=true`."
+        );
+    }
+}
+
 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
 #[serial]
 async fn raw_generate_rejects_wrong_model() {
@@ -0,0 +1,119 @@
+//! OpenSSL server-config construction for TLS termination.
+//!
+//! Builds an OpenSSL [`SslContext`] from the uvicorn-style `ssl_*` arguments
+//! (certificate chain, private key, mTLS client verifier, optional cipher list).
+//! The `tls-listener` crate drives the handshake on each accepted connection.
+//!
+//! Crypto runs through whichever OpenSSL the binary links (system by default,
+//! vendored when built with that feature).
+
+use std::path::Path;
+use std::time::Duration;
+
+use anyhow::{Context as _, Result};
+use openssl::ssl::{
+    AlpnError, SslAcceptor, SslAcceptorBuilder, SslContext, SslContextBuilder, SslFiletype,
+    SslMethod, SslOptions, SslVerifyMode, select_next_proto,
+};
+
+use crate::config::TlsConfig;
+
+/// Time a client has to complete the TLS handshake before the connection is dropped.
+pub(crate) const TLS_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(60);
+
+/// ALPN wire bytes for HTTP/2 (length-prefixed).
+const ALPN_H2: &[u8] = b"\x02h2";
+
+/// Build the shared OpenSSL acceptor from validated [`TlsConfig`]: the full
+/// certificate chain, the private key (`key_file`, or the certificate file when
+/// unset), the mTLS client verifier, and an optional cipher list.
+///
+/// Starts from the Mozilla intermediate baseline (forward-secret AEAD suites,
+/// TLS 1.2 floor, server cipher preference, no compression), a slightly
+/// stricter subset of the Python frontend's default suites; `--ssl-ciphers`
+/// overrides it.
+fn build_server_builder(tls: &TlsConfig) -> Result<SslAcceptorBuilder> {
+    let cert_file = tls.cert_file.as_deref().context("--ssl-certfile is required to enable TLS")?;
+
+    let mut builder = SslAcceptor::mozilla_intermediate_v5(SslMethod::tls_server())
+        .context("failed to initialize TLS")?;
+    builder.set_options(SslOptions::CIPHER_SERVER_PREFERENCE);
+
+    // Load the whole chain (leaf + intermediates), not just the leaf, so
+    // deployments behind an intermediate CA serve a complete chain.
+    ensure_exists(cert_file, "--ssl-certfile")?;
+    builder.set_certificate_chain_file(cert_file).with_context(|| {
+        format!("failed to parse certificate chain in --ssl-certfile {cert_file:?}")
+    })?;
+
+    // When `key_file` is unset the key is read from the certificate file
+    // (combined PEM).
+    let key_file = tls.key_file.as_deref().unwrap_or(cert_file);
+    ensure_exists(key_file, "private key file")?;
+    builder
+        .set_private_key_file(key_file, SslFiletype::PEM)
+        .with_context(|| format!("failed to parse private key in {key_file:?}"))?;
+    builder
+        .check_private_key()
+        .context("the certificate and private key do not match")?;
+
+    configure_client_auth(&mut builder, tls)?;
+
+    if let Some(ciphers) = tls.ciphers.as_deref().filter(|c| !c.is_empty()) {
+        builder
+            .set_cipher_list(ciphers)
+            .with_context(|| format!("invalid --ssl-ciphers {ciphers:?}"))?;
+    }
+
+    Ok(builder)
+}
+
+/// Build the HTTP [`SslContext`] (HTTP/1.1; no ALPN, matching uvicorn).
+pub(crate) fn build_server_config(tls: &TlsConfig) -> Result<SslContext> {
+    Ok(build_server_builder(tls)?.build().into_context())
+}
+
+/// Build the gRPC [`SslContext`]: identical to [`build_server_config`] but
+/// negotiates ALPN `h2`, which HTTP/2 over TLS requires.
+pub(crate) fn build_grpc_server_config(tls: &TlsConfig) -> Result<SslContext> {
+    let mut builder = build_server_builder(tls)?;
+    builder.set_alpn_select_callback(|_ssl, client| {
+        select_next_proto(ALPN_H2, client).ok_or(AlpnError::NOACK)
+    });
+    Ok(builder.build().into_context())
+}
+
+/// Fail loudly with a flag-named message when a configured file is missing,
+/// distinguishing it from a malformed-PEM error raised later by OpenSSL (whose
+/// `ErrorStack` does not name the offending file).
+fn ensure_exists(path: &str, what: &str) -> Result<()> {
+    std::fs::metadata(Path::new(path))
+        .map(drop)
+        .with_context(|| format!("failed to read {what} {path:?}"))
+}
+
+/// Apply the `cert_reqs` client-certificate policy: 0 = none, 1 = optional
+/// (verify if presented, allow anonymous), 2 = required. `PEER` without a custom
+/// verify callback still rejects a presented-but-untrusted certificate.
+fn configure_client_auth(builder: &mut SslContextBuilder, tls: &TlsConfig) -> Result<()> {
+    if tls.cert_reqs == 0 {
+        builder.set_verify(SslVerifyMode::NONE);
+        return Ok(());
+    }
+
+    let ca_file = tls
+        .ca_certs
+        .as_deref()
+        .context("--ssl-ca-certs is required for client certificate verification")?;
+    ensure_exists(ca_file, "--ssl-ca-certs")?;
+    builder
+        .set_ca_file(ca_file)
+        .with_context(|| format!("failed to parse --ssl-ca-certs {ca_file:?}"))?;
+
+    let mut mode = SslVerifyMode::PEER;
+    if tls.cert_reqs == 2 {
+        mode |= SslVerifyMode::FAIL_IF_NO_PEER_CERT;
+    }
+    builder.set_verify(mode);
+    Ok(())
+}
@@ -0,0 +1,688 @@
+//! TLS tests: `build_server_config` unit checks plus end-to-end OpenSSL handshakes
+//! through the production `serve_listener` path, with a trivial router since TLS
+//! terminates below the app.
+
+use std::pin::Pin;
+use std::time::Duration;
+
+use axum::Router;
+use axum::routing::get;
+use openssl::asn1::Asn1Time;
+use openssl::bn::{BigNum, MsbOption};
+use openssl::ec::{EcGroup, EcKey};
+use openssl::hash::MessageDigest;
+use openssl::nid::Nid;
+use openssl::pkey::{PKey, Private};
+use openssl::ssl::{SslConnector, SslFiletype, SslMethod, SslVersion};
+use openssl::x509::extension::{BasicConstraints, KeyUsage, SubjectAlternativeName};
+use openssl::x509::{X509, X509NameBuilder};
+use tempfile::TempDir;
+use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+use tokio::net::TcpStream;
+use tokio_openssl::SslStream;
+use tokio_util::sync::CancellationToken;
+
+use crate::config::{HttpListenerMode, TlsConfig};
+use crate::listener::Listener;
+use crate::{ConnectionTimeouts, serve_listener, tls};
+
+// ============================================================================
+// Test infrastructure
+// ============================================================================
+
+/// A throwaway CA + server/client/untrusted/chain cert set as PEM files in a
+/// temp dir; dropping it deletes them.
+pub(crate) struct TestCerts {
+    dir: TempDir,
+}
+
+impl TestCerts {
+    pub(crate) fn generate() -> Self {
+        let dir = tempfile::tempdir().expect("tempdir");
+
+        let (ca, ca_key) = build_ca();
+        let (server, server_key) = build_leaf("server", &["127.0.0.1", "localhost"], &ca, &ca_key);
+        let (client, client_key) = build_leaf("client", &[], &ca, &ca_key);
+        let (untrusted, untrusted_key) = build_self_signed("untrusted client");
+
+        // Leaf signed by an intermediate (itself signed by the root); the cert
+        // file holds leaf + intermediate, for the chain-serving test.
+        let (intermediate, intermediate_key) = build_intermediate(&ca, &ca_key);
+        let (chain_leaf, chain_leaf_key) = build_leaf(
+            "chain",
+            &["127.0.0.1", "localhost"],
+            &intermediate,
+            &intermediate_key,
+        );
+
+        let server_pem = pem(&server);
+        let server_key_pem = key_pem(&server_key);
+        let files = [
+            ("ca.pem", pem(&ca)),
+            ("server.pem", server_pem.clone()),
+            ("server.key", server_key_pem.clone()),
+            ("client.pem", pem(&client)),
+            ("client.key", key_pem(&client_key)),
+            ("untrusted_client.pem", pem(&untrusted)),
+            ("untrusted_client.key", key_pem(&untrusted_key)),
+            (
+                "server_combined.pem",
+                format!("{server_pem}{server_key_pem}"),
+            ),
+            (
+                "server_chain.pem",
+                format!("{}{}", pem(&chain_leaf), pem(&intermediate)),
+            ),
+            ("server_chain.key", key_pem(&chain_leaf_key)),
+        ];
+        for (name, contents) in files {
+            std::fs::write(dir.path().join(name), contents).expect("write fixture");
+        }
+        Self { dir }
+    }
+
+    /// Absolute path to a fixture by name; the file need not exist.
+    pub(crate) fn path(&self, name: &str) -> String {
+        self.dir.path().join(name).to_str().expect("utf-8 path").to_string()
+    }
+}
+
+fn gen_key() -> PKey<Private> {
+    let group = EcGroup::from_curve_name(Nid::X9_62_PRIME256V1).expect("ec group");
+    let ec = EcKey::generate(&group).expect("ec key");
+    PKey::from_ec_key(ec).expect("pkey")
+}
+
+fn serial() -> openssl::asn1::Asn1Integer {
+    let mut bn = BigNum::new().expect("bignum");
+    bn.rand(159, MsbOption::MAYBE_ZERO, false).expect("rand serial");
+    bn.to_asn1_integer().expect("asn1 serial")
+}
+
+fn x509_name(cn: &str) -> openssl::x509::X509Name {
+    let mut builder = X509NameBuilder::new().expect("name builder");
+    builder.append_entry_by_text("CN", cn).expect("cn");
+    builder.build()
+}
+
+fn pem(cert: &X509) -> String {
+    String::from_utf8(cert.to_pem().expect("cert pem")).expect("utf-8 cert")
+}
+
+fn key_pem(key: &PKey<Private>) -> String {
+    String::from_utf8(key.private_key_to_pem_pkcs8().expect("key pem")).expect("utf-8 key")
+}
+
+/// A self-signed CA used to sign the server/client leaf certs.
+fn build_ca() -> (X509, PKey<Private>) {
+    let key = gen_key();
+    let name = x509_name("vLLM Test CA");
+    let mut builder = X509::builder().expect("x509 builder");
+    builder.set_version(2).expect("version");
+    builder.set_serial_number(&serial()).expect("serial");
+    builder.set_subject_name(&name).expect("subject");
+    builder.set_issuer_name(&name).expect("issuer");
+    builder.set_pubkey(&key).expect("pubkey");
+    builder
+        .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
+        .expect("set nb");
+    builder
+        .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
+        .expect("set na");
+    builder
+        .append_extension(BasicConstraints::new().critical().ca().build().expect("bc"))
+        .expect("ext bc");
+    builder
+        .append_extension(
+            KeyUsage::new().critical().key_cert_sign().crl_sign().build().expect("ku"),
+        )
+        .expect("ext ku");
+    builder.sign(&key, MessageDigest::sha256()).expect("sign ca");
+    (builder.build(), key)
+}
+
+/// A CA-signed leaf cert with optional subject-alternative names (IP or DNS).
+fn build_leaf(cn: &str, sans: &[&str], ca: &X509, ca_key: &PKey<Private>) -> (X509, PKey<Private>) {
+    let key = gen_key();
+    let mut builder = X509::builder().expect("x509 builder");
+    builder.set_version(2).expect("version");
+    builder.set_serial_number(&serial()).expect("serial");
+    builder.set_subject_name(&x509_name(cn)).expect("subject");
+    builder.set_issuer_name(ca.subject_name()).expect("issuer");
+    builder.set_pubkey(&key).expect("pubkey");
+    builder
+        .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
+        .expect("set nb");
+    builder
+        .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
+        .expect("set na");
+    builder
+        .append_extension(BasicConstraints::new().build().expect("bc"))
+        .expect("ext bc");
+    if !sans.is_empty() {
+        let mut san = SubjectAlternativeName::new();
+        for entry in sans {
+            if entry.parse::<std::net::IpAddr>().is_ok() {
+                san.ip(entry);
+            } else {
+                san.dns(entry);
+            }
+        }
+        let ext = san.build(&builder.x509v3_context(Some(ca), None)).expect("san");
+        builder.append_extension(ext).expect("ext san");
+    }
+    builder.sign(ca_key, MessageDigest::sha256()).expect("sign leaf");
+    (builder.build(), key)
+}
+
+/// A self-signed leaf not chained to the CA, for the untrusted-client test.
+fn build_self_signed(cn: &str) -> (X509, PKey<Private>) {
+    let key = gen_key();
+    let name = x509_name(cn);
+    let mut builder = X509::builder().expect("x509 builder");
+    builder.set_version(2).expect("version");
+    builder.set_serial_number(&serial()).expect("serial");
+    builder.set_subject_name(&name).expect("subject");
+    builder.set_issuer_name(&name).expect("issuer");
+    builder.set_pubkey(&key).expect("pubkey");
+    builder
+        .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
+        .expect("set nb");
+    builder
+        .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
+        .expect("set na");
+    builder
+        .append_extension(BasicConstraints::new().build().expect("bc"))
+        .expect("ext bc");
+    builder.sign(&key, MessageDigest::sha256()).expect("sign self");
+    (builder.build(), key)
+}
+
+/// A CA-capable intermediate signed by the root, for the full-chain test.
+fn build_intermediate(ca: &X509, ca_key: &PKey<Private>) -> (X509, PKey<Private>) {
+    let key = gen_key();
+    let mut builder = X509::builder().expect("x509 builder");
+    builder.set_version(2).expect("version");
+    builder.set_serial_number(&serial()).expect("serial");
+    builder
+        .set_subject_name(&x509_name("vLLM Test Intermediate CA"))
+        .expect("subject");
+    builder.set_issuer_name(ca.subject_name()).expect("issuer");
+    builder.set_pubkey(&key).expect("pubkey");
+    builder
+        .set_not_before(&Asn1Time::days_from_now(0).expect("nb"))
+        .expect("set nb");
+    builder
+        .set_not_after(&Asn1Time::days_from_now(3650).expect("na"))
+        .expect("set na");
+    builder
+        .append_extension(BasicConstraints::new().critical().ca().build().expect("bc"))
+        .expect("ext bc");
+    builder
+        .append_extension(
+            KeyUsage::new().critical().key_cert_sign().crl_sign().build().expect("ku"),
+        )
+        .expect("ext ku");
+    builder.sign(ca_key, MessageDigest::sha256()).expect("sign intermediate");
+    (builder.build(), key)
+}
+
+pub(crate) fn server_tls(certs: &TestCerts, cert_reqs: i32) -> TlsConfig {
+    TlsConfig {
+        cert_file: Some(certs.path("server.pem")),
+        key_file: Some(certs.path("server.key")),
+        ca_certs: (cert_reqs != 0).then(|| certs.path("ca.pem")),
+        cert_reqs,
+        ciphers: None,
+    }
+}
+
+/// A plaintext-listener TLS config for `build_server_config` checks (`cert_reqs`
+/// 0, no client auth), with the cert/key files chosen by the caller.
+fn build_tls(certs: &TestCerts, cert: &str, key: Option<&str>) -> TlsConfig {
+    TlsConfig {
+        cert_file: Some(certs.path(cert)),
+        key_file: key.map(|k| certs.path(k)),
+        ca_certs: None,
+        cert_reqs: 0,
+        ciphers: None,
+    }
+}
+
+/// Generous per-connection timeouts that never fire during the fast tests.
+const TEST_TIMEOUTS: ConnectionTimeouts = ConnectionTimeouts {
+    handshake: Duration::from_secs(60),
+    header_read: Duration::from_secs(5),
+    keep_alive_enabled: true,
+};
+
+async fn spawn_server(tls_config: Option<TlsConfig>) -> (String, CancellationToken) {
+    spawn_server_with_timeouts(tls_config, TEST_TIMEOUTS).await
+}
+
+/// Bind an ephemeral listener and serve a trivial router via the production
+/// `serve_listener`, optionally with TLS. The listener is bound (and thus
+/// accepting into the backlog) before returning, so a client may connect
+/// immediately without a sleep.
+async fn spawn_server_with_timeouts(
+    tls_config: Option<TlsConfig>,
+    timeouts: ConnectionTimeouts,
+) -> (String, CancellationToken) {
+    let listener = Listener::bind(&HttpListenerMode::BindTcp {
+        host: "127.0.0.1".to_string(),
+        port: 0,
+    })
+    .await
+    .expect("bind listener");
+    let addr = listener.local_addr().expect("local addr");
+
+    let server_config =
+        tls_config.map(|cfg| tls::build_server_config(&cfg).expect("build server config"));
+    let app = Router::new().route("/health", get(|| async { "ok" }));
+    let shutdown = CancellationToken::new();
+    let server_shutdown = shutdown.clone();
+    tokio::spawn(async move {
+        let _ = serve_listener(
+            listener,
+            server_config,
+            app,
+            server_shutdown.cancelled_owned(),
+            timeouts,
+        )
+        .await;
+    });
+    (addr, shutdown)
+}
+
+/// Open a TLS connection trusting the test CA and finish the handshake,
+/// optionally presenting a client identity (`<name>.pem` + `<name>.key`) for
+/// mTLS. Hostname verification is disabled (the IP-SAN match is not under test);
+/// chain verification stays on, so an untrusted server cert is still rejected.
+async fn connect_tls(
+    certs: &TestCerts,
+    addr: &str,
+    identity: Option<&str>,
+) -> std::io::Result<Pin<Box<SslStream<TcpStream>>>> {
+    let tcp = TcpStream::connect(addr).await?;
+
+    let mut builder = SslConnector::builder(SslMethod::tls_client()).expect("connector builder");
+    builder.set_ca_file(certs.path("ca.pem")).expect("trust ca");
+    if let Some(name) = identity {
+        builder
+            .set_certificate_chain_file(certs.path(&format!("{name}.pem")))
+            .expect("client cert");
+        builder
+            .set_private_key_file(certs.path(&format!("{name}.key")), SslFiletype::PEM)
+            .expect("client key");
+    }
+    let connector = builder.build();
+    let mut config = connector.configure().expect("configure");
+    config.set_verify_hostname(false);
+    let ssl = config.into_ssl("127.0.0.1").expect("ssl");
+
+    let mut stream = Box::pin(SslStream::new(ssl, tcp).expect("client ssl stream"));
+    stream.as_mut().connect().await.map_err(std::io::Error::other)?;
+    Ok(stream)
+}
+
+/// Issue an HTTPS GET (with `Connection: close`), optionally with an mTLS identity.
+async fn https_get(
+    certs: &TestCerts,
+    addr: &str,
+    identity: Option<&str>,
+) -> std::io::Result<String> {
+    let mut stream = connect_tls(certs, addr, identity).await?;
+    stream
+        .write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n")
+        .await?;
+    let mut response = String::new();
+    stream.read_to_string(&mut response).await?;
+    Ok(response)
+}
+
+/// Attempt a handshake offering only a legacy CBC+SHA1 suite over TLS 1.2,
+/// capping the version so TLS 1.3 cannot rescue the negotiation.
+async fn legacy_suite_handshake(certs: &TestCerts, addr: &str) -> std::io::Result<()> {
+    let tcp = TcpStream::connect(addr).await?;
+
+    let mut builder = SslConnector::builder(SslMethod::tls_client()).expect("connector builder");
+    builder.set_ca_file(certs.path("ca.pem")).expect("trust ca");
+    builder.set_max_proto_version(Some(SslVersion::TLS1_2)).expect("cap tls1.2");
+    builder
+        .set_cipher_list("ECDHE-ECDSA-AES256-SHA:@SECLEVEL=0")
+        .expect("legacy cipher");
+    let connector = builder.build();
+    let mut config = connector.configure().expect("configure");
+    config.set_verify_hostname(false);
+    let ssl = config.into_ssl("127.0.0.1").expect("ssl");
+
+    let stream = SslStream::new(ssl, tcp).expect("client ssl stream");
+    tokio::pin!(stream);
+    stream.as_mut().connect().await.map_err(std::io::Error::other)
+}
+
+async fn plain_get(addr: &str) -> std::io::Result<String> {
+    let mut tcp = TcpStream::connect(addr).await?;
+    tcp.write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n")
+        .await?;
+    let mut response = String::new();
+    tcp.read_to_string(&mut response).await?;
+    Ok(response)
+}
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+#[test]
+fn builds_from_combined_pem() {
+    // Key omitted: it is read from the combined cert+key file.
+    let certs = TestCerts::generate();
+    assert!(tls::build_server_config(&build_tls(&certs, "server_combined.pem", None)).is_ok());
+}
+
+#[test]
+fn rejects_missing_cert_file() {
+    let certs = TestCerts::generate();
+    assert!(tls::build_server_config(&build_tls(&certs, "does_not_exist.pem", None)).is_err());
+}
+
+#[test]
+fn accepts_valid_cipher_list() {
+    let certs = TestCerts::generate();
+    let mut cfg = build_tls(&certs, "server.pem", Some("server.key"));
+    cfg.ciphers = Some("ECDHE-ECDSA-AES256-GCM-SHA384".to_string());
+    assert!(tls::build_server_config(&cfg).is_ok());
+}
+
+#[test]
+fn rejects_invalid_cipher_list() {
+    let certs = TestCerts::generate();
+    let mut cfg = build_tls(&certs, "server.pem", Some("server.key"));
+    cfg.ciphers = Some("THIS-IS-NOT-A-CIPHER".to_string());
+    assert!(tls::build_server_config(&cfg).is_err());
+}
+
+#[test]
+fn rejects_mismatched_cert_and_key() {
+    // check_private_key must reject a key that does not match the certificate.
+    let certs = TestCerts::generate();
+    let tls = build_tls(&certs, "client.pem", Some("server.key"));
+    assert!(tls::build_server_config(&tls).is_err());
+}
+
+#[tokio::test]
+async fn https_request_succeeds_over_tls() {
+    let certs = TestCerts::generate();
+    let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 0))).await;
+    let response = https_get(&certs, &addr, None).await.expect("https request");
+    assert!(response.starts_with("HTTP/1.1 200"), "{response}");
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn serves_full_certificate_chain() {
+    // Cert file holds leaf + intermediate; a client trusting only the root can
+    // verify only if the server sends the intermediate, guarding against a
+    // leaf-only load.
+    let certs = TestCerts::generate();
+    let tls = TlsConfig {
+        cert_file: Some(certs.path("server_chain.pem")),
+        key_file: Some(certs.path("server_chain.key")),
+        ca_certs: None,
+        cert_reqs: 0,
+        ciphers: None,
+    };
+    let (addr, shutdown) = spawn_server(Some(tls)).await;
+    let response = https_get(&certs, &addr, None).await.expect("chained https request");
+    assert!(response.starts_with("HTTP/1.1 200"), "{response}");
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn rejects_legacy_cipher_only_client() {
+    let certs = TestCerts::generate();
+    let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 0))).await;
+    let result = legacy_suite_handshake(&certs, &addr).await;
+    assert!(result.is_err(), "legacy-only client must be rejected");
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn ssl_ciphers_override_widens_past_preset() {
+    // Counterpart to rejects_legacy_cipher_only_client: --ssl-ciphers set to that
+    // same legacy suite lets the client through, proving the override beats the preset.
+    let certs = TestCerts::generate();
+    let mut tls = server_tls(&certs, 0);
+    tls.ciphers = Some("ECDHE-ECDSA-AES256-SHA:@SECLEVEL=0".to_string());
+    let (addr, shutdown) = spawn_server(Some(tls)).await;
+    let result = legacy_suite_handshake(&certs, &addr).await;
+    assert!(
+        result.is_ok(),
+        "override must allow the legacy suite: {result:?}"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn mtls_required_rejects_client_without_certificate() {
+    let certs = TestCerts::generate();
+    let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 2))).await;
+    let result = https_get(&certs, &addr, None).await;
+    assert!(
+        result.is_err(),
+        "handshake must fail without a client certificate"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn mtls_required_accepts_valid_client_certificate() {
+    let certs = TestCerts::generate();
+    let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 2))).await;
+    let response = https_get(&certs, &addr, Some("client")).await.expect("mtls request");
+    assert!(response.starts_with("HTTP/1.1 200"), "{response}");
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn mtls_optional_allows_anonymous_and_authenticated() {
+    let certs = TestCerts::generate();
+    let (addr, shutdown) = spawn_server(Some(server_tls(&certs, 1))).await;
+    let anonymous = https_get(&certs, &addr, None).await.expect("anonymous request");
+    assert!(anonymous.starts_with("HTTP/1.1 200"), "{anonymous}");
+    let authenticated =
+        https_get(&certs, &addr, Some("client")).await.expect("authenticated request");
+    assert!(authenticated.starts_with("HTTP/1.1 200"), "{authenticated}");
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn mtls_rejects_untrusted_client_certificate() {
+    // Optional (1) still verifies a presented cert, so a self-signed cert not
+    // chained to the CA is rejected in both modes, not just required (2).
+    let certs = TestCerts::generate();
+    for cert_reqs in [1, 2] {
+        let (addr, shutdown) = spawn_server(Some(server_tls(&certs, cert_reqs))).await;
+        let result = https_get(&certs, &addr, Some("untrusted_client")).await;
+        assert!(
+            result.is_err(),
+            "cert_reqs={cert_reqs}: untrusted client cert must be rejected"
+        );
+        shutdown.cancel();
+    }
+}
+
+#[tokio::test]
+async fn plain_http_serves_when_tls_is_disabled() {
+    let (addr, shutdown) = spawn_server(None).await;
+    let response = plain_get(&addr).await.expect("http request");
+    assert!(response.starts_with("HTTP/1.1 200"), "{response}");
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn tls_handshake_timeout_drops_silent_client() {
+    // Silent client (no ClientHello) must be dropped at the handshake deadline.
+    let certs = TestCerts::generate();
+    let timeouts = ConnectionTimeouts {
+        handshake: Duration::from_millis(150),
+        header_read: Duration::from_secs(5),
+        keep_alive_enabled: true,
+    };
+    let (addr, shutdown) = spawn_server_with_timeouts(Some(server_tls(&certs, 0)), timeouts).await;
+
+    let mut tcp = TcpStream::connect(&addr).await.expect("connect");
+    let mut buf = [0u8; 1];
+    let read = tokio::time::timeout(Duration::from_secs(5), tcp.read(&mut buf)).await;
+    assert!(
+        matches!(read, Ok(Ok(0)) | Ok(Err(_))),
+        "server must drop a stalled TLS handshake (expected close, got {read:?})"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn keep_alive_timeout_closes_idle_connection() {
+    // Idle keep-alive connection must be closed at the deadline.
+    let timeouts = ConnectionTimeouts {
+        handshake: Duration::from_secs(60),
+        header_read: Duration::from_millis(150),
+        keep_alive_enabled: true,
+    };
+    let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
+
+    let mut tcp = TcpStream::connect(&addr).await.expect("connect");
+    // No `Connection: close`, so it stays alive until the idle deadline.
+    tcp.write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
+        .await
+        .expect("write request");
+
+    let drained = tokio::time::timeout(Duration::from_secs(5), async {
+        let mut buf = [0u8; 1024];
+        loop {
+            match tcp.read(&mut buf).await {
+                Ok(0) => return Ok(()),
+                Ok(_) => continue,
+                Err(err) => return Err(err),
+            }
+        }
+    })
+    .await;
+    assert!(
+        matches!(drained, Ok(Ok(()))),
+        "server must close an idle keep-alive connection (got {drained:?})"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn keep_alive_timeout_closes_idle_tls_connection() {
+    // The keep-alive idle bound lives in serve_connections, below TLS; assert it
+    // still fires through tls-listener's post-handshake SslStream, not just plaintext.
+    let certs = TestCerts::generate();
+    let timeouts = ConnectionTimeouts {
+        handshake: Duration::from_secs(60),
+        header_read: Duration::from_millis(150),
+        keep_alive_enabled: true,
+    };
+    let (addr, shutdown) = spawn_server_with_timeouts(Some(server_tls(&certs, 0)), timeouts).await;
+
+    let mut stream = connect_tls(&certs, &addr, None).await.expect("handshake");
+    // No `Connection: close`, so the connection stays alive until the idle deadline.
+    stream
+        .write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
+        .await
+        .expect("write request");
+
+    let closed = tokio::time::timeout(Duration::from_secs(5), async {
+        let mut buf = [0u8; 1024];
+        loop {
+            // A clean close_notify (Ok(0)) or an abrupt TLS EOF both mean the
+            // server closed; only the outer timeout (still open) is a failure.
+            match stream.read(&mut buf).await {
+                Ok(0) | Err(_) => break,
+                Ok(_) => continue,
+            }
+        }
+    })
+    .await;
+    assert!(
+        closed.is_ok(),
+        "server must close an idle keep-alive TLS connection at the deadline"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn idle_timeout_closes_silent_client() {
+    // Silent client closed by the header-read timeout (http1-only arms it from byte 0).
+    let timeouts = ConnectionTimeouts {
+        handshake: Duration::from_secs(60),
+        header_read: Duration::from_millis(150),
+        keep_alive_enabled: true,
+    };
+    let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
+
+    let mut tcp = TcpStream::connect(&addr).await.expect("connect");
+    let mut buf = [0u8; 1];
+    let read = tokio::time::timeout(Duration::from_secs(5), tcp.read(&mut buf)).await;
+    assert!(
+        matches!(read, Ok(Ok(0)) | Ok(Err(_))),
+        "server must close a silent client (expected close, got {read:?})"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn keep_alive_zero_disables_keep_alive() {
+    // 0 disables keep-alive (serve, then close), like uvicorn's timeout_keep_alive=0.
+    let timeouts = ConnectionTimeouts {
+        handshake: Duration::from_secs(60),
+        header_read: Duration::from_secs(5),
+        keep_alive_enabled: false,
+    };
+    let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
+
+    let mut tcp = TcpStream::connect(&addr).await.expect("connect");
+    tcp.write_all(b"GET /health HTTP/1.1\r\nHost: 127.0.0.1\r\n\r\n")
+        .await
+        .expect("write request");
+
+    let mut response = String::new();
+    let read =
+        tokio::time::timeout(Duration::from_secs(5), tcp.read_to_string(&mut response)).await;
+    assert!(
+        read.is_ok(),
+        "server must close after one response, not hang"
+    );
+    assert!(response.starts_with("HTTP/1.1 200"), "{response}");
+    // Assert `Connection: close`, not just 200: a 0 header-read timeout would also
+    // serve an immediate request, so 200 alone wouldn't prove keep-alive is off.
+    assert!(
+        response.to_ascii_lowercase().contains("connection: close"),
+        "keep-alive must be disabled (expected Connection: close): {response}"
+    );
+    shutdown.cancel();
+}
+
+#[tokio::test]
+async fn disabled_keep_alive_still_closes_silent_client() {
+    // Even with keep-alive off, the head read stays bounded, so a silent client
+    // is dropped rather than held open.
+    let timeouts = ConnectionTimeouts {
+        handshake: Duration::from_secs(60),
+        header_read: Duration::from_millis(150),
+        keep_alive_enabled: false,
+    };
+    let (addr, shutdown) = spawn_server_with_timeouts(None, timeouts).await;
+
+    let mut tcp = TcpStream::connect(&addr).await.expect("connect");
+    let mut buf = [0u8; 1];
+    let read = tokio::time::timeout(Duration::from_secs(5), tcp.read(&mut buf)).await;
+    assert!(
+        matches!(read, Ok(Ok(0)) | Ok(Err(_))),
+        "disabled keep-alive must still close a silent client (got {read:?})"
+    );
+    shutdown.cancel();
+}
@@ -777,6 +777,7 @@ class precompiled_wheel_utils:
                            "vllm/vllm_flash_attn/_vllm_fa3_C.abi3.so",
                            "vllm/cumem_allocator.abi3.so",
                            "vllm/spinloop.abi3.so",
+                            "vllm/fs_io_C.abi3.so",
                            # ROCm-specific libraries
                            "vllm/_rocm_C.abi3.so",
                        }
@@ -1104,6 +1105,7 @@ if _is_cuda() or _is_hip():

 if sys.version_info >= (3, 11):
    ext_modules.append(CMakeExtension(name="vllm.spinloop"))
+    ext_modules.append(CMakeExtension(name="vllm.fs_io_C"))

 if _is_hip():
    ext_modules.append(CMakeExtension(name="vllm._rocm_C"))
@@ -23,7 +23,7 @@ def test_python_error():
    error happening from the C++ side.
    """
    allocator = get_mem_allocator_instance()
-    total_bytes = current_platform.mem_get_info()[1]
+    total_bytes = torch.accelerator.get_memory_info()[1]
    alloc_bytes = int(total_bytes * 0.7)
    tensors = []
    with allocator.use_memory_pool():
@@ -64,9 +64,9 @@ def test_basic_cumem():
    output = x + y + z
    assert torch.allclose(output, torch.ones_like(output) * 3)

-    free_bytes = current_platform.mem_get_info()[0]
+    free_bytes = torch.accelerator.get_memory_info()[0]
    allocator.sleep()
-    free_bytes_after_sleep = current_platform.mem_get_info()[0]
+    free_bytes_after_sleep = torch.accelerator.get_memory_info()[0]
    assert free_bytes_after_sleep > free_bytes
    allocator.wake_up()

@@ -99,9 +99,9 @@ def test_cumem_with_cudagraph():
    with torch.cuda.graph(model_graph):
        y = model(x)

-    free_bytes = current_platform.mem_get_info()[0]
+    free_bytes = torch.accelerator.get_memory_info()[0]
    allocator.sleep()
-    free_bytes_after_sleep = current_platform.mem_get_info()[0]
+    free_bytes_after_sleep = torch.accelerator.get_memory_info()[0]
    assert free_bytes_after_sleep > free_bytes
    allocator.wake_up()

@@ -132,7 +132,7 @@ def test_cumem_with_cudagraph():
    ],
 )
 def test_end_to_end(model: str):
-    free, total = current_platform.mem_get_info()
+    free, total = torch.accelerator.get_memory_info()
    used_bytes_baseline = total - free  # in case other process is running
    llm = LLM(model, enable_sleep_mode=True)
    prompt = "How are you?"
@@ -144,7 +144,7 @@ def test_end_to_end(model: str):
    # test sleep level 1 here.
    llm.sleep(level=1)

-    free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
+    free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
    used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
    # now the memory usage is mostly cudagraph memory pool,
    # and it should be less than the model weights (1B model, 2GiB weights)
@@ -164,7 +164,7 @@ def test_end_to_end(model: str):
    llm.sleep(level=1)
    llm.wake_up(tags=["weights"])

-    free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
+    free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
    used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline

    # should just reallocate memory for weights (1B model, ~2GiB weights)
@@ -181,7 +181,7 @@ def test_end_to_end(model: str):
@create_new_process_for_each_test()
 def test_deep_sleep():
    model = "hmellor/tiny-random-LlamaForCausalLM"
-    free, total = current_platform.mem_get_info()
+    free, total = torch.accelerator.get_memory_info()
    used_bytes_baseline = total - free  # in case other process is running
    llm = LLM(model, enable_sleep_mode=True)
    prompt = "How are you?"
@@ -191,13 +191,13 @@ def test_deep_sleep():
    # Put the engine to deep sleep
    llm.sleep(level=2)

-    free_gpu_bytes_after_sleep, total = current_platform.mem_get_info()
+    free_gpu_bytes_after_sleep, total = torch.accelerator.get_memory_info()
    used_bytes = total - free_gpu_bytes_after_sleep - used_bytes_baseline
    assert used_bytes < 3 * GiB_bytes

    llm.wake_up(tags=["weights"])
    llm.collective_rpc("reload_weights")
-    free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
+    free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
    used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
    assert used_bytes < 4 * GiB_bytes

@@ -213,7 +213,7 @@ def test_deep_sleep():
 def test_deep_sleep_async():
    async def test():
        model = "hmellor/tiny-random-LlamaForCausalLM"
-        free, total = current_platform.mem_get_info()
+        free, total = torch.accelerator.get_memory_info()
        used_bytes_baseline = total - free  # in case other process is running
        engine_args = AsyncEngineArgs(
            model=model,
@@ -232,7 +232,7 @@ def test_deep_sleep_async():

        await llm.wake_up(tags=["weights"])
        await llm.collective_rpc("reload_weights")
-        free_gpu_bytes_wake_up_w, total = current_platform.mem_get_info()
+        free_gpu_bytes_wake_up_w, total = torch.accelerator.get_memory_info()
        used_bytes = total - free_gpu_bytes_wake_up_w - used_bytes_baseline
        assert used_bytes < 4 * GiB_bytes

@@ -1587,7 +1587,13 @@ class AssetHandler(http.server.BaseHTTPRequestHandler):
        self.send_header("Content-Type", ctype)
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
-        self.wfile.write(data)
+        try:
+            self.wfile.write(data)
+        except (BrokenPipeError, ConnectionResetError) as e:
+            logger.debug(
+                "Client disconnected while serving test asset %s: %r", filename, e
+            )
+            self.close_connection = True


 def _find_free_port() -> int:
@@ -29,9 +29,43 @@ from vllm.distributed.weight_transfer.nccl_engine import (
    NCCLWeightTransferInitInfo,
    NCCLWeightTransferUpdateInfo,
 )
+from vllm.platforms import current_platform
 from vllm.utils.network_utils import get_open_port


+def _weight_transfer_ray_env_vars() -> dict[str, str]:
+    if not current_platform.is_rocm():
+        return {}
+
+    return {
+        "RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES": "1",
+        "RAY_EXPERIMENTAL_NOSET_HIP_VISIBLE_DEVICES": "1",
+        "RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES": "1",
+    }
+
+
+def _init_ray_for_weight_transfer() -> None:
+    if ray.is_initialized():
+        return
+    ray.init(
+        ignore_reinit_error=True,
+        runtime_env={"env_vars": _weight_transfer_ray_env_vars()},
+    )
+
+
+def _get_ray_assigned_device() -> torch.device:
+    gpu_ids = ray.get_gpu_ids()
+    if not gpu_ids:
+        return torch.device("cuda:0")
+    return torch.device(f"cuda:{int(gpu_ids[0])}")
+
+
+def _set_ray_assigned_device() -> torch.device:
+    device = _get_ray_assigned_device()
+    torch.accelerator.set_device(device)
+    return device
+
+
 def create_mock_parallel_config(
    rank: int = 0,
    world_size: int = 1,
@@ -321,6 +355,8 @@ def trainer_broadcast_tensor(
    """Trainer task that broadcasts a tensor via NCCL."""
    import torch

+    device = _set_ray_assigned_device()
+
    from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
    from vllm.distributed.utils import StatelessProcessGroup

@@ -331,12 +367,11 @@ def trainer_broadcast_tensor(
        rank=0,
        world_size=world_size,
    )
-    # Ray sets CUDA_VISIBLE_DEVICES, so device 0 is the assigned GPU
-    comm = PyNcclCommunicator(pg, device=0)
+    comm = PyNcclCommunicator(pg, device=device.index)

    # Create and broadcast the tensor
    dtype = getattr(torch, tensor_dtype)
-    tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device="cuda:0")
+    tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device=device)
    comm.broadcast(tensor_to_send, src=0, stream=torch.cuda.current_stream())
    torch.accelerator.synchronize()

@@ -356,6 +391,8 @@ def inference_receive_tensor(

    import torch

+    _set_ray_assigned_device()
+
    from vllm.config.parallel import ParallelConfig
    from vllm.config.weight_transfer import WeightTransferConfig
    from vllm.distributed.weight_transfer.nccl_engine import (
@@ -435,7 +472,7 @@ def test_nccl_weight_transfer_between_processes():
    This test verifies that the NCCLWeightTransferEngine can receive
    tensors broadcast by a trainer process via NCCL.
    """
-    ray.init(ignore_reinit_error=True)
+    _init_ray_for_weight_transfer()

    master_address = "127.0.0.1"
    master_port = get_open_port()
@@ -473,6 +510,8 @@ def trainer_broadcast_sparse_tensor(
    """Trainer task that broadcasts sparse patches via NCCL."""
    import torch

+    device = _set_ray_assigned_device()
+
    from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
    from vllm.distributed.utils import StatelessProcessGroup
    from vllm.distributed.weight_transfer.base import SparseWeightPatch
@@ -487,12 +526,12 @@ def trainer_broadcast_sparse_tensor(
        rank=0,
        world_size=world_size,
    )
-    comm = PyNcclCommunicator(pg, device=0)
+    comm = PyNcclCommunicator(pg, device=device.index)

    patch = SparseWeightPatch(
        name="test.weight",
-        indices=torch.tensor([1, 7, 25], dtype=torch.int32, device="cuda:0"),
-        values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device="cuda:0"),
+        indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
+        values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
    )
    NCCLWeightTransferEngine.trainer_send_sparse_weights(
        iter([patch]),
@@ -513,6 +552,8 @@ def inference_receive_sparse_tensor(

    import torch

+    device = _set_ray_assigned_device()
+
    from vllm.config.parallel import ParallelConfig
    from vllm.config.weight_transfer import WeightTransferConfig
    from vllm.distributed.weight_transfer.nccl_engine import (
@@ -540,7 +581,7 @@ def inference_receive_sparse_tensor(
        )
    )

-    target = torch.zeros(30, dtype=torch.float32, device="cuda")
+    target = torch.zeros(30, dtype=torch.float32, device=device)

    def apply_sparse_patches(patches: list[SparseWeightPatch]):
        for patch in patches:
@@ -556,9 +597,9 @@ def inference_receive_sparse_tensor(
    engine.receive_sparse_weights(update_info, apply_sparse_patches)
    torch.accelerator.synchronize()

-    expected = torch.zeros(30, dtype=torch.float32, device="cuda")
+    expected = torch.zeros(30, dtype=torch.float32, device=device)
    expected[[1, 7, 25]] = torch.tensor(
-        [10.0, 20.0, 30.0], dtype=torch.float32, device="cuda"
+        [10.0, 20.0, 30.0], dtype=torch.float32, device=device
    )
    success = torch.equal(target, expected)
    engine.shutdown()
@@ -574,7 +615,7 @@ def inference_receive_sparse_tensor(
 )
 def test_nccl_sparse_weight_transfer_between_processes():
    """Test NCCL sparse weight transfer from trainer to inference process."""
-    ray.init(ignore_reinit_error=True)
+    _init_ray_for_weight_transfer()

    master_address = "127.0.0.1"
    master_port = get_open_port()
@@ -933,16 +974,18 @@ class TrainerActor:
    """Trainer actor that creates and holds CUDA IPC handles."""

    def __init__(self, tensor_shape: list[int], tensor_dtype: str):
+        device = _set_ray_assigned_device()
+
        # Create tensor on GPU and keep it alive
        dtype = getattr(torch, tensor_dtype)
-        self.tensor = torch.ones(tensor_shape, dtype=dtype, device="cuda:0")
+        self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device)
        self.tensor.fill_(42.0)  # Fill with 42 to verify correct transfer

        # Create IPC handle (tensor must stay alive for IPC to work)
        # reduce_tensor returns (rebuild_func, args); we only send args
        # since the receiver imports rebuild_cuda_tensor directly.
        _, ipc_args = reduce_tensor(self.tensor)
-        gpu_uuid = get_physical_gpu_id(0)
+        gpu_uuid = get_physical_gpu_id(device.index)

        torch.accelerator.synchronize()

@@ -974,6 +1017,8 @@ def inference_receive_ipc_tensor(

    import torch

+    _set_ray_assigned_device()
+
    from vllm.config.parallel import ParallelConfig
    from vllm.config.weight_transfer import WeightTransferConfig
    from vllm.distributed.weight_transfer.ipc_engine import (
@@ -1072,7 +1117,7 @@ def test_ipc_weight_transfer_between_processes(mode: str):
    from ray.util.placement_group import placement_group
    from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy

-    ray.init(ignore_reinit_error=True)
+    _init_ray_for_weight_transfer()

    # Create a placement group to ensure both processes are on the same GPU
    # Use fractional GPUs so both tasks can share the same GPU bundle
@@ -1,5 +1,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+from typing import Any
+
+import pytest

 # Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
 TEST_IMAGE_ASSETS = [
@@ -8,3 +13,70 @@ TEST_IMAGE_ASSETS = [
    "1280px-Venn_diagram_rgb.svg.png",  # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/1280px-Venn_diagram_rgb.svg.png",
    "RGBA_comp.png",  # "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/RGBA_comp.png",
 ]
+
+
+def _shutdown_llm(llm: Any, gpu_memory_utilization: float) -> None:
+    from vllm.distributed import cleanup_dist_env_and_memory
+    from vllm.platforms import current_platform
+
+    try:
+        shutdown_timeout = 60.0 if current_platform.is_rocm() else None
+        llm.llm_engine.engine_core.shutdown(timeout=shutdown_timeout)
+    except Exception:
+        pass
+
+    del llm
+
+    try:
+        import torch
+
+        torch._dynamo.reset()
+    except Exception:
+        pass
+
+    cleanup_dist_env_and_memory()
+
+    if current_platform.is_rocm():
+        from tests.utils import wait_for_rocm_memory_to_settle
+
+        wait_for_rocm_memory_to_settle(threshold_ratio=1.0 - gpu_memory_utilization)
+
+
+@contextmanager
+def managed_llm(*args: Any, **kwargs: Any) -> Iterator[Any]:
+    from vllm import LLM
+
+    llm = LLM(*args, **kwargs)
+    gpu_memory_utilization = (
+        llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
+    )
+    try:
+        yield llm
+    finally:
+        _shutdown_llm(llm, gpu_memory_utilization)
+
+
+def _make_managed_llm_factory() -> Iterator[Callable[..., Any]]:
+    from vllm import LLM
+
+    llms: list[tuple[Any, float]] = []
+
+    def make_llm(*args: Any, **kwargs: Any) -> Any:
+        llm = LLM(*args, **kwargs)
+        gpu_memory_utilization = (
+            llm.llm_engine.vllm_config.cache_config.gpu_memory_utilization
+        )
+        llms.append((llm, gpu_memory_utilization))
+        return llm
+
+    try:
+        yield make_llm
+    finally:
+        while llms:
+            llm, gpu_memory_utilization = llms.pop()
+            _shutdown_llm(llm, gpu_memory_utilization)
+
+
+@pytest.fixture
+def multimodal_llm_factory() -> Iterator[Callable[..., Any]]:
+    yield from _make_managed_llm_factory()
@@ -1,19 +1,13 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import weakref
-
 import pytest

 from tests.entrypoints.multimodal.conftest import TEST_IMAGE_ASSETS
-from vllm import LLM
-from vllm.distributed import cleanup_dist_env_and_memory


@pytest.fixture(scope="function")
-def vision_llm():
-    # pytest caches the fixture so we use weakref.proxy to
-    # enable garbage collection
-    llm = LLM(
+def vision_llm(multimodal_llm_factory):
+    return multimodal_llm_factory(
        model="microsoft/Phi-3.5-vision-instruct",
        max_model_len=4096,
        max_num_seqs=5,
@@ -23,12 +17,6 @@ def vision_llm():
        seed=0,
    )

-    yield weakref.proxy(llm)
-
-    del llm
-
-    cleanup_dist_env_and_memory()
-

@pytest.mark.parametrize(
    "image_urls", [[TEST_IMAGE_ASSETS[0], TEST_IMAGE_ASSETS[1]]], indirect=True
@@ -69,6 +69,7 @@ def test_inject_into_mm_cache(
    image_urls,
    mm_processor_cache_type,
    caplog_vllm,
+    multimodal_llm_factory,
 ):
    """Test that inject_into_mm_cache() injects pre-processed mm_kwargs into
    the processor cache and MM cache hit metrics are updated correctly.
@@ -78,7 +79,7 @@ def test_inject_into_mm_cache(
    2. Extract cached kwargs, call inject_into_mm_cache with a new hash,
       then generate with a pre-rendered input -> verifies injection works
    """
-    llm = LLM(
+    llm = multimodal_llm_factory(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
        max_num_seqs=5,
@@ -145,11 +146,12 @@ def test_inject_into_mm_cache(
 def test_inject_into_mm_cache_without_cache(
    num_gpus_available,
    image_urls,
+    multimodal_llm_factory,
 ):
    """Test that inject_into_mm_cache works gracefully when processor cache
    is disabled (mm_processor_cache_gb=0). Should not crash.
    """
-    llm = LLM(
+    llm = multimodal_llm_factory(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
        max_num_seqs=5,
@@ -61,8 +61,9 @@ def test_mm_cache_stats(
    image_urls,
    mm_processor_cache_type,
    caplog_vllm,
+    multimodal_llm_factory,
 ):
-    llm = LLM(
+    llm = multimodal_llm_factory(
        model="llava-hf/llava-1.5-7b-hf",
        max_model_len=4096,
        max_num_seqs=5,
@@ -1,13 +1,11 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

-import weakref
-
 import pytest

+from tests.entrypoints.multimodal.conftest import managed_llm
 from vllm import LLM, SamplingParams
 from vllm.assets.image import ImageAsset
-from vllm.distributed import cleanup_dist_env_and_memory

 MODEL = "llava-hf/llava-1.5-7b-hf"
 PROMPT = "USER: <image>\nDescribe this image briefly.\nASSISTANT:"
@@ -17,20 +15,15 @@ TEXT_ONLY_PROMPT = "USER: What is 2 + 2?\nASSISTANT:"
@pytest.fixture(scope="module")
 def llm():
    """LLM with enable_mm_embeds=True and all modality limits zeroed out."""
-    llm = LLM(
+    with managed_llm(
        model=MODEL,
        max_model_len=2048,
        enforce_eager=True,
        gpu_memory_utilization=0.8,
        enable_mm_embeds=True,
        limit_mm_per_prompt={"image": 0},
-    )
-
-    yield weakref.proxy(llm)
-
-    del llm
-
-    cleanup_dist_env_and_memory()
+    ) as llm:
+        yield llm


@pytest.mark.skip_global_cleanup
@@ -17,10 +17,9 @@ from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat
 from vllm.entrypoints.openai.engine.protocol import GenerationError
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.render.serving import ServingRender
+from vllm.entrypoints.scale_out.render.serving import ServingRender
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers.hf import HfRenderer
-from vllm.renderers.online_derenderer import OnlineDerenderer
 from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.tokenizers.registry import cached_tokenizer_from_config
 from vllm.v1.engine.async_llm import AsyncLLM
@@ -206,15 +205,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
        chat_template=None,
        chat_template_content_format="auto",
    )
-    online_derenderer = OnlineDerenderer(
-        model_config=engine.model_config,
-        renderer=engine.renderer,
-        request_logger=None,
-        chat_template=None,
-        chat_template_content_format="auto",
-    )

-    serving_render = ServingRender(models, online_renderer, online_derenderer)
+    serving_render = ServingRender(models, online_renderer)

    async def _fake_preprocess_chat(*args, **kwargs):
        # return conversation, engine_inputs
@@ -76,3 +76,60 @@ async def test_chat_logit_bias_invalid(client):
    assert error.status_code == 400
    assert str(invalid_token_id) in error_message
    assert str(vocab_size) in error_message
+
+
+@pytest.mark.asyncio
+async def test_chat_logit_bias_non_integer_key(client):
+    """Test that a non-integer logit_bias key is rejected with a clean,
+    informative error instead of a raw 'invalid literal for int()' message."""
+    with pytest.raises(openai.BadRequestError) as excinfo:
+        await client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[{"role": "user", "content": "Testing invalid logit bias key"}],
+            max_tokens=5,
+            logit_bias={"not_a_token_id": 50},
+        )
+
+    error = excinfo.value
+    error_message = str(error)
+
+    assert error.status_code == 400
+    assert "not_a_token_id" in error_message
+    assert "logit_bias" in error_message
+
+
+@pytest.mark.asyncio
+async def test_chat_logit_bias_non_numeric_value(client):
+    """Test that a non-numeric logit_bias value is rejected with a message
+    that names the specific offending token, not just a generic TypeError."""
+    with pytest.raises(openai.BadRequestError) as excinfo:
+        await client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[{"role": "user", "content": "Testing invalid logit bias value"}],
+            max_tokens=5,
+            logit_bias={"1": "not_a_number"},
+        )
+
+    error = excinfo.value
+    error_message = str(error)
+
+    assert error.status_code == 400
+    assert "logit_bias" in error_message
+
+
+@pytest.mark.asyncio
+async def test_chat_logit_bias_multiple_non_integer_keys(client):
+    """Test that ALL invalid logit_bias keys are reported together,
+    not just the first one encountered."""
+    with pytest.raises(openai.BadRequestError) as excinfo:
+        await client.chat.completions.create(
+            model=MODEL_NAME,
+            messages=[{"role": "user", "content": "Testing multiple bad keys"}],
+            max_tokens=5,
+            logit_bias={"bad1": 50.0, "bad2": 20.0},
+        )
+
+    error_message = str(excinfo.value)
+    assert excinfo.value.status_code == 400
+    assert "bad1" in error_message
+    assert "bad2" in error_message
@@ -14,10 +14,9 @@ from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
 from vllm.entrypoints.openai.engine.protocol import GenerationError
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.render.serving import ServingRender
+from vllm.entrypoints.scale_out.render.serving import ServingRender
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers.hf import HfRenderer
-from vllm.renderers.online_derenderer import OnlineDerenderer
 from vllm.renderers.online_renderer import OnlineRenderer
 from vllm.tokenizers.registry import cached_tokenizer_from_config
 from vllm.v1.engine.async_llm import AsyncLLM
@@ -191,15 +190,8 @@ def _build_serving_render(engine: AsyncLLM) -> ServingRender:
        chat_template=None,
        chat_template_content_format="auto",
    )
-    online_derenderer = OnlineDerenderer(
-        model_config=engine.model_config,
-        renderer=engine.renderer,
-        request_logger=None,
-        chat_template=None,
-        chat_template_content_format="auto",
-    )

-    serving_render = ServingRender(models, online_renderer, online_derenderer)
+    serving_render = ServingRender(models, online_renderer)

    async def _fake_preprocess_chat(*args, **kwargs):
        # return conversation, engine_inputs
@@ -142,6 +142,7 @@ class TestHarmonyToResponseOutput:
        )
        assert output_items[0].call_id.startswith("call_")
        assert output_items[0].id.startswith("fc_")
+        assert output_items[0].status == "completed"

    def test_commentary_with_python_recipient_creates_reasoning(self):
        """Test that commentary with recipient='python' creates reasoning items."""
@@ -3,7 +3,7 @@
 """Unit tests for the token-offsets request/response protocol wiring:
 the request flag flowing into ``TokenizeParams`` and the ``GenerateRequest``
 serialization boundary. End-to-end behavior is covered by
-``tests/entrypoints/serve/render/test_render.py``; plain Pydantic field
+``tests/entrypoints/scale_out/render/test_render.py``; plain Pydantic field
 storage is not retested here.
 """

@@ -12,7 +12,7 @@ from unittest.mock import Mock
 from vllm.config import ModelConfig
 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
 from vllm.entrypoints.openai.completion.protocol import CompletionRequest
-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
 from vllm.sampling_params import SamplingParams


@@ -0,0 +1,160 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""
+Unit tests for stop_token_ids propagation from default_sampling_params
+to SamplingParams in ChatCompletionRequest and CompletionRequest.
+
+Regression test for https://github.com/vllm-project/vllm/issues/22519
+where gpt-oss model stop tokens (e.g., </call> = 200012) were loaded into
+default_sampling_params at server startup but silently discarded on every
+request because to_sampling_params() never fell back to defaults.
+"""
+
+import pytest
+
+from vllm.entrypoints.openai.chat_completion.protocol import (
+    ChatCompletionRequest,
+)
+from vllm.entrypoints.openai.completion.protocol import (
+    CompletionRequest,
+)
+
+
+class TestChatCompletionStopTokenIds:
+    """Test stop_token_ids merging in ChatCompletionRequest.to_sampling_params()."""
+
+    @pytest.fixture
+    def minimal_chat_request(self):
+        return ChatCompletionRequest(
+            model="test-model",
+            messages=[{"role": "user", "content": "hello"}],
+        )
+
+    def test_default_stop_token_ids_applied(self, minimal_chat_request):
+        """Server-default stop_token_ids are applied when client sends none."""
+        default_sampling_params = {
+            "stop_token_ids": [200012, 200002],
+        }
+
+        sampling_params = minimal_chat_request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params=default_sampling_params,
+        )
+
+        assert set(sampling_params.stop_token_ids) == {200012, 200002}
+
+    def test_client_stop_token_ids_merged_with_defaults(self):
+        """Client-specified stop_token_ids are merged with server defaults."""
+        request = ChatCompletionRequest(
+            model="test-model",
+            messages=[{"role": "user", "content": "hello"}],
+            stop_token_ids=[99999],
+        )
+        default_sampling_params = {
+            "stop_token_ids": [200012, 200002],
+        }
+
+        sampling_params = request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params=default_sampling_params,
+        )
+
+        assert set(sampling_params.stop_token_ids) == {200012, 200002, 99999}
+        assert sampling_params.stop_token_ids == [99999, 200012, 200002]
+
+    def test_no_stop_token_ids_anywhere(self, minimal_chat_request):
+        """When neither client nor server specifies stop_token_ids, result is empty."""
+        sampling_params = minimal_chat_request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params={},
+        )
+
+        assert not sampling_params.stop_token_ids
+
+    def test_only_client_stop_token_ids(self):
+        """Client stop_token_ids work when no server defaults exist."""
+        request = ChatCompletionRequest(
+            model="test-model",
+            messages=[{"role": "user", "content": "hello"}],
+            stop_token_ids=[42, 43],
+        )
+
+        sampling_params = request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params={},
+        )
+
+        assert set(sampling_params.stop_token_ids) == {42, 43}
+
+    def test_duplicate_stop_token_ids_deduplicated(self):
+        """Overlapping stop_token_ids between client and server are deduplicated."""
+        request = ChatCompletionRequest(
+            model="test-model",
+            messages=[{"role": "user", "content": "hello"}],
+            stop_token_ids=[200012, 55555],
+        )
+        default_sampling_params = {
+            "stop_token_ids": [200012, 200002],
+        }
+
+        sampling_params = request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params=default_sampling_params,
+        )
+
+        assert set(sampling_params.stop_token_ids) == {200012, 200002, 55555}
+        assert sampling_params.stop_token_ids == [200012, 55555, 200002]
+        assert len(sampling_params.stop_token_ids) == 3
+
+
+class TestCompletionStopTokenIds:
+    """Test stop_token_ids merging in CompletionRequest.to_sampling_params()."""
+
+    @pytest.fixture
+    def minimal_completion_request(self):
+        return CompletionRequest(
+            model="test-model",
+            prompt="hello",
+        )
+
+    def test_default_stop_token_ids_applied(self, minimal_completion_request):
+        """Server-default stop_token_ids are applied when client sends none."""
+        default_sampling_params = {
+            "stop_token_ids": [200012, 200002],
+        }
+
+        sampling_params = minimal_completion_request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params=default_sampling_params,
+        )
+
+        assert set(sampling_params.stop_token_ids) == {200012, 200002}
+
+    def test_client_stop_token_ids_merged_with_defaults(self):
+        """Client-specified stop_token_ids are merged with server defaults."""
+        request = CompletionRequest(
+            model="test-model",
+            prompt="hello",
+            stop_token_ids=[99999],
+        )
+        default_sampling_params = {
+            "stop_token_ids": [200012, 200002],
+        }
+
+        sampling_params = request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params=default_sampling_params,
+        )
+
+        assert set(sampling_params.stop_token_ids) == {200012, 200002, 99999}
+        assert sampling_params.stop_token_ids == [99999, 200012, 200002]
+
+    def test_no_stop_token_ids_anywhere(self, minimal_completion_request):
+        """When neither client nor server specifies stop_token_ids, result is empty."""
+        sampling_params = minimal_completion_request.to_sampling_params(
+            max_tokens=100,
+            default_sampling_params={},
+        )
+
+        assert not sampling_params.stop_token_ids
@@ -12,11 +12,11 @@ from vllm.config.multimodal import MultiModalConfig
 from vllm.entrypoints.openai.engine.protocol import StreamOptions
 from vllm.entrypoints.openai.models.protocol import BaseModelPath
 from vllm.entrypoints.openai.models.serving import OpenAIServingModels
-from vllm.entrypoints.serve.disagg.protocol import (
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
    GenerateRequest,
    GenerateResponse,
 )
-from vllm.entrypoints.serve.disagg.serving import ServingTokens
+from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
 from vllm.logprobs import Logprob
 from vllm.outputs import CompletionOutput, RequestOutput
 from vllm.renderers import renderer_from_config
@@ -1,14 +1,17 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Roundtrip tests for multimodal serde used by the disagg generate endpoint."""
+"""
+Roundtrip tests for multimodal serde used by the
+token_in_token_out generate endpoint.
+"""

 import torch

-from vllm.entrypoints.serve.disagg.mm_serde import (
+from vllm.entrypoints.scale_out.token_in_token_out.mm_serde import (
    decode_mm_kwargs_item,
    encode_mm_kwargs_item,
 )
-from vllm.entrypoints.serve.disagg.protocol import (
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import (
    MultiModalFeatures,
    PlaceholderRangeInfo,
 )
@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for the disagg request/response protocol.
+"""Unit tests for the token_in_token_out request/response protocol.

 These tests intentionally avoid spinning up a server — they exercise the
 pydantic validators on ``GenerateRequest`` directly so they run fast and
@@ -9,7 +9,7 @@ fail loudly if the validator semantics ever drift.

 import json

-from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
+from vllm.entrypoints.scale_out.token_in_token_out.protocol import GenerateRequest
 from vllm.sampling_params import SamplingParams


@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

-from vllm.entrypoints.serve.disagg.serving import ServingTokens
+from vllm.entrypoints.scale_out.token_in_token_out.serving import ServingTokens
 from vllm.logprobs import Logprob


@@ -33,6 +33,8 @@ QUARK_MXFP4_TORCH_COMPATIBLE = find_spec("quark") is not None and (
    else True
 )

+DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200
+

 def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict:
    """Run GSM8K evaluation using our isolated script."""
@@ -127,7 +129,11 @@ def test_gsm8k_correctness(config_filename):
        ]
    )

-    env_dict = eval_config.get("env", None)
+    startup_max_wait_seconds = eval_config.get(
+        "startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS
+    )
+    env_dict = dict(eval_config.get("env") or {})
+    env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds))

    print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}")
    print(f"Expected metric threshold: {eval_config['accuracy_threshold']}")
@@ -139,6 +145,7 @@ def test_gsm8k_correctness(config_filename):
            "rocm_request_timeout_seconds", request_timeout_seconds
        )
    print(f"Request timeout: {request_timeout_seconds}s")
+    print(f"Startup max wait: {startup_max_wait_seconds}s")
    print(f"Server args: {' '.join(server_args)}")
    print(f"Environment variables: {env_dict}")

@@ -147,7 +154,7 @@ def test_gsm8k_correctness(config_filename):
        eval_config["model_name"],
        server_args,
        env_dict=env_dict,
-        max_wait_seconds=eval_config.get("startup_max_wait_seconds", 600),
+        max_wait_seconds=startup_max_wait_seconds,
    ) as remote_server:
        server_url = remote_server.url_for("v1")
        print(f"Server started at: {server_url}")
@@ -0,0 +1,261 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the fused_qk_norm_rope helion kernel
+
+Run `pytest tests/kernels/helion/test_fused_qk_norm_rope.py`.
+"""
+
+from typing import Any
+
+import pytest
+import torch
+from torch._subclasses.fake_tensor import FakeTensorMode
+
+from tests.kernels.helion.utils import skip_if_platform_unsupported
+from vllm.benchmarks.lib.utils import default_vllm_config
+from vllm.kernels.helion.case_key import CaseKey
+from vllm.kernels.helion.config_manager import ConfigManager
+from vllm.kernels.helion.ops.fused_qk_norm_rope import (
+    _pick_cache,
+    baseline,
+    fused_qk_norm_rope,
+    pick_config,
+)
+from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
+from vllm.utils.import_utils import has_helion
+
+if not has_helion():
+    pytest.skip(
+        "Helion is not installed. Install with: pip install vllm[helion]",
+        allow_module_level=True,
+    )
+
+
+@default_vllm_config()
+def _generate_fake_input(
+    num_tokens: int, num_q_heads: int, num_kv_heads: int
+) -> tuple[Any, ...]:
+    with FakeTensorMode():
+        head_dim = 128
+        eps = 1e-6
+        is_neox = True
+        rotary_ratio = 1.0
+        device = "cuda"
+        dtype = torch.bfloat16
+        total_dim = (num_q_heads + 2 * num_kv_heads) * head_dim
+        qkv = torch.randn(num_tokens, total_dim, dtype=dtype, device=device)
+        positions = torch.arange(num_tokens, dtype=torch.long, device=device)
+        q_weight = torch.normal(
+            mean=1.0,
+            std=1.0,
+            size=(head_dim,),
+            dtype=qkv.dtype,
+            device=device,
+        )
+        k_weight = torch.normal(
+            mean=1.0,
+            std=1.0,
+            size=(head_dim,),
+            dtype=qkv.dtype,
+            device=device,
+        )
+        rotary_dim = int(head_dim * rotary_ratio)
+        rope = RotaryEmbedding(
+            head_size=head_dim,
+            rotary_dim=rotary_dim,
+            max_position_embeddings=4096,
+            base=10000.0,
+            is_neox_style=is_neox,
+            dtype=dtype,
+        ).to(device)
+        args = (
+            qkv,
+            num_q_heads,
+            num_kv_heads,
+            num_kv_heads,
+            head_dim,
+            eps,
+            q_weight,
+            k_weight,
+            rope.cos_sin_cache,
+            is_neox,
+            positions.view(-1),
+        )
+        return args
+
+
+@pytest.fixture(autouse=True)
+def reset_config_manager_singleton():
+    ConfigManager.reset_instance()
+    ConfigManager()
+    yield
+    ConfigManager.reset_instance()
+
+
+class TestFusedQkNormRopeConfigPicker:
+    def setup_method(self):
+        _pick_cache.clear()
+
+    def test_config_picker_exact_match(self):
+        config_keys = [
+            CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
+            CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
+        ]
+
+        args = _generate_fake_input(16, 4096, 128)
+        selected_key = pick_config(args, config_keys)
+        assert selected_key == CaseKey(
+            {"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}
+        )
+
+    def test_config_picker_closest_match(self):
+        config_keys = [
+            CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
+            CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
+            CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
+            CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
+            CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
+            CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
+            CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
+            CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
+        ]
+
+        args = _generate_fake_input(20, 3000, 70)
+        selected_key = pick_config(args, config_keys)
+        assert selected_key == CaseKey(
+            {"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}
+        )
+
+    def test_config_picker_no_configs(self):
+        config_keys: list[dict] = []
+
+        args = _generate_fake_input(16, 4096, 128)
+        selected_key = pick_config(args, config_keys)
+        assert selected_key is None
+
+    def test_config_picker_fallback_to_largest(self):
+        config_keys = [
+            CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 16}),
+            CaseKey({"q_heads": 2048, "kv_heads": 64, "num_tokens": 32}),
+            CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 16}),
+            CaseKey({"q_heads": 2048, "kv_heads": 128, "num_tokens": 32}),
+            CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 16}),
+            CaseKey({"q_heads": 4096, "kv_heads": 64, "num_tokens": 32}),
+            CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 16}),
+            CaseKey({"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}),
+        ]
+
+        args = _generate_fake_input(64, 8192, 256)
+        selected_key = pick_config(args, config_keys)
+        assert selected_key == CaseKey(
+            {"q_heads": 4096, "kv_heads": 128, "num_tokens": 32}
+        )
+
+
+class TestFusedQkNormRopeCorrectness:
+    @pytest.mark.parametrize(
+        "num_heads, num_kv_heads, head_dim", [(16, 4, 128), (64, 8, 128)]
+    )
+    @pytest.mark.parametrize("num_tokens", [1, 7, 1024, 1025])
+    @pytest.mark.parametrize("is_neox", [False, True])
+    @pytest.mark.parametrize("rotary_ratio", [1.0, 0.5, 0.25])
+    @pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
+    @default_vllm_config()
+    def test_fused_qk_norm_rope(
+        self,
+        num_heads: int,
+        num_kv_heads: int,
+        head_dim: int,
+        num_tokens: int,
+        is_neox: bool,
+        rotary_ratio: float,
+        dtype: torch.dtype,
+    ):
+        skip_if_platform_unsupported("fused_qk_norm_rope")
+
+        torch.manual_seed(42)
+        eps = 1e-6
+        device = "cuda"
+        total_dim = (num_heads + 2 * num_kv_heads) * head_dim
+        ref_qkv = torch.empty(
+            num_tokens, total_dim, dtype=dtype, device=device
+        ).uniform_(-0.1, 0.1)
+        ops_qkv = ref_qkv.clone()
+        positions = torch.arange(num_tokens, dtype=torch.long, device=device)
+        q_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
+        k_weight = torch.empty(head_dim, dtype=dtype, device=device).uniform_(0.8, 1.2)
+        rotary_dim = int(head_dim * rotary_ratio)
+        rope = RotaryEmbedding(
+            head_size=head_dim,
+            rotary_dim=rotary_dim,
+            max_position_embeddings=40960,
+            base=10000.0,
+            is_neox_style=is_neox,
+            dtype=dtype,
+        ).to(device)
+
+        baseline(
+            ref_qkv,
+            num_heads,
+            num_kv_heads,
+            num_kv_heads,
+            head_dim,
+            eps,
+            q_weight,
+            k_weight,
+            rope.cos_sin_cache,
+            is_neox,
+            positions.view(-1),
+        )
+
+        fused_qk_norm_rope(
+            ops_qkv,
+            num_heads,
+            num_kv_heads,
+            num_kv_heads,
+            head_dim,
+            eps,
+            q_weight,
+            k_weight,
+            rope.cos_sin_cache,
+            is_neox,
+            positions.view(-1),
+        )
+
+        if dtype == torch.bfloat16:
+            atol = 5e-2
+            rtol = 5e-2
+        else:
+            atol = 1e-2
+            rtol = 1e-2
+
+        torch.testing.assert_close(
+            ref_qkv,
+            ops_qkv,
+            atol=atol,
+            rtol=rtol,
+        )
+
+
+class TestFusedQkNormRopeIntegration:
+    def test_kernel_registration_integration(self):
+        from vllm.kernels.helion.register import get_registered_kernels
+
+        registered_kernels = get_registered_kernels()
+        assert "fused_qk_norm_rope" in registered_kernels
+
+        kernel_wrapper = registered_kernels["fused_qk_norm_rope"]
+        assert kernel_wrapper.op_name == "fused_qk_norm_rope"
+        assert kernel_wrapper._config_picker is not None
+        assert kernel_wrapper._mutates_args == ["qkv"]
+
+    def test_fake_impl_functionality(self):
+        skip_if_platform_unsupported("fused_qk_norm_rope")
+        from vllm.kernels.helion.register import get_registered_kernels
+
+        registered_kernels = get_registered_kernels()
+        kernel_wrapper = registered_kernels["fused_qk_norm_rope"]
+        fake_impl = kernel_wrapper._fake_impl
+
+        args = _generate_fake_input(16, 4096, 128)
+        assert fake_impl(*args) is None
@@ -347,7 +347,7 @@ def test_selective_state_update(dim, dstate, has_z, itype):
    rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
    if itype == torch.bfloat16:
        rtol, atol = 1e-2, 5e-2
-        if torch.version.hip:
+        if current_platform.is_rocm() or current_platform.is_xpu():
            atol *= 2
    # set seed
    set_random_seed(0)
@@ -437,7 +437,7 @@ def test_selective_state_update_varlen(dim, dstate, has_z, itype, max_seq_len):
    rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
    if itype == torch.bfloat16:
        rtol, atol = 5e-2, 1.5e-1
-        if torch.version.hip:
+        if current_platform.is_rocm() or current_platform.is_xpu():
            atol *= 2
    # set seed
    set_random_seed(0)
@@ -700,7 +700,7 @@ def test_selective_state_update_with_batch_indices(
    rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
    if itype == torch.bfloat16:
        rtol, atol = 1e-1, 1e-1
-        if torch.version.hip:
+        if current_platform.is_rocm() or current_platform.is_xpu():
            atol *= 2
    # set seed
    torch.random.manual_seed(0)
@@ -865,7 +865,7 @@ def test_selective_state_update_with_num_accepted_tokens(
    rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
    if itype == torch.bfloat16:
        rtol, atol = 5e-2, 1.5e-1
-        if torch.version.hip:
+        if current_platform.is_rocm() or current_platform.is_xpu():
            atol *= 2

    set_random_seed(0)
@@ -991,7 +991,7 @@ def test_selective_state_update_varlen_with_num_accepted(
    rtol, atol = (3e-4, 1e-3) if itype == torch.float32 else (5e-3, 1e-2)
    if itype == torch.bfloat16:
        rtol, atol = 5e-2, 1.5e-1
-        if torch.version.hip:
+        if current_platform.is_rocm() or current_platform.is_xpu():
            atol *= 2

    set_random_seed(0)
@@ -0,0 +1,180 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Equivalence test for ``precopy_mamba_align_fused_kernel``.
+
+The V2 "align" pre-copy must migrate mamba state across block boundaries with
+byte-identical semantics to the V1 copy specs (``get_conv_copy_spec`` /
+``get_temporal_copy_spec``):
+
+* conv state (SD layout, conv_width > 0): shift the sliding window by
+  ``token_bias`` tokens -- ``state[bt[src_col], token_bias:]`` ->
+  ``state[bt[dst_col], :conv_width - token_bias]``.
+* temporal state (conv_width == 0): ``token_bias`` selects the accepted
+  speculative column -- ``state[bt[src_col + token_bias]]`` ->
+  ``state[bt[dst_col]]``.
+
+The kernel must also no-op when ``src_col < 0`` (fresh request) or
+``src_col == dst_col`` (no boundary crossed).
+"""
+
+from __future__ import annotations
+
+import torch
+
+from vllm.platforms import current_platform
+from vllm.v1.worker.mamba_utils import precopy_mamba_align_fused_kernel
+
+try:
+    import pytest
+
+    pytestmark = pytest.mark.skipif(
+        not current_platform.is_cuda(),
+        reason="precopy_mamba_align_fused_kernel needs CUDA/Triton",
+    )
+    _parametrize = pytest.mark.parametrize
+except ModuleNotFoundError:  # allow running directly as ``python <thisfile>``
+    pytest = None
+
+    def _parametrize(_name, _values):
+        def _deco(fn):
+            return fn
+
+        return _deco
+
+
+NUM_LAYERS = 3
+CONV_WIDTH = 4  # conv_kernel - 1 + num_spec
+CONV_DIM = 96
+SSM_SHAPE = (4, 16, 16)
+MAX_COLS = 8
+
+
+def _build_state(num_blocks, device):
+    """Per-layer (conv SD [nb, width, dim] bf16, ssm [nb, *shape] fp32) pools."""
+    convs, ssms = [], []
+    for _ in range(NUM_LAYERS):
+        convs.append(
+            torch.randn(
+                num_blocks, CONV_WIDTH, CONV_DIM, dtype=torch.bfloat16, device=device
+            )
+        )
+        ssms.append(
+            torch.randn(num_blocks, *SSM_SHAPE, dtype=torch.float32, device=device)
+        )
+    return convs, ssms
+
+
+def _build_meta(convs, ssms, device):
+    """Flattened per-(layer, state-type) metadata, ordered conv, ssm per layer."""
+    n = NUM_LAYERS * 2
+    base = torch.zeros(n, dtype=torch.int64, device=device)
+    blk_stride = torch.zeros(n, dtype=torch.int64, device=device)
+    elem = torch.zeros(n, dtype=torch.int32, device=device)
+    inner = torch.zeros(n, dtype=torch.int64, device=device)
+    width = torch.zeros(n, dtype=torch.int32, device=device)
+    group = torch.zeros(n, dtype=torch.int32, device=device)
+    drc = torch.zeros(n, dtype=torch.int32, device=device)  # DS rows (unused, SD)
+    drs = torch.zeros(n, dtype=torch.int64, device=device)
+    i = 0
+    for layer in range(NUM_LAYERS):
+        conv, ssm = convs[layer], ssms[layer]
+        # conv (SD): width = size(1), inner = stride(1)
+        base[i] = conv.data_ptr()
+        blk_stride[i] = conv.stride(0) * conv.element_size()
+        elem[i] = conv.element_size()
+        width[i] = conv.size(1)
+        inner[i] = conv.stride(1)
+        i += 1
+        # ssm (temporal): width = 0, inner = elems per block
+        base[i] = ssm.data_ptr()
+        blk_stride[i] = ssm.stride(0) * ssm.element_size()
+        elem[i] = ssm.element_size()
+        width[i] = 0
+        inner[i] = ssm[0].numel()
+        i += 1
+    return base, blk_stride, elem, inner, width, group, drc, drs
+
+
+def _reference(convs, ssms, bt, src_col, dst_col, bias, num_reqs):
+    """Apply the V1 copy semantics on clones, reading from the pre-copy state."""
+    conv_pre = [c.clone() for c in convs]
+    ssm_pre = [s.clone() for s in ssms]
+    conv_ref = [c.clone() for c in convs]
+    ssm_ref = [s.clone() for s in ssms]
+    for r in range(num_reqs):
+        sc, dc, tb = int(src_col[r]), int(dst_col[r]), int(bias[r])
+        if sc < 0 or sc == dc:
+            continue
+        sblk, dblk = int(bt[r, sc]), int(bt[r, dc])
+        tblk = int(bt[r, sc + tb])  # temporal src column shifted by bias
+        for layer in range(NUM_LAYERS):
+            conv_ref[layer][dblk, : CONV_WIDTH - tb] = conv_pre[layer][sblk, tb:]
+            ssm_ref[layer][dblk] = ssm_pre[layer][tblk]
+    return conv_ref, ssm_ref
+
+
+@_parametrize("num_reqs", [1, 4, 16])
+@_parametrize("token_bias", [0, 1, 2])
+def test_precopy_matches_v1_copy_specs(num_reqs, token_bias):
+    device = torch.device("cuda")
+    torch.manual_seed(0)
+    # Distinct physical block per (req, col) so copies never alias.
+    num_blocks = num_reqs * MAX_COLS + 1
+    bt = torch.empty(num_reqs, MAX_COLS, dtype=torch.int32, device=device)
+    for r in range(num_reqs):
+        bt[r] = torch.arange(
+            1 + r * MAX_COLS, 1 + (r + 1) * MAX_COLS, dtype=torch.int32, device=device
+        )
+
+    # Per-req columns: req 0 fresh (src=-1, skip), req 1 same block (skip),
+    # the rest cross from col 1 -> col 0 with the given spec token bias.
+    src_col = torch.full((num_reqs,), 1, dtype=torch.int32, device=device)
+    dst_col = torch.zeros(num_reqs, dtype=torch.int32, device=device)
+    bias = torch.full((num_reqs,), token_bias, dtype=torch.int32, device=device)
+    if num_reqs >= 1:
+        src_col[0] = -1  # fresh -> no copy
+    if num_reqs >= 2:
+        dst_col[1] = 1  # src_col == dst_col -> no copy
+
+    convs, ssms = _build_state(num_blocks, device)
+    conv_ref, ssm_ref = _reference(
+        convs, ssms, bt.cpu(), src_col.cpu(), dst_col.cpu(), bias.cpu(), num_reqs
+    )
+
+    base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
+        convs, ssms, device
+    )
+    bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
+    idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
+    grid = (num_reqs, NUM_LAYERS * 2)
+    precopy_mamba_align_fused_kernel[grid](
+        dst_col,
+        src_col,
+        bias,
+        bt_ptrs,
+        bt.stride(0),
+        base,
+        blk_stride,
+        elem,
+        inner,
+        width,
+        group,
+        drc,
+        drs,
+        idx_mapping,
+        num_reqs,
+        COPY_BLOCK_SIZE=1024,
+        CONV_STATE_DIM_FIRST=False,
+    )
+    torch.accelerator.synchronize()
+
+    for layer in range(NUM_LAYERS):
+        torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
+        torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
+
+
+if __name__ == "__main__":
+    for nr in (1, 4, 16):
+        for tb in (0, 1, 2):
+            test_precopy_matches_v1_copy_specs(nr, tb)
+            print(f"OK num_reqs={nr} token_bias={tb}")
@@ -406,7 +406,7 @@ def test_fused_moe_int64_overflow(workspace_init):
    Reproduces the scenario from PR #34279.
    """
    # ~12 GB GPU memory needed for intermediate caches
-    free_mem = torch.cuda.mem_get_info()[0]
+    free_mem = torch.accelerator.get_memory_info()[0]
    if free_mem < 12 * 1024**3:
        pytest.skip("Insufficient GPU memory for overflow test")

@@ -97,7 +97,7 @@ def test_sharded_state_loader(
    ctx = mp.get_context("spawn")

    platform_args = {}
-    if current_platform.is_rocm():
+    if current_platform.is_rocm() or current_platform.is_xpu():
        platform_args["max_num_seqs"] = 1

    # Run in separate processes for memory & CUDA isolation
@@ -160,5 +160,126 @@ class TestMaybeRemapKvScaleName:
        assert result is None


+class TestKvCacheScaleMapper:
+    """The `WeightsMapper` returned by `get_cache_scale_mapper` replaces the
+    per-model `maybe_remap_kv_scale_name` calls. It must remap the same set of
+    checkpoint formats (the non-`params_dict`-dependent ones) and be idempotent
+    so it composes safely with a model's own qkv/gate_up `hf_to_vllm_mapper`."""
+
+    def _mapper(self):
+        # `get_cache_scale_mapper` does not use `self`; call it on the base
+        # class to get the default (non-config-specific) mapper.
+        from vllm.model_executor.layers.quantization.base_config import (
+            QuantizationConfig,
+        )
+
+        return QuantizationConfig.get_cache_scale_mapper()
+
+    def _map(self, name: str) -> str | None:
+        return self._mapper()._map_name(name)
+
+    @pytest.mark.parametrize(
+        "name,expected",
+        [
+            # Qwen3-MoE / llm-compressor fused qkv_proj
+            (
+                "model.layers.0.self_attn.qkv_proj.k_scale",
+                "model.layers.0.self_attn.attn.k_scale",
+            ),
+            (
+                "model.layers.0.self_attn.qkv_proj.v_scale",
+                "model.layers.0.self_attn.attn.v_scale",
+            ),
+            # ModelOpt / NVFP4 k_proj/v_proj
+            (
+                "model.layers.0.self_attn.k_proj.k_scale",
+                "model.layers.0.self_attn.attn.k_scale",
+            ),
+            (
+                "model.layers.0.self_attn.v_proj.v_scale",
+                "model.layers.0.self_attn.attn.v_scale",
+            ),
+            # deprecated fused kv_scale and bare scales
+            (
+                "model.layers.0.self_attn.kv_scale",
+                "model.layers.0.self_attn.attn.k_scale",
+            ),
+            (
+                "model.layers.0.self_attn.k_scale",
+                "model.layers.0.self_attn.attn.k_scale",
+            ),
+            # NemotronH mixer
+            (
+                "model.layers.0.mixer.k_proj.k_scale",
+                "model.layers.0.mixer.attn.k_scale",
+            ),
+            # already in vLLM form -> unchanged (idempotent)
+            (
+                "model.layers.0.self_attn.attn.k_scale",
+                "model.layers.0.self_attn.attn.k_scale",
+            ),
+            # non-kv scales must not be touched
+            (
+                "model.layers.0.self_attn.k_proj.weight_scale",
+                "model.layers.0.self_attn.k_proj.weight_scale",
+            ),
+            (
+                "model.layers.0.self_attn.k_proj.input_scale",
+                "model.layers.0.self_attn.k_proj.input_scale",
+            ),
+            # regular weights untouched
+            (
+                "model.layers.0.self_attn.q_proj.weight",
+                "model.layers.0.self_attn.q_proj.weight",
+            ),
+        ],
+    )
+    def test_remap(self, name, expected):
+        assert self._map(name) == expected
+
+    @pytest.mark.parametrize(
+        "name",
+        [
+            "model.layers.0.self_attn.k_scale",
+            "model.layers.0.self_attn.k_proj.k_scale",
+            "model.layers.0.self_attn.qkv_proj.v_scale",
+            "model.layers.0.mixer.k_proj.k_scale",
+        ],
+    )
+    def test_idempotent(self, name):
+        once = self._map(name)
+        assert once is not None
+        assert self._map(once) == once
+
+    def test_composes_with_qkv_mapper(self):
+        """Applied together with a model's qkv/gate_up mapper, the regex scale
+        rules run before the substr rename, so scales are normalized to `.attn.`
+        and regular projections are still fused correctly."""
+        from vllm.model_executor.models.utils import WeightsMapper
+
+        model_mapper = WeightsMapper(
+            orig_to_new_substr={
+                ".q_proj": ".qkv_proj.q",
+                ".k_proj": ".qkv_proj.k",
+                ".v_proj": ".qkv_proj.v",
+            }
+        )
+        # AutoWeightsLoader does `mapper |= cache_scale_mapper`
+        combined = model_mapper | self._mapper()
+
+        assert (
+            combined._map_name("model.layers.0.self_attn.q_proj.weight")
+            == "model.layers.0.self_attn.qkv_proj.q.weight"
+        )
+        assert (
+            combined._map_name("model.layers.0.self_attn.k_proj.k_scale")
+            == "model.layers.0.self_attn.attn.k_scale"
+        )
+        assert (
+            combined._map_name("model.layers.0.self_attn.k_scale")
+            == "model.layers.0.self_attn.attn.k_scale"
+        )
+
+
 if __name__ == "__main__":
    test_download_weights_from_hf()
@@ -5,6 +5,7 @@ import pytest
 import torch
 from transformers import AutoModelForTokenClassification

+from tests.models.registry import HF_EXAMPLE_MODELS
 from tests.models.utils import softmax
 from vllm.platforms import current_platform
 from vllm.utils.torch_utils import set_random_seed
@@ -136,6 +137,9 @@ def test_openai_privacy_filter(
    model: str,
    dtype: str,
 ) -> None:
+    model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+    model_info.check_transformers_version(on_fail="skip")
+
    with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
        vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)

@@ -83,7 +83,7 @@ def _ru_maxrss_bytes() -> int | None:

 def _gpu_used_bytes() -> int:
    torch.accelerator.synchronize()
-    free_bytes, total_bytes = current_platform.mem_get_info()
+    free_bytes, total_bytes = torch.accelerator.get_memory_info()
    return int(total_bytes - free_bytes)


@@ -20,6 +20,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.cache import MultiModalProcessorOnlyCache
 from vllm.multimodal.inputs import batched_tensors_equal
 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
+from vllm.platforms import current_platform
 from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
 from vllm.utils.mistral import is_mistral_tokenizer

@@ -83,6 +84,12 @@ MM_DATA_PATCHES = {
    "glmasr": glmasr_patch_mm_data,
 }

+_XPU_EXCLUDED_MODEL_IDS = {
+    "baidu/Unlimited-OCR",
+    "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4",
+    "Qwen/Qwen2.5-Omni-7B-AWQ",
+}
+

 def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):
    for model_arch in model_arch_list:
@@ -97,7 +104,14 @@ def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]):


 def _get_model_ids_to_test(model_arch_list: AbstractSet[str]):
-    return list(_iter_model_ids_to_test(model_arch_list))
+    model_ids = list(_iter_model_ids_to_test(model_arch_list))
+
+    if current_platform.is_xpu():
+        for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS:
+            while excluded_model_id in model_ids:
+                model_ids.remove(excluded_model_id)
+
+    return model_ids


 def get_model_ids_to_test():
@@ -20,6 +20,7 @@ from vllm.model_executor.models.registry import (
    _SPECULATIVE_DECODING_MODELS,
    _TEXT_GENERATION_MODELS,
    ModelRegistry,
+    _LazyRegisteredModel,
 )
 from vllm.platforms import current_platform

@@ -127,6 +128,22 @@ def test_registry_is_pp(model_arch, is_pp, init_cuda):
            )


+def test_lazy_modelinfo_package_hash_includes_submodules(tmp_path):
+    package_dir = tmp_path / "model_package"
+    package_dir.mkdir()
+    init_file = package_dir / "__init__.py"
+    init_file.write_text("from .model import Model\n", encoding="utf-8")
+    model_file = package_dir / "model.py"
+    model_file.write_text("class Model: pass\n", encoding="utf-8")
+
+    first_hash = _LazyRegisteredModel._get_modelinfo_module_hash(init_file)
+
+    model_file.write_text("class Model:\n    supports_pp = True\n", encoding="utf-8")
+    second_hash = _LazyRegisteredModel._get_modelinfo_module_hash(init_file)
+
+    assert first_hash != second_hash
+
+
 def test_hf_registry_coverage():
    untested_archs = (
        ModelRegistry.get_supported_archs() - HF_EXAMPLE_MODELS.get_supported_archs()
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
dependabot[bot]andGitHub	95dcefaaa5	Bump actions/setup-python from 6.1.0 to 6.3.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 6.1.0 to 6.3.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/83679a892e2d95755f2dac6acb0bfd1e9ac5d548...ece7cb06caefa5fff74198d8649806c4678c61a1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: 6.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com>	2026-06-30 12:18:49 +00:00
dependabot[bot]GitHubdependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	536047755e	Bump actions/checkout from 6.0.1 to 7.0.0 (#33057 ) Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2026-06-30 13:16:20 +01:00
JianweiZheng GitHub Claude Wentao Ye Harry Mellor	1907d3854a	[Bugfix] Reject negative values for max_logprobs and long_prefill_token_threshold (#44002 ) Signed-off-by: jwzheng96 <jianweizheng@pku.edu.cn> Signed-off-by: JianweiZheng <32029023+jwzheng96@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>	2026-06-30 13:01:03 +01:00
Chaojun ZhangandGitHub	ea9ddf59fc	[XPU][CI] Enable shared loader test (#45977 ) Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>	2026-06-30 11:20:33 +00:00
Cheng Jiang GitHub chengvjiang Andreas Karatzas	8cf7c4d8ad	[Attention Backend] add HPC-Ops Attention backend (#46020 ) Signed-off-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: chengvjiang <chengvjiang@tencent.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>	2026-06-30 18:17:43 +08:00
Agata Dobrzyniewicz GitHub Kunshang Ji	8e9d70fdd5	[Kernel][XPU] Adjust kernel unit tests for XPU (#45140 ) Signed-off-by: Dobrzyniewicz, Agata <agata.dobrzyniewicz@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>	2026-06-30 09:57:27 +00:00
Juan Pérez de AlgabaandGitHub	364ee36af1	fix(security): prevent image decompression bomb OOM denial of service (#47010 ) Signed-off-by: jperezde <jperezde@redhat.com>	2026-06-30 09:39:22 +00:00
Nicolò LucchesiandGitHub	06fae69114	[Misc] Mistral label alert (#47132 ) Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>	2026-06-30 09:02:07 +00:00
Li, Jiang GitHub Harry Mellor	14f8660a18	[CI/Build] Add CPU test dependency pre-commit hooks (#47032 ) Signed-off-by: jiang1.li <jiang1.li@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>	2026-06-30 07:59:13 +00:00
Aman Paswan GitHub amanambak Chauncey	aed541def4	[Bugfix][Responses] Set completed status for Harmony function calls (#46945 ) Signed-off-by: amanambak <aman.paswan@ambak.com> Co-authored-by: amanambak <aman.paswan@ambak.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>	2026-06-30 07:55:14 +00:00
Chauncey GitHubmergify[bot] <37929162+mergify[bot]@users.noreply.github.com>	2bc20e8aba	[Frontend] Add Streaming Parser Engine and new Kimi k2.5/k2.6/k2.7 Parser (#46610 ) Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>	2026-06-30 07:53:17 +00:00
Chaojun ZhangandGitHub	8cc242335d	[XPU] Optimize XPU worker shutdown logic to prevent resource leak (#46433 ) Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>	2026-06-30 15:27:21 +08:00
Andreas KaratzasandGitHub	ba22cb6765	[ROCm][Ray][CI] Keep assigned GPU visible for weight transfer (#47000 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com>	2026-06-30 14:59:18 +08:00
Uros MarkovicandGitHub	81bcced482	[Bugfix][ROCm] Preserve MoE weight padding for unquantized Triton path (#46381 ) Signed-off-by: Uros Markovic <umarkovi@amd.com>	2026-06-30 14:47:57 +08:00
Kunshang JiandGitHub	fb42e5219e	[Platform] Replace `torch.cuda.mem_get_info` with `torch.accelerator.get_memory_info` (#44825 ) Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Signed-off-by: Kunshang Ji <jikunshang95@gmail.com>	2026-06-30 14:39:52 +08:00
Dakai AnandGitHub	0feca7ffa8	PD disagg with Mooncake Connector: GDN support (Qwen3.5) and MLA support (Deepseek-V4-Flash) (#46807 )	2026-06-29 23:29:04 -07:00
Muhammad Fawaz GitHub Mahad Durrani	97b5ce5c39	[Bugfix] Raise VLLMValidationError for non-integer logit_bias keys (#46612 ) Signed-off-by: muhammadfawaz1 <135441198+muhammadfawaz1@users.noreply.github.com> Co-authored-by: Mahad Durrani <114791389+mahadrehmann@users.noreply.github.com>	2026-06-30 06:18:59 +00:00
Andreas KaratzasandGitHub	4236514098	[ROCm][CI][Multimodal] Use ROCm-aware FA availability check for Unlimited-OCR (#47004 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com>	2026-06-30 14:03:13 +08:00
Blas Rodriguez IrizarandGitHub	e45c8a9f4b	[Rust Frontend] Start current wave for a stale DP FirstRequest (#46833 ) Signed-off-by: Blas Rodriguez Irizar <rodrigblas@gmail.com>	2026-06-30 05:13:09 +00:00
Wei ZhaoandGitHub	b153dd3f28	[Bugfix] Use larger workspace size for Flashinfer MLA LSE (#47074 ) Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>	2026-06-29 22:11:03 -07:00
ReidandGitHub	930f8dc0a1	[Bugfix][Rust Frontend] Reject prompt_logprobs for streaming generate (#46839 ) Signed-off-by: reidliu41 <reid201711@gmail.com>	2026-06-30 05:10:07 +00:00
ReidandGitHub	a16dbd5b85	[Rust Frontend] Avoid LoRA registry scans without active LoRA requests (#47040 ) Signed-off-by: reidliu41 <reid201711@gmail.com>	2026-06-30 04:58:19 +00:00
liranschour GitHub Or Ozeri Or Ozeri	bec232a914	Secondary tier implementation for PD disaggregation (#42285 ) Signed-off-by: Liran Schour <lirans@il.ibm.com> Signed-off-by: liranschour <liranschour@users.noreply.github.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>	2026-06-30 07:51:44 +03:00
linitra24 GitHub Jee Jee Li	b5c9e1ac33	[LoRA] Add language-backbone LoRA support for MiniCPM-V 4.6 (#46740 ) Signed-off-by: linitra24 <Joy25810@foxmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>	2026-06-30 04:19:31 +00:00
Yejing Lai GitHub Kunshang Ji	ae2c4f3db7	[XPU][UT]Fix xpu pass_config.fuse_norm_quant assert issue (#46804 ) Signed-off-by: Lai, Yejing <yejing.lai@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>	2026-06-29 21:13:44 -07:00
ganeshandGitHub	fca432e60a	[Bugfix] Propagate default stop_token_ids to per-request SamplingParams (#35076 ) Signed-off-by: sriganesh123 <arjulasriganesh@gmail.com>	2026-06-30 12:10:09 +08:00
hcl GitHubmergify[bot] <37929162+mergify[bot]@users.noreply.github.com>	af1ee8c475	fix(config): reject negative max_logprobs (except -1) and long_prefill_token_threshold (#44070 ) Signed-off-by: Chenglun Hu <chenglunhu@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>	2026-06-30 04:02:36 +00:00
Summer Yang GitHub Claude	5b4cb69523	[Bugfix][MLA] Fix LSE log-base mismatch in DCP + FlashInfer MLA decode (#47079 ) Signed-off-by: girasoley <girasoleyang@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>	2026-06-29 19:15:02 -07:00
peizhang56 GitHub Claude	9fc0c08026	[ROCm][CI] Make tests/v1/shutdown an importable package (#47085 ) Signed-off-by: pei.zhang <pei.zhang@amd.com> Co-authored-by: Claude <noreply@anthropic.com>	2026-06-29 21:01:27 -05:00
peizhang56 GitHub Andreas Karatzas Claude	f2b5fabb23	[ROCm][CI] Move LM Eval Large Models (8 GPUs) to mi300 pool (#47094 ) Signed-off-by: pei.zhang <pei.zhang@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: Claude <noreply@anthropic.com>	2026-06-29 20:59:08 -05:00
Tahsin Tunan GitHub Bugen Zhaomergify[bot] <37929162+mergify[bot]@users.noreply.github.com>	b8cb75b149	[Rust Frontend] Add static HTTPS and mTLS support for HTTP and gRPC (#45890 ) Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Tahsin Tunan <tahsintunan@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>	2026-06-30 01:45:59 +00:00
Thien TranandGitHub	43916891b2	[GDN] Improve kkt kernel of CuteDSL prefill backend (#46346 ) Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>	2026-06-29 18:34:18 -07:00
Ashwin Giridharan GitHub Cursor Agent Simon Mo	cda05ee8c4	[Bugfix][Reasoning] Fix thinking_token_budget not enforced on re-entry after forced end (#43757 ) Signed-off-by: Ashwin Giridharan <girida@amazon.com> Signed-off-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Cursor Agent <cursor-agent@cursor.com> Co-authored-by: Simon Mo <simon.mo@hey.com>	2026-06-30 01:04:25 +00:00
weishuandGitHub	77654d080c	[KVTransfer] MultiConnector: merge kv_transfer_params dicts across connectors (#46777 ) Signed-off-by: deng451e <838677410@qq.com>	2026-06-30 00:25:05 +00:00
Wentao YeandGitHub	75698e60b3	[Bug] Fix sparse attention issue for GLM5.2 non-torch compile path (#47083 ) Signed-off-by: yewentao256 <zhyanwentao@126.com>	2026-06-29 15:45:53 -07:00
Andreas KaratzasandGitHub	8632c884dc	[ROCm][CI] Use spawn around the threaded OTLP test (#47003 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com>	2026-06-29 16:34:05 -05:00
peizhang56 GitHub Claude	c3734e8334	[CI][Bugfix] Add cohere_melody to ROCm test requirements (#47072 ) Signed-off-by: pei.zhang <pei.zhang@amd.com> Co-authored-by: Claude <noreply@anthropic.com>	2026-06-29 16:29:47 -05:00
Andreas Karatzas GitHub Tyler Michael Smith	53f7553f09	[ROCm][DeepEP] Stabilize high-throughput DBO for DP+EP (#46990 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>	2026-06-29 14:28:02 -07:00
Andreas Karatzas GitHub Codex	4eb227992a	[ROCm][CI] Make memory sampling less racy in tests and sleep mode (#45490 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Codex <codex@example.invalid> Co-authored-by: Codex <codex@example.invalid>	2026-06-29 14:26:41 -07:00
Micah WilliamsonandGitHub	ebcf511ec3	[ROCm][CI] Soft Fail `Spec Decode Ngram + Suffix` and `Entrypoints Integration (LLM)` AMD Mirrors (#47067 ) Signed-off-by: Micah Williamson <micah.williamson@amd.com>	2026-06-29 16:24:08 -05:00
Matthew BonanniandGitHub	8fc1b2d046	Fix FA4 dynamic_causal for full attention layers (#46659 ) Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>	2026-06-29 14:23:34 -07:00
Harry MellorandGitHub	5316638a5e	Fix transient dependency issues caused by `requirements/common.txt` (#47015 ) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>	2026-06-29 14:20:33 -07:00
zhrrrandGitHub	61ab70ec3b	[Model Runner V2] support mamba hybrid models align prefix cache (#42406 ) Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com>	2026-06-29 14:09:16 -07:00
Woosuk KwonandGitHub	a309d4fe60	Support DCP with FlashInfer MLA (#43729 ) Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>	2026-06-29 13:24:29 -07:00
zofia GitHub Kunshang Ji	72f639927f	[XPU] [RMSNorm] revert weightless change on xpu (#46987 ) Signed-off-by: Zhu, Zufang <zufang.zhu@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>	2026-06-29 19:03:06 +00:00
Nick HillandGitHub	8ad4a01825	[ModelRunner V2] Simplify recent UnlimitedOCR-related changes (#46975 ) Signed-off-by: Nick Hill <nickhill123@gmail.com>	2026-06-29 09:56:17 -07:00
Jee Jee LiandGitHub	7be582697b	[Bugfix] Fix DeepseekV2Model hidden_size (#46986 ) Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>	2026-06-29 16:44:05 +00:00
Roberto L. Castro GitHub Lucas Wilkinson	030c9523bd	[Perf][1/N] Expand Triton kernel warmup coverage, DSv4 (#46634 ) Signed-off-by: LopezCastroRoberto <rocastro@redhat.com> Signed-off-by: Roberto L. Castro <38211239+LopezCastroRoberto@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>	2026-06-29 16:40:34 +00:00
Wei Zhao GitHub Jee Jee Li	4708292d48	Bump flashinfer version to 0.6.13 (#46683 ) Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>	2026-06-29 09:30:57 -07:00
Jason Li GitHub Xin Li	debec6440b	Add MiniMax-M3 modelopt nvfp4 support (#46756 ) Signed-off-by: Xin Li <xinli@nvidia.com> Signed-off-by: jasonlizhengjian <jasonlizhengjian@gmail.com> Co-authored-by: Xin Li <xinli@nvidia.com>	2026-06-29 09:29:39 -07:00
Varun Sundar Rabindranath GitHub Varun Sundar Rabindranath	c8fb2963bd	[FS-Offloading] Batch Lookup in C (#46713 ) Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>	2026-06-29 09:28:32 -07:00
HDCharlesandGitHub	379acd4e4f	[Bugfix][Quantization] Fix W8A8 int-quantized scheme selection regression (#46860 ) Signed-off-by: HDCharles <charlesdavidhernandez@gmail.com>	2026-06-29 15:55:42 +00:00
Martin HickeyandGitHub	07d33e575b	[MyPy] Fix mypy incompatible assignment errors in LRUCacheLoRAModelManager (#44657 ) Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>	2026-06-29 16:42:35 +01:00
Varun Sundar Rabindranath GitHub Varun Sundar Rabindranath	36bbecd643	[BugFix] Revert "[KV Offload] Use background thread for mmap / cpu_tensors pinning" (#46958 ) Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>	2026-06-29 07:54:34 -07:00
Nicolò LucchesiandGitHub	6149187a4c	[Kernel] Triton MLA logits workspace (#46819 ) Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>	2026-06-29 07:54:29 -07:00
Xiaohong (Sean) ChenandGitHub	49e28e8e91	[Kernel][Helion][1/N] Add Helion kernel for fused_qk_norm_rope (#44010 ) Signed-off-by: Sean Chen <seachen@redhat.com>	2026-06-29 22:54:15 +08:00
Michael Goin GitHub Claude Opus 4.8	0ca39c4f1f	[Bugfix] Capture final-layer aux hidden state in deepseek_v2 backbone (#46973 ) Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>	2026-06-29 10:00:31 -04:00
Blas Rodriguez IrizarandGitHub	6185d73882	[Rust Frontend] Keep literal "null" string for string-typed tool params (#46827 ) Signed-off-by: Blas Rodriguez Irizar <rodrigblas@gmail.com>	2026-06-29 13:46:33 +00:00
bnellnm GitHub Robert Shaw	bc8481af09	[MoE Refactor] Standardize Humming MoE experts + utilities (#43373 ) Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>	2026-06-29 06:19:29 -07:00
Yan Ma GitHub Kunshang Ji	59575da46d	[XPU] exclude unsupported models for test_tensor_sechma.py (#47008 ) Signed-off-by: Yan Ma <yan.ma@intel.com> Signed-off-by: Kunshang Ji <kunshang.ji@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>	2026-06-29 12:30:28 +00:00
wang.yuqiandGitHub	3483240b7e	[Frontend] Consolidate scale out entrypoints (#44512 ) Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>	2026-06-29 03:18:53 -07:00
Roberto L. CastroandGitHub	eddfd4cf21	[Perf][2/N] Expand Triton kernel warmup coverage, Qwen (#46750 ) Signed-off-by: LopezCastroRoberto <rocastro@redhat.com>	2026-06-29 10:10:07 +00:00
Martin HickeyandGitHub	a4e3cb40d0	[mypy] Enable mypy for tests directory (#47018 ) Signed-off-by: Martin Hickey <martin.hickey@ie.ibm.com>	2026-06-29 09:29:09 +00:00
soaringkandGitHub	ab132ee98b	Fix model info cache for package models (#46567 ) Signed-off-by: soaringk <k3vin.zhang@gmail.com>	2026-06-29 09:17:54 +00:00
Alden Lobo GitHub Alden Lobo	e186107870	[Bugfix] Use native SiLU activation in CPU fused MoE (#45961 ) Signed-off-by: Alden Lobo <alden.lobo@arm.com> Co-authored-by: Alden Lobo <alden.lobo@arm.com>	2026-06-29 09:12:20 +00:00
John GitHub Harry Mellor	0e207dac78	[Bugfix] Transformers backend: apply learned lm_head.bias for tied-embedding models (#46835 ) Signed-off-by: John Langford <jl@hunch.net> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>	2026-06-29 08:59:15 +00:00
wang.yuqiandGitHub	9e86352c60	[CI Failure] Add transformers version check for openai/privacy-filter (#47011 ) Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>	2026-06-29 08:57:26 +00:00
Harry MellorandGitHub	5051698e41	Remove unnecessary `load_weights` methods (#44589 ) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>	2026-06-29 01:52:23 -07:00
Andreas KaratzasandGitHub	db28ae2d07	[ROCm][CI] Explicitly tear down multimodal offline LLMs (#46999 ) Signed-off-by: Andreas Karatzas <akaratza@amd.com>	2026-06-29 07:59:24 +00:00
Harry MellorandGitHub	f6bb8682ee	Fix docs on main (#47009 ) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>	2026-06-29 15:50:57 +08:00