From c9dddc144b01bbbb3ee70f529575aeda95ed1778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Fri, 10 Apr 2026 15:40:40 +0200 Subject: [PATCH] [CI] Add Nixl+OffloadingConnector e2e integration tests (#39200) Signed-off-by: NickLucche --- .buildkite/test_areas/distributed.yaml | 14 ++ .../run_multi_connector_accuracy_test.sh | 205 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100755 tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 19815e14b4e..62cc63833c5 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -268,6 +268,20 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 2 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh + - label: NixlConnector PD + Spec Decode acceptance (2 GPUs) timeout_in_minutes: 30 device: a100 diff --git a/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh new file mode 100755 index 00000000000..2e71858983e --- /dev/null +++ b/tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh @@ -0,0 +1,205 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Integration accuracy test for MultiConnector (NixlConnector + OffloadingConnector). +# +# Launches a P/D setup where both prefill and decode instances use MultiConnector +# wrapping NixlConnector and OffloadingConnector, then runs gsm8k accuracy via +# test_accuracy.py. +# +# By default runs two configurations: +# 1. Normal KV layout (NixlConnector without cross-layer blocks) +# 2. Cross-layer KV layout (NixlConnector with enable_cross_layers_blocks) +# +# Usage: +# bash tests/v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh +# +# Environment variables: +# MODEL_NAMES - model to test (default: Qwen/Qwen3-0.6B) +# GPU_MEMORY_UTILIZATION - GPU memory fraction (default: 0.6) +# VLLM_SERVE_EXTRA_ARGS - comma-separated extra args for vllm serve +# SKIP_CROSS_LAYERS - set to 1 to skip the cross-layer layout test +# SKIP_NORMAL_LAYOUT - set to 1 to skip the normal layout test +set -xe + +# ── Configuration ──────────────────────────────────────────────────────── + +MODEL_NAMES=${MODEL_NAMES:-} +if [[ -n "$MODEL_NAMES" ]]; then + MODELS=("$MODEL_NAMES") +else + MODELS=("Qwen/Qwen3-0.6B") +fi + +GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.6} +BLOCK_SIZE=${BLOCK_SIZE:-128} +VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-} + +GIT_ROOT=$(git rev-parse --show-toplevel) +SMI_BIN=$(which nvidia-smi || which rocm-smi || echo "") + +# ── KV transfer configs ───────────────────────────────────────────────── + +# Normal layout: OffloadingConnector prefers cross-layer but NixlConnector +# does not, so MultiConnector.prefer_cross_layer_blocks = False. +KV_CONFIG_NORMAL='{ + "kv_connector":"MultiConnector", + "kv_role":"kv_both", + "kv_connector_extra_config":{ + "connectors":[ + {"kv_connector":"NixlConnector","kv_role":"kv_both"}, + {"kv_connector":"OffloadingConnector","kv_role":"kv_both", + "kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}} + ] + } +}' +# Remove whitespace for CLI safety +KV_CONFIG_NORMAL=$(echo "$KV_CONFIG_NORMAL" | tr -d '[:space:]') + +# Cross-layer layout: both connectors prefer cross-layer blocks. +KV_CONFIG_CROSS_LAYERS='{ + "kv_connector":"MultiConnector", + "kv_role":"kv_both", + "kv_connector_extra_config":{ + "connectors":[ + {"kv_connector":"NixlConnector","kv_role":"kv_both", + "kv_connector_extra_config":{"enable_cross_layers_blocks":"True"}}, + {"kv_connector":"OffloadingConnector","kv_role":"kv_both", + "kv_connector_extra_config":{"cpu_bytes_to_use":1000000000}} + ] + } +}' +KV_CONFIG_CROSS_LAYERS=$(echo "$KV_CONFIG_CROSS_LAYERS" | tr -d '[:space:]') + +# ── Helpers ────────────────────────────────────────────────────────────── + +trap 'kill $(jobs -pr) 2>/dev/null' SIGINT SIGTERM EXIT + +wait_for_server() { + local port=$1 + timeout 1200 bash -c " + until curl -s localhost:${port}/v1/completions > /dev/null; do + sleep 1 + done" && return 0 || return 1 +} + +cleanup_instances() { + echo "Cleaning up any running vLLM instances..." + pkill -f "vllm serve" || true + sleep 2 +} + +get_num_gpus() { + if [[ "$SMI_BIN" == *"nvidia"* ]]; then + $SMI_BIN --query-gpu=name --format=csv,noheader | wc -l + elif [[ "$SMI_BIN" == *"rocm"* ]]; then + $SMI_BIN -l | grep -c GPU + else + echo "1" + fi +} + +# ── Run tests for one model with a given KV config ─────────────────────── + +run_tests_for_model() { + local model_name=$1 + local kv_config=$2 + local label=$3 + + echo "================================================================" + echo "Testing model: $model_name ($label)" + echo "KV config: $kv_config" + echo "================================================================" + + local PREFILL_PORT=8100 + local DECODE_PORT=8200 + local PREFILL_GPU=0 + local DECODE_GPU=1 + local PREFILL_SIDE_CHANNEL_PORT=5559 + local DECODE_SIDE_CHANNEL_PORT=5659 + + # ── Start prefill instance ── + echo "Starting prefill instance on GPU $PREFILL_GPU, port $PREFILL_PORT" + BASE_CMD="CUDA_VISIBLE_DEVICES=$PREFILL_GPU \ + VLLM_KV_CACHE_LAYOUT='HND' \ + UCX_NET_DEVICES=all \ + VLLM_NIXL_SIDE_CHANNEL_PORT=$PREFILL_SIDE_CHANNEL_PORT \ + vllm serve $model_name \ + --port $PREFILL_PORT \ + --enforce-eager \ + --block-size ${BLOCK_SIZE} \ + --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + --tensor-parallel-size 1 \ + --kv-transfer-config '$kv_config'" + + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" + for arg in "${extra_args[@]}"; do + BASE_CMD="${BASE_CMD} $arg" + done + fi + eval "$BASE_CMD &" + + # ── Start decode instance ── + echo "Starting decode instance on GPU $DECODE_GPU, port $DECODE_PORT" + BASE_CMD="CUDA_VISIBLE_DEVICES=$DECODE_GPU \ + VLLM_KV_CACHE_LAYOUT='HND' \ + UCX_NET_DEVICES=all \ + VLLM_NIXL_SIDE_CHANNEL_PORT=$DECODE_SIDE_CHANNEL_PORT \ + vllm serve $model_name \ + --port $DECODE_PORT \ + --enforce-eager \ + --block-size ${BLOCK_SIZE} \ + --gpu-memory-utilization $GPU_MEMORY_UTILIZATION \ + --tensor-parallel-size 1 \ + --kv-transfer-config '$kv_config'" + + if [[ -n "$VLLM_SERVE_EXTRA_ARGS" ]]; then + IFS=',' read -r -a extra_args <<< "$VLLM_SERVE_EXTRA_ARGS" + for arg in "${extra_args[@]}"; do + BASE_CMD="${BASE_CMD} $arg" + done + fi + eval "$BASE_CMD &" + + # ── Wait for servers ── + echo "Waiting for prefill instance on port $PREFILL_PORT to start..." + wait_for_server "$PREFILL_PORT" + echo "Waiting for decode instance on port $DECODE_PORT to start..." + wait_for_server "$DECODE_PORT" + + # ── Start proxy ── + PROXY_CMD="python3 ${GIT_ROOT}/tests/v1/kv_connector/nixl_integration/toy_proxy_server.py --port 8192" + PROXY_CMD+=" --prefiller-hosts localhost" + PROXY_CMD+=" --prefiller-ports $PREFILL_PORT" + PROXY_CMD+=" --decoder-hosts localhost" + PROXY_CMD+=" --decoder-ports $DECODE_PORT" + + echo "Starting proxy server with command: $PROXY_CMD" + $PROXY_CMD & + sleep 5 + + # ── Run accuracy test ── + echo "Running accuracy tests for $model_name ($label)" + TEST_MODEL=$model_name python3 -m pytest -s -x \ + "${GIT_ROOT}"/tests/v1/kv_connector/nixl_integration/test_accuracy.py + + # ── Cleanup ── + cleanup_instances + sleep 3 +} + +# ── Main ───────────────────────────────────────────────────────────────── + +for model in "${MODELS[@]}"; do + if [[ -z "${SKIP_NORMAL_LAYOUT:-}" ]]; then + run_tests_for_model "$model" "$KV_CONFIG_NORMAL" "MultiConnector normal layout" + fi + + if [[ -z "${SKIP_CROSS_LAYERS:-}" ]]; then + run_tests_for_model "$model" "$KV_CONFIG_CROSS_LAYERS" "MultiConnector cross-layer layout" + fi +done + +echo "All MultiConnector accuracy tests passed!"