diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index f3690939667..89736eec127 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -1,25 +1,37 @@ #!/bin/bash -# This script runs test inside the corresponding ROCm docker container. +# This script runs tests inside the corresponding ROCm docker container. +# It handles both single-node and multi-node test configurations. +# +# Multi-node detection: Instead of matching on fragile group names, we detect +# multi-node jobs structurally by looking for the bracket command syntax +# "[node0_cmds] && [node1_cmds]" or via the NUM_NODES environment variable. set -o pipefail # Export Python path export PYTHONPATH=".." -# Print ROCm version -echo "--- Confirming Clean Initial State" -while true; do - sleep 3 - if grep -q clean /opt/amdgpu/etc/gpu_state; then - echo "GPUs state is \"clean\"" - break - fi -done +############################################################################### +# Helper Functions +############################################################################### -echo "--- ROCm info" -rocminfo +wait_for_clean_gpus() { + local timeout=${1:-300} + local start=$SECONDS + echo "--- Waiting for clean GPU state (timeout: ${timeout}s)" + while true; do + if grep -q clean /opt/amdgpu/etc/gpu_state; then + echo "GPUs state is \"clean\"" + return + fi + if (( SECONDS - start >= timeout )); then + echo "Error: GPUs did not reach clean state within ${timeout}s" >&2 + exit 1 + fi + sleep 3 + done +} -# cleanup older docker images cleanup_docker() { # Get Docker's root directory docker_root=$(docker info -f '{{.DockerRootDir}}') @@ -28,15 +40,12 @@ cleanup_docker() { exit 1 fi echo "Docker root directory: $docker_root" - # Check disk usage of the filesystem where Docker's root directory is located + disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') - # Define the threshold threshold=70 if [ "$disk_usage" -gt "$threshold" ]; then echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..." - # Remove dangling images (those that are not tagged and not used by any container) docker image prune -f - # Remove unused volumes / force the system prune for old images as well. docker volume prune -f && docker system prune --force --filter "until=72h" --all echo "Docker images and volumes cleanup completed." else @@ -45,193 +54,258 @@ cleanup_docker() { } cleanup_network() { - for node in $(seq 0 $((NUM_NODES-1))); do - if docker pr -a -q -f name="node${node}" | grep -q .; then - docker stop "node${node}" + local max_nodes=${NUM_NODES:-2} + for node in $(seq 0 $((max_nodes - 1))); do + if docker ps -a -q -f name="node${node}" | grep -q .; then + docker stop "node${node}" || true fi done - if docker network ls | grep docker-net; then - docker network rm docker-net + if docker network ls | grep -q docker-net; then + docker network rm docker-net || true fi } -# Call the cleanup docker function +is_multi_node() { + local cmds="$1" + # Primary signal: NUM_NODES environment variable set by the pipeline + if [[ "${NUM_NODES:-1}" -gt 1 ]]; then + return 0 + fi + # Fallback: detect the bracket syntax structurally + # Pattern: [...] && [...] (per-node command arrays) + if [[ "$cmds" =~ \[.*\].*\&\&.*\[.*\] ]]; then + return 0 + fi + return 1 +} + +############################################################################### +# Pytest marker re-quoting +# +# When commands are passed through Buildkite -> shell -> $* -> bash -c, +# quotes around pytest -m marker expressions get stripped: +# pytest -v -s -m 'not cpu_test' v1/core +# becomes: +# pytest -v -s -m not cpu_test v1/core +# +# pytest then interprets "cpu_test" as a file path, not part of the marker. +# This function detects unquoted multi-word marker expressions and re-quotes +# them so they survive the final bash -c expansion. +############################################################################### + +re_quote_pytest_markers() { + local cmds="$1" + # Pattern: -m not -> -m 'not ' + # Handles the common cases: 'not cpu_test', 'not slow_test', etc. + cmds=$(echo "$cmds" | sed -E "s/-m not ([a-zA-Z_][a-zA-Z0-9_]*)/-m 'not \1'/g") + echo "$cmds" +} + +############################################################################### +# ROCm-specific pytest command rewrites +# +# These apply ignore flags and environment overrides for tests that are not +# yet supported or behave differently on ROCm hardware. Kept as a single +# function so new exclusions are easy to add in one place. +############################################################################### + +apply_rocm_test_overrides() { + local cmds="$1" + + # --- Model registry filter --- + if [[ $cmds == *"pytest -v -s models/test_registry.py"* ]]; then + cmds=${cmds//"pytest -v -s models/test_registry.py"/"pytest -v -s models/test_registry.py -k 'not BambaForCausalLM and not GritLM and not Mamba2ForCausalLM and not Zamba2ForCausalLM'"} + fi + + # --- LoRA: disable custom paged attention --- + if [[ $cmds == *"pytest -v -s lora"* ]]; then + cmds=${cmds//"pytest -v -s lora"/"VLLM_ROCM_CUSTOM_PAGED_ATTN=0 pytest -v -s lora"} + fi + + # --- Kernel ignores --- + if [[ $cmds == *" kernels/core"* ]]; then + cmds="${cmds} \ + --ignore=kernels/core/test_fused_quant_layernorm.py \ + --ignore=kernels/core/test_permute_cols.py" + fi + + if [[ $cmds == *" kernels/attention"* ]]; then + cmds="${cmds} \ + --ignore=kernels/attention/test_attention_selector.py \ + --ignore=kernels/attention/test_encoder_decoder_attn.py \ + --ignore=kernels/attention/test_flash_attn.py \ + --ignore=kernels/attention/test_flashinfer.py \ + --ignore=kernels/attention/test_prefix_prefill.py \ + --ignore=kernels/attention/test_cascade_flash_attn.py \ + --ignore=kernels/attention/test_mha_attn.py \ + --ignore=kernels/attention/test_lightning_attn.py \ + --ignore=kernels/attention/test_attention.py" + fi + + if [[ $cmds == *" kernels/quantization"* ]]; then + cmds="${cmds} \ + --ignore=kernels/quantization/test_int8_quant.py \ + --ignore=kernels/quantization/test_machete_mm.py \ + --ignore=kernels/quantization/test_block_fp8.py \ + --ignore=kernels/quantization/test_block_int8.py \ + --ignore=kernels/quantization/test_marlin_gemm.py \ + --ignore=kernels/quantization/test_cutlass_scaled_mm.py \ + --ignore=kernels/quantization/test_int8_kernel.py" + fi + + if [[ $cmds == *" kernels/mamba"* ]]; then + cmds="${cmds} \ + --ignore=kernels/mamba/test_mamba_mixer2.py \ + --ignore=kernels/mamba/test_causal_conv1d.py \ + --ignore=kernels/mamba/test_mamba_ssm_ssd.py" + fi + + if [[ $cmds == *" kernels/moe"* ]]; then + cmds="${cmds} \ + --ignore=kernels/moe/test_moe.py \ + --ignore=kernels/moe/test_cutlass_moe.py \ + --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py" + fi + + # --- Entrypoint ignores --- + if [[ $cmds == *" entrypoints/openai "* ]]; then + cmds=${cmds//" entrypoints/openai "/" entrypoints/openai \ + --ignore=entrypoints/openai/test_audio.py \ + --ignore=entrypoints/openai/test_shutdown.py \ + --ignore=entrypoints/openai/test_completion.py \ + --ignore=entrypoints/openai/test_models.py \ + --ignore=entrypoints/openai/test_lora_adapters.py \ + --ignore=entrypoints/openai/test_return_tokens_as_ids.py \ + --ignore=entrypoints/openai/test_root_path.py \ + --ignore=entrypoints/openai/test_tokenization.py \ + --ignore=entrypoints/openai/test_prompt_validation.py "} + fi + + if [[ $cmds == *" entrypoints/llm "* ]]; then + cmds=${cmds//" entrypoints/llm "/" entrypoints/llm \ + --ignore=entrypoints/llm/test_chat.py \ + --ignore=entrypoints/llm/test_accuracy.py \ + --ignore=entrypoints/llm/test_init.py \ + --ignore=entrypoints/llm/test_prompt_validation.py "} + fi + + # Clean up escaped newlines from --ignore appends + cmds=$(echo "$cmds" | sed 's/ \\ / /g') + + echo "$cmds" +} + +############################################################################### +# Main +############################################################################### + +# --- GPU initialization --- +echo "--- Confirming Clean Initial State" +wait_for_clean_gpus + +echo "--- ROCm info" +rocminfo + +# --- Docker housekeeping --- cleanup_docker echo "--- Resetting GPUs" - echo "reset" > /opt/amdgpu/etc/gpu_state +wait_for_clean_gpus -while true; do - sleep 3 - if grep -q clean /opt/amdgpu/etc/gpu_state; then - echo "GPUs state is \"clean\"" - break - fi -done - +# --- Pull test image --- echo "--- Pulling container" image_name="rocm/vllm-ci:${BUILDKITE_COMMIT}" container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)" docker pull "${image_name}" remove_docker_container() { - docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true + docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true } trap remove_docker_container EXIT +# --- Prepare commands --- echo "--- Running container" HF_CACHE="$(realpath ~)/huggingface" mkdir -p "${HF_CACHE}" HF_MOUNT="/root/.cache/huggingface" -commands=$@ +commands="$*" echo "Raw commands: $commands" -commands=${commands//"pytest -v -s basic_correctness/test_basic_correctness.py"/"pytest -v -s basic_correctness/test_basic_correctness.py"} - -if [[ $commands == *"pytest -v -s models/test_registry.py"* ]]; then - commands=${commands//"pytest -v -s models/test_registry.py"/"pytest -v -s models/test_registry.py -k 'not BambaForCausalLM and not GritLM and not Mamba2ForCausalLM and not Zamba2ForCausalLM'"} -fi - -commands=${commands//"pytest -v -s compile/test_basic_correctness.py"/"pytest -v -s compile/test_basic_correctness.py"} - -if [[ $commands == *"pytest -v -s lora"* ]]; then - commands=${commands//"pytest -v -s lora"/"VLLM_ROCM_CUSTOM_PAGED_ATTN=0 pytest -v -s lora"} -fi - -#ignore certain kernels tests -if [[ $commands == *" kernels/core"* ]]; then - commands="${commands} \ - --ignore=kernels/core/test_fused_quant_layernorm.py \ - --ignore=kernels/core/test_permute_cols.py" -fi - -if [[ $commands == *" kernels/attention"* ]]; then - commands="${commands} \ - --ignore=kernels/attention/test_attention_selector.py \ - --ignore=kernels/attention/test_encoder_decoder_attn.py \ - --ignore=kernels/attention/test_flash_attn.py \ - --ignore=kernels/attention/test_flashinfer.py \ - --ignore=kernels/attention/test_prefix_prefill.py \ - --ignore=kernels/attention/test_cascade_flash_attn.py \ - --ignore=kernels/attention/test_mha_attn.py \ - --ignore=kernels/attention/test_lightning_attn.py \ - --ignore=kernels/attention/test_attention.py" -fi - -if [[ $commands == *" kernels/quantization"* ]]; then - commands="${commands} \ - --ignore=kernels/quantization/test_int8_quant.py \ - --ignore=kernels/quantization/test_machete_mm.py \ - --ignore=kernels/quantization/test_block_fp8.py \ - --ignore=kernels/quantization/test_block_int8.py \ - --ignore=kernels/quantization/test_marlin_gemm.py \ - --ignore=kernels/quantization/test_cutlass_scaled_mm.py \ - --ignore=kernels/quantization/test_int8_kernel.py" -fi - -if [[ $commands == *" kernels/mamba"* ]]; then - commands="${commands} \ - --ignore=kernels/mamba/test_mamba_mixer2.py \ - --ignore=kernels/mamba/test_causal_conv1d.py \ - --ignore=kernels/mamba/test_mamba_ssm_ssd.py" -fi - -if [[ $commands == *" kernels/moe"* ]]; then - commands="${commands} \ - --ignore=kernels/moe/test_moe.py \ - --ignore=kernels/moe/test_cutlass_moe.py \ - --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py" -fi - -#ignore certain Entrypoints/openai tests -if [[ $commands == *" entrypoints/openai "* ]]; then - commands=${commands//" entrypoints/openai "/" entrypoints/openai \ - --ignore=entrypoints/openai/test_audio.py \ - --ignore=entrypoints/openai/test_shutdown.py \ - --ignore=entrypoints/openai/test_completion.py \ - --ignore=entrypoints/openai/test_models.py \ - --ignore=entrypoints/openai/test_lora_adapters.py \ - --ignore=entrypoints/openai/test_return_tokens_as_ids.py \ - --ignore=entrypoints/openai/test_root_path.py \ - --ignore=entrypoints/openai/test_tokenization.py \ - --ignore=entrypoints/openai/test_prompt_validation.py "} -fi - -#ignore certain Entrypoints/llm tests -if [[ $commands == *" entrypoints/llm "* ]]; then - commands=${commands//" entrypoints/llm "/" entrypoints/llm \ - --ignore=entrypoints/llm/test_chat.py \ - --ignore=entrypoints/llm/test_accuracy.py \ - --ignore=entrypoints/llm/test_init.py \ - --ignore=entrypoints/llm/test_prompt_validation.py "} -fi - -commands=$(echo "$commands" | sed 's/ \\ / /g') +# Fix quoting before ROCm overrides (so overrides see correct structure) +commands=$(re_quote_pytest_markers "$commands") +commands=$(apply_rocm_test_overrides "$commands") echo "Final commands: $commands" -# --ignore=entrypoints/openai/test_encoder_decoder.py \ -# --ignore=entrypoints/openai/test_embedding.py \ -# --ignore=entrypoints/openai/test_oot_registration.py -# --ignore=entrypoints/openai/test_accuracy.py \ -# --ignore=entrypoints/openai/test_models.py <= Fails on MI250 but passes on MI300 as of 2025-03-13 - - MYPYTHONPATH=".." -# Test that we're launching on the machine that has -# proper access to GPUs +# Verify GPU access render_gid=$(getent group render | cut -d: -f3) if [[ -z "$render_gid" ]]; then echo "Error: 'render' group not found. This is required for GPU access." >&2 exit 1 fi -if [[ $commands == *"VLLM_TEST_GROUP_NAME=mi325_4-2-node-tests-4-gpus-in-total"* ]]; then - +# --- Route: multi-node vs single-node --- +if is_multi_node "$commands"; then + echo "--- Multi-node job detected" export DCKR_VER=$(docker --version | sed 's/Docker version \(.*\), build .*/\1/') - if [[ "$commands" =~ ^(.*)"["(.*)"] && ["(.*)"]"$ ]]; then - prefix=$( echo "${BASH_REMATCH[1]}" | sed 's/;//g') - echo "PREFIX: ${prefix}" - export composite_command="(command rocm-smi || true)" - myIFS=$IFS - IFS=',' - read -ra node0 <<< ${BASH_REMATCH[2]} - read -ra node1 <<< ${BASH_REMATCH[3]} - IFS=$myIFS - for i in "${!node0[@]}";do - command_node_0=$(echo ${node0[i]} | sed 's/\"//g') - command_node_1=$(echo ${node1[i]} | sed 's/\"//g') - - export commands="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'" - echo "COMMANDS: ${commands}" - composite_command=$(echo "${composite_command} && ${commands}") - done - /bin/bash -c "${composite_command}" - cleanup_network + # Parse the bracket syntax: prefix ; [node0_cmds] && [node1_cmds] + # BASH_REMATCH[1] = prefix (everything before first bracket) + # BASH_REMATCH[2] = comma-separated node0 commands + # BASH_REMATCH[3] = comma-separated node1 commands + if [[ "$commands" =~ ^(.*)\[(.*)"] && ["(.*)\]$ ]]; then + prefix=$(echo "${BASH_REMATCH[1]}" | sed 's/;//g') + echo "PREFIX: ${prefix}" + + export composite_command="(command rocm-smi || true)" + saved_IFS=$IFS + IFS=',' + read -ra node0 <<< "${BASH_REMATCH[2]}" + read -ra node1 <<< "${BASH_REMATCH[3]}" + IFS=$saved_IFS + + if [[ ${#node0[@]} -ne ${#node1[@]} ]]; then + echo "Warning: node0 has ${#node0[@]} commands, node1 has ${#node1[@]}. They will be paired by index." + fi + + for i in "${!node0[@]}"; do + command_node_0=$(echo "${node0[i]}" | sed 's/\"//g') + command_node_1=$(echo "${node1[i]}" | sed 's/\"//g') + + step_cmd="./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 ${image_name} '${command_node_0}' '${command_node_1}'" + echo "COMMANDS: ${step_cmd}" + composite_command="${composite_command} && ${step_cmd}" + done + + /bin/bash -c "${composite_command}" + cleanup_network else - echo "Failed to parse node commands! Exiting." - cleanup_network - exit 111 + echo "Multi-node job detected but failed to parse bracket command syntax." + echo "Expected format: prefix ; [node0_cmd1, node0_cmd2] && [node1_cmd1, node1_cmd2]" + echo "Got: $commands" + cleanup_network + exit 111 fi else + echo "--- Single-node job" echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES" docker run \ - --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ - --network=host \ - --shm-size=16gb \ - --group-add "$render_gid" \ - --rm \ - -e HF_TOKEN \ - -e AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY \ - -v "${HF_CACHE}:${HF_MOUNT}" \ - -e "HF_HOME=${HF_MOUNT}" \ - -e "PYTHONPATH=${MYPYTHONPATH}" \ - --name "${container_name}" \ - "${image_name}" \ - /bin/bash -c "${commands}" + --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \ + --network=host \ + --shm-size=16gb \ + --group-add "$render_gid" \ + --rm \ + -e HF_TOKEN \ + -e AWS_ACCESS_KEY_ID \ + -e AWS_SECRET_ACCESS_KEY \ + -v "${HF_CACHE}:${HF_MOUNT}" \ + -e "HF_HOME=${HF_MOUNT}" \ + -e "PYTHONPATH=${MYPYTHONPATH}" \ + --name "${container_name}" \ + "${image_name}" \ + /bin/bash -c "${commands}" fi diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 1ccc823ef60..ffdf4b83c0e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -340,6 +340,11 @@ steps: - vllm/ - tests/v1/tracing commands: + - "pip install \ + 'opentelemetry-sdk>=1.26.0' \ + 'opentelemetry-api>=1.26.0' \ + 'opentelemetry-exporter-otlp>=1.26.0' \ + 'opentelemetry-semantic-conventions-ai>=0.4.1'" - pytest -v -s v1/tracing ##### fast check tests ##### @@ -1958,6 +1963,11 @@ steps: - vllm/ - tests/v1/tracing commands: + - "pip install \ + 'opentelemetry-sdk>=1.26.0' \ + 'opentelemetry-api>=1.26.0' \ + 'opentelemetry-exporter-otlp>=1.26.0' \ + 'opentelemetry-semantic-conventions-ai>=0.4.1'" - pytest -v -s v1/tracing ##### fast check tests ##### diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index df748a5fcac..9b5b002f4b7 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -145,7 +145,7 @@ steps: num_devices: 2 commands: - pytest -v -s tests/distributed/test_context_parallel.py - - cd examples/offline_inference/new_weight_syncing && VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 rlhf_async_new_apis.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/offline_inference/new_weight_syncing/rlhf_async_new_apis.py - VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput - pytest -v -s tests/v1/distributed/test_dbo.py diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 82ce2f42005..4f2380592d9 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -28,3 +28,11 @@ steps: - pytest -v -s v1/engine/test_preprocess_error_handling.py # Run the rest of v1/engine tests - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py + mirror: + amd: + device: mi325_8 + depends_on: + - image-build-amd + commands: + - pytest -v -s v1/e2e + - pytest -v -s v1/engine diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 6aebb9aabe3..5c58e97ef16 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -24,11 +24,6 @@ steps: - pytest -v -s entrypoints/llm --ignore=entrypoints/llm/test_generate.py --ignore=entrypoints/llm/test_collective_rpc.py - pytest -v -s entrypoints/llm/test_generate.py # it needs a clean process - pytest -v -s entrypoints/offline_mode # Needs to avoid interference with other tests - mirror: - amd: - device: mi325_1 - depends_on: - - image-build-amd - label: Entrypoints Integration (API Server 1) timeout_in_minutes: 130 @@ -65,6 +60,11 @@ steps: commands: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -v -s entrypoints/pooling + mirror: + amd: + device: mi325_1 + depends_on: + - image-build-amd - label: Entrypoints Integration (Responses API) timeout_in_minutes: 50 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 3f43b8d429a..afc8fc49a2a 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -115,6 +115,7 @@ steps: - pytest -v -s tests/kernels/moe/test_nvfp4_moe.py - pytest -v -s tests/kernels/moe/test_ocp_mx_moe.py - pytest -v -s tests/kernels/moe/test_flashinfer.py + - pytest -v -s tests/kernels/moe/test_flashinfer_moe.py - pytest -v -s tests/kernels/moe/test_cutedsl_moe.py # e2e - pytest -v -s tests/models/quantization/test_nvfp4.py @@ -156,14 +157,3 @@ steps: - pytest -v -s kernels/moe/test_deepep_moe.py - pytest -v -s kernels/moe/test_pplx_cutlass_moe.py # - pytest -v -s kernels/moe/test_pplx_moe.py - failing on main - -- label: Kernels Fp4 MoE Test (B200) - timeout_in_minutes: 60 - device: b200 - num_devices: 1 - optional: true - commands: - - pytest -v -s kernels/moe/test_cutedsl_moe.py - - pytest -v -s kernels/moe/test_flashinfer_moe.py - - pytest -v -s kernels/moe/test_nvfp4_moe.py - - pytest -v -s kernels/moe/test_ocp_mx_moe.py diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index c2e916164b3..5c5a9dbcbb6 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -16,6 +16,7 @@ steps: - pytest -v -s v1/sample - pytest -v -s v1/logits_processors - pytest -v -s v1/worker + # TODO: create another `optional` test group for slow tests - pytest -v -s -m 'not slow_test' v1/spec_decode - pytest -v -s -m 'not cpu_test' v1/kv_connector/unit - pytest -v -s -m 'not cpu_test' v1/metrics @@ -25,6 +26,11 @@ steps: # Integration test for streaming correctness (requires special branch). - pip install -U git+https://github.com/robertgshaw2-redhat/lm-evaluation-harness.git@streaming-api - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine + mirror: + amd: + device: mi325_1 + depends_on: + - image-build-amd - label: V1 Others (CPU) depends_on: @@ -88,6 +94,11 @@ steps: - vllm/ - tests/v1/tracing commands: + - "pip install \ + 'opentelemetry-sdk>=1.26.0' \ + 'opentelemetry-api>=1.26.0' \ + 'opentelemetry-exporter-otlp>=1.26.0' \ + 'opentelemetry-semantic-conventions-ai>=0.4.1'" - pytest -v -s v1/tracing - label: Python-only Installation diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 8982dccc4de..a3bd21ccff3 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -55,6 +55,15 @@ steps: - uv pip install --system --no-build-isolation 'git+https://github.com/state-spaces/mamba@v2.3.0' - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2' - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' + mirror: + amd: + device: mi325_1 + depends_on: + - image-build-amd + commands: + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.5.2' + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' - label: Language Models Test (PPL) timeout_in_minutes: 110 @@ -73,6 +82,11 @@ steps: - tests/models/language/pooling commands: - pytest -v -s models/language/pooling -m 'not core_model' + mirror: + amd: + device: mi325_1 + depends_on: + - image-build-amd - label: Language Models Test (MTEB) timeout_in_minutes: 110 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 315d6435436..adf50a185e5 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,14 +5,14 @@ /vllm/compilation @zou3519 @youkaichao @ProExpertProg /vllm/distributed/kv_transfer @NickLucche @ApostaC @orozery /vllm/lora @jeejeelee -/vllm/model_executor/layers/attention @LucasWilkinson +/vllm/model_executor/layers/attention @LucasWilkinson @MatthewBonanni /vllm/model_executor/layers/fused_moe @mgoin @pavanimajety /vllm/model_executor/layers/quantization @mgoin @robertgshaw2-redhat @tlrmchlsmth @yewentao256 @pavanimajety /vllm/model_executor/layers/mamba @tdoublep /vllm/model_executor/model_loader @22quinn /vllm/model_executor/layers/batch_invariant.py @yewentao256 /vllm/multimodal @DarkLight1337 @ywang96 @NickLucche @tjtanaa -/vllm/vllm_flash_attn @LucasWilkinson +/vllm/vllm_flash_attn @LucasWilkinson @MatthewBonanni CMakeLists.txt @tlrmchlsmth @LucasWilkinson # Any change to the VllmConfig changes can have a large user-facing impact, @@ -43,14 +43,14 @@ CMakeLists.txt @tlrmchlsmth @LucasWilkinson /vllm/tool_parsers @aarnphm @chaunceyjiang # vLLM V1 -/vllm/v1/attention @LucasWilkinson +/vllm/v1/attention @LucasWilkinson @MatthewBonanni /vllm/v1/attention/backend.py @WoosukKwon @zhuohan123 @youkaichao @alexm-redhat @njhill /vllm/v1/attention/backends/mla @pavanimajety /vllm/v1/attention/backends/flashinfer.py @mgoin @pavanimajety /vllm/v1/attention/backends/triton_attn.py @tdoublep /vllm/v1/core @WoosukKwon @robertgshaw2-redhat @njhill @ywang96 @alexm-redhat @heheda12345 @ApostaC @orozery /vllm/v1/sample @22quinn @houseroad @njhill -/vllm/v1/spec_decode @benchislett @luccafong +/vllm/v1/spec_decode @benchislett @luccafong @MatthewBonanni /vllm/v1/structured_output @mgoin @russellb @aarnphm @benchislett /vllm/v1/kv_cache_interface.py @heheda12345 /vllm/v1/kv_offload @ApostaC @orozery diff --git a/csrc/cpu/cpu_attn.cpp b/csrc/cpu/cpu_attn.cpp index 641f95a2b1d..a582b4b4d7c 100644 --- a/csrc/cpu/cpu_attn.cpp +++ b/csrc/cpu/cpu_attn.cpp @@ -16,6 +16,8 @@ torch::Tensor get_scheduler_metadata( isa = cpu_attention::ISA::VEC16; } else if (isa_hint == "neon") { isa = cpu_attention::ISA::NEON; + } else if (isa_hint == "vxe") { + isa = cpu_attention::ISA::VXE; } else { TORCH_CHECK(false, "Unsupported CPU attention ISA hint: " + isa_hint); } @@ -100,6 +102,8 @@ void cpu_attn_reshape_and_cache( return cpu_attention::ISA::VEC16; } else if (isa == "neon") { return cpu_attention::ISA::NEON; + } else if (isa == "vxe") { + return cpu_attention::ISA::VXE; } else { TORCH_CHECK(false, "Invalid ISA type: " + isa); } diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp index fbe0e8778d8..c15799fa950 100644 --- a/csrc/cpu/cpu_attn_impl.hpp +++ b/csrc/cpu/cpu_attn_impl.hpp @@ -12,7 +12,7 @@ #include "cpu/utils.hpp" namespace cpu_attention { -enum class ISA { AMX, VEC, VEC16, NEON }; +enum class ISA { AMX, VEC, VEC16, NEON, VXE }; template class AttentionImpl {}; diff --git a/csrc/cpu/cpu_attn_vxe.hpp b/csrc/cpu/cpu_attn_vxe.hpp new file mode 100644 index 00000000000..45db4ebd739 --- /dev/null +++ b/csrc/cpu/cpu_attn_vxe.hpp @@ -0,0 +1,386 @@ +#ifndef CPU_ATTN_VXE_HPP +#define CPU_ATTN_VXE_HPP + +#include "cpu_attn_impl.hpp" +#include +#include + +namespace cpu_attention { + +namespace { + +// s390x Vector = 16 bytes (128 bits) +#define BLOCK_SIZE_ALIGNMENT 32 +#define HEAD_SIZE_ALIGNMENT 32 +#define MAX_Q_HEAD_NUM_PER_ITER 16 + +template +FORCE_INLINE void load_row8_B_as_f32(const kv_cache_t* p, __vector float& b0, + __vector float& b1); + +// [1] Float Specialization +template <> +FORCE_INLINE void load_row8_B_as_f32(const float* p, __vector float& b0, + __vector float& b1) { + // Explicitly cast to long long for offset, and float* for pointer + b0 = vec_xl((long long)0, const_cast(p)); + b1 = vec_xl((long long)0, const_cast(p + 4)); +} + +// [2] BFloat16 Specialization (Big Endian Fix) +template <> +FORCE_INLINE void load_row8_B_as_f32(const c10::BFloat16* p, + __vector float& b0, + __vector float& b1) { + // 1. Load 8 BF16s (16 bytes) into one vector + // Explicit cast to unsigned short* for vec_xl to return vector unsigned short + __vector unsigned short raw = vec_xl((long long)0, (unsigned short*)p); + + // 2. Prepare Zero vector + __vector unsigned short zeros = vec_splat_u16(0); + + // 3. Merge High/Low to expand BF16 -> Float32 + // On Big Endian, a float is [BF16_bits | 16_zero_bits] + b0 = (__vector float)vec_mergeh(raw, zeros); + b1 = (__vector float)vec_mergel(raw, zeros); +} + +template <> +FORCE_INLINE void load_row8_B_as_f32(const c10::Half* p, + __vector float& b0, + __vector float& b1) { + alignas(16) float tmp[8]; + + // Manual unroll / conversion + tmp[0] = static_cast(p[0]); + tmp[1] = static_cast(p[1]); + tmp[2] = static_cast(p[2]); + tmp[3] = static_cast(p[3]); + tmp[4] = static_cast(p[4]); + tmp[5] = static_cast(p[5]); + tmp[6] = static_cast(p[6]); + tmp[7] = static_cast(p[7]); + + // Explicit arguments for intrinsic: (long long offset, float* ptr) + b0 = vec_xl((long long)0, (float*)tmp); + b1 = vec_xl((long long)0, (float*)(tmp + 4)); +} + +template +FORCE_INLINE void gemm_micro_s390x_Mx8_Ku4( + const float* __restrict A, // [M x K] + const kv_cache_t* __restrict B, // [K x 8] + float* __restrict C, // [M x 8] + int64_t lda, int64_t ldb, int64_t ldc, int32_t K, bool accumulate) { + static_assert(1 <= M && M <= 8, "M must be in [1,8]"); + +// Helper macros to unroll codegen for M rows +#define ROWS_APPLY(OP) OP(0) OP(1) OP(2) OP(3) OP(4) OP(5) OP(6) OP(7) +#define IF_M(i) if constexpr (M > (i)) + + // 1. Define A pointers +#define DECL_A(i) const float* a##i = A + (i) * lda; + ROWS_APPLY(DECL_A) +#undef DECL_A + + // 2. Define Accumulators (2 vectors covers 8 columns) +#define DECL_ACC(i) __vector float acc##i##_0, acc##i##_1; + ROWS_APPLY(DECL_ACC) +#undef DECL_ACC + + // 3. Initialize Accumulators (Load C or Zero) +#define INIT_ACC(i) \ + IF_M(i) { \ + if (accumulate) { \ + acc##i##_0 = \ + vec_xl((long long)0, const_cast(C + (i) * ldc + 0)); \ + acc##i##_1 = \ + vec_xl((long long)0, const_cast(C + (i) * ldc + 4)); \ + } else { \ + acc##i##_0 = vec_splats(0.0f); \ + acc##i##_1 = vec_splats(0.0f); \ + } \ + } + ROWS_APPLY(INIT_ACC) +#undef INIT_ACC + + int32_t k = 0; + + for (; k + 3 < K; k += 4) { + // Load 4 values of A for each Row M: A[k...k+3] +#define LOAD_A4(i) \ + __vector float a##i##v; \ + IF_M(i) a##i##v = vec_xl((long long)0, const_cast(a##i + k)); + ROWS_APPLY(LOAD_A4) +#undef LOAD_A4 + + // Helper: FMA for specific lane L of A + // s390x: vec_madd(b, vec_splat(a, lane), acc) +#define FMAS_LANE(i, aiv, L) \ + IF_M(i) { \ + __vector float a_broad = vec_splat(aiv, L); \ + acc##i##_0 = vec_madd(b0, a_broad, acc##i##_0); \ + acc##i##_1 = vec_madd(b1, a_broad, acc##i##_1); \ + } + + // Unroll K=0..3 + { + __vector float b0, b1; + load_row8_B_as_f32(B + (int64_t)(k + 0) * ldb, b0, b1); +#define STEP_K0(i) FMAS_LANE(i, a##i##v, 0) + ROWS_APPLY(STEP_K0) +#undef STEP_K0 + } + { + __vector float b0, b1; + load_row8_B_as_f32(B + (int64_t)(k + 1) * ldb, b0, b1); +#define STEP_K1(i) FMAS_LANE(i, a##i##v, 1) + ROWS_APPLY(STEP_K1) +#undef STEP_K1 + } + { + __vector float b0, b1; + load_row8_B_as_f32(B + (int64_t)(k + 2) * ldb, b0, b1); +#define STEP_K2(i) FMAS_LANE(i, a##i##v, 2) + ROWS_APPLY(STEP_K2) +#undef STEP_K2 + } + + { + __vector float b0, b1; + load_row8_B_as_f32(B + (int64_t)(k + 3) * ldb, b0, b1); +#define STEP_K3(i) FMAS_LANE(i, a##i##v, 3) + ROWS_APPLY(STEP_K3) +#undef STEP_K3 + } +#undef FMAS_LANE + } + + for (; k < K; ++k) { + __vector float b0, b1; + load_row8_B_as_f32(B + (int64_t)k * ldb, b0, b1); +#define TAIL_ROW(i) \ + IF_M(i) { \ + __vector float ai = vec_splats(*(a##i + k)); \ + acc##i##_0 = vec_madd(b0, ai, acc##i##_0); \ + acc##i##_1 = vec_madd(b1, ai, acc##i##_1); \ + } + ROWS_APPLY(TAIL_ROW) +#undef TAIL_ROW + } + +#define STORE_ROW(i) \ + IF_M(i) { \ + vec_xst(acc##i##_0, 0, C + (i) * ldc + 0); \ + vec_xst(acc##i##_1, 0, C + (i) * ldc + 4); \ + } + ROWS_APPLY(STORE_ROW) +#undef STORE_ROW + +#undef ROWS_APPLY +#undef IF_M +} + +template +FORCE_INLINE void gemm_macro_s390x_Mx8_Ku4(const float* __restrict A, + const kv_cache_t* __restrict B, + float* __restrict C, int32_t M, + int32_t K, int64_t lda, int64_t ldb, + int64_t ldc, bool accumulate) { + static_assert(N % 8 == 0, "N must be a multiple of 8"); + for (int32_t m = 0; m < M;) { + int32_t mb = (M - m >= 8) ? 8 : (M - m >= 4) ? 4 : (M - m >= 2) ? 2 : 1; + const float* Ab = A + m * lda; + float* Cb = C + m * ldc; + + for (int32_t n = 0; n < N; n += 8) { + const kv_cache_t* Bn = B + n; + float* Cn = Cb + n; + switch (mb) { + case 8: + gemm_micro_s390x_Mx8_Ku4<8, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, K, + accumulate); + break; + case 4: + gemm_micro_s390x_Mx8_Ku4<4, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, K, + accumulate); + break; + case 2: + gemm_micro_s390x_Mx8_Ku4<2, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, K, + accumulate); + break; + default: + gemm_micro_s390x_Mx8_Ku4<1, kv_cache_t>(Ab, Bn, Cn, lda, ldb, ldc, K, + accumulate); + break; + } + } + m += mb; + } +} + +template +class TileGemmS390X { + public: + template + FORCE_INLINE static void gemm(const int32_t m_size, + float* __restrict__ a_tile, + kv_cache_t* __restrict__ b_tile, + float* __restrict__ c_tile, const int64_t lda, + const int64_t ldb, const int64_t ldc, + const int32_t block_size, + const int32_t dynamic_k_size, + const bool accum_c) { + if constexpr (phase == AttentionGemmPhase::QK) { + gemm_macro_s390x_Mx8_Ku4( + a_tile, b_tile, c_tile, m_size, k_size, lda, ldb, ldc, accum_c); + } else { + gemm_macro_s390x_Mx8_Ku4( + a_tile, b_tile, c_tile, m_size, dynamic_k_size, lda, ldb, ldc, + accum_c); + } + } +}; + +} // namespace + +template +class AttentionImpl { + public: + using query_t = scalar_t; + using q_buffer_t = float; + using kv_cache_t = scalar_t; + using logits_buffer_t = float; + using partial_output_buffer_t = float; + using prob_buffer_t = float; + + constexpr static int64_t BlockSizeAlignment = BLOCK_SIZE_ALIGNMENT; + constexpr static int64_t HeadDimAlignment = HEAD_SIZE_ALIGNMENT; + constexpr static int64_t MaxQHeadNumPerIteration = MAX_Q_HEAD_NUM_PER_ITER; + constexpr static int64_t HeadDim = head_dim; + constexpr static ISA ISAType = ISA::VXE; + constexpr static bool scale_on_logits = + false; // Scale is applied to Q during copy + + public: + AttentionImpl() {} + + template