diff --git a/.buildkite/intel_jobs/model_runner_v2_intel.yaml b/.buildkite/intel_jobs/model_runner_v2_intel.yaml new file mode 100644 index 00000000000..67ce57ebd75 --- /dev/null +++ b/.buildkite/intel_jobs/model_runner_v2_intel.yaml @@ -0,0 +1,54 @@ +group: Model Runner V2 Intel +depends_on: + - image-build-xpu +steps: +- label: Model Runner V2 Core Tests (Intel) + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - vllm/v1/core/sched/ + - vllm/v1/attention/ + - tests/v1/engine/test_llm_engine.py + - tests/v1/e2e/ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_USE_V2_MODEL_RUNNER=1 && + cd tests && + pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" && + ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" && + pytest -v -s v1/e2e/general/test_min_tokens.py' + +- label: Model Runner V2 Examples (Intel) + timeout_in_minutes: 45 + device: intel_gpu + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/core/sched/ + - vllm/v1/worker/gpu_worker.py + - examples/basic/offline_inference/ + - examples/generate/multimodal/ + - examples/features/ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_USE_V2_MODEL_RUNNER=1 && + cd examples && + python3 basic/offline_inference/chat.py && + python3 basic/offline_inference/generate.py --model facebook/opt-125m && + python3 generate/multimodal/vision_language_offline.py --seed 0 && + python3 features/automatic_prefix_caching/prefix_caching_offline.py' diff --git a/.buildkite/scripts/hardware_ci/run-intel-test.sh b/.buildkite/scripts/hardware_ci/run-intel-test.sh index 0eb93f5a6b3..246ea7de50e 100755 --- a/.buildkite/scripts/hardware_ci/run-intel-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-test.sh @@ -243,8 +243,10 @@ container_name="xpu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head # ---- Command source selection ---- commands="" +commands_source="" if [[ -n "${VLLM_TEST_COMMANDS:-}" ]]; then commands="${VLLM_TEST_COMMANDS}" + commands_source="env" echo "Commands sourced from VLLM_TEST_COMMANDS (quoting preserved)" elif [[ $# -gt 0 ]]; then all_yaml=true @@ -303,8 +305,12 @@ if [[ -z "$commands" ]]; then fi echo "Raw commands: $commands" -commands=$(re_quote_pytest_markers "$commands") -echo "After re-quoting: $commands" +if [[ "$commands_source" != "env" ]]; then + commands=$(re_quote_pytest_markers "$commands") + echo "After re-quoting: $commands" +else + echo "Skipping re-quoting for VLLM_TEST_COMMANDS input" +fi commands=$(apply_intel_test_overrides "$commands") echo "Final commands: $commands" diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 9f6d4e5a75c..d1470029216 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -238,7 +238,17 @@ if flashinfer_comm is not None: use_oneshot=use_oneshot, fp32_acc=fp32_acc, weight_bias=weight_bias, - trigger_completion_at_end=num_tokens > PDL_ADVANCE_LAUNCH_TOKENS, + # The one-shot Lamport all-reduce signals PDL completion before its + # output buffer is committed when trigger_completion_at_end is + # False, so the next PDL-launched kernel can read the uninitialized + # Lamport buffer and produce NaN. This only fires for + # num_tokens <= PDL_ADVANCE_LAUNCH_TOKENS (the batch=1 / spec-decode + # shapes, where the one-shot path is always selected). Complete at + # the end for the one-shot path; the two-shot path is synchronized + # and keeps the early completion. Related one-shot instability in + # the same kernel: flashinfer-ai/flashinfer#1223. + trigger_completion_at_end=use_oneshot + or num_tokens > PDL_ADVANCE_LAUNCH_TOKENS, ) def call_trtllm_fused_allreduce_norm_fake( diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 052874fac03..29302584880 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -14,7 +14,7 @@ import torch.nn.functional as F from transformers import DeepseekV2Config, DeepseekV3Config import vllm.envs as envs -from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture +from vllm.compilation.breakable_cudagraph import eager_break_during_capture from vllm.model_executor.layers.linear import ( ColumnParallelLinear, MergedColumnParallelLinear, @@ -331,8 +331,8 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): ) # Metadata-independent input GEMMs + RMSNorm stay in the captured - # graph. For C4A layers, the inner sparse_attn_indexer custom op - # runs in the eager break. + # graph; the metadata-dependent rest (q up-proj + kv-insert, indexer, + # compressor, MLA attention) runs in the eager break. qr_kv, kv_score, indexer_kv_score, indexer_weights = ( self.attn_gemm_parallel_execute(hidden_states) ) @@ -345,6 +345,9 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): self.eps, ) + # attention_impl is wrapped with @eager_break_during_capture: this is + # where the breakable cudagraph capture breaks (the attention op runs + # eagerly between captured graph segments). self.attention_impl( hidden_states, qr, @@ -420,6 +423,7 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): return qr_kv, kv_score, indexer_kv_score, indexer_weights + @eager_break_during_capture def attention_impl( self, hidden_states: torch.Tensor, @@ -447,42 +451,31 @@ class DeepseekV4Attention(nn.Module, AttentionLayerBase, ABC): def wq_b_kv_insert() -> torch.Tensor: q = self.wq_b(qr).view(-1, self.n_local_heads, self.head_dim) - return self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata) + q = self._fused_qnorm_rope_kv_insert(q, kv, positions, attn_metadata) + return q - run_indexer = lambda: indexer( - hidden_states, - qr, - indexer_kv_score, - indexer_weights, - positions, - self.indexer_rotary_emb, + # 3-way overlap (matches TRT-LLM PR #14142 Level 1): default runs + # wq_b+kv_insert; slot [0] runs the full indexer; slot [1] runs the + # MLA compressor. Slot [2] is reserved for the indexer's inner + # overlap. ROCm (aux_streams is None) falls back to sequential. + q, _ = execute_in_parallel( + wq_b_kv_insert, + [ + lambda: indexer( + hidden_states, + qr, + indexer_kv_score, + indexer_weights, + positions, + self.indexer_rotary_emb, + ), + lambda: compressor(kv_score, positions, self.rotary_emb), + ], + self.ln_events[0], + [self.ln_events[1], self.ln_events[2]], + [aux_streams[0], aux_streams[1]] if aux_streams is not None else None, + enable=aux_streams is not None, ) - run_compressor = lambda: compressor(kv_score, positions, self.rotary_emb) - - if BreakableCUDAGraphCapture.is_active(): - q, _ = maybe_execute_in_parallel( - wq_b_kv_insert, - run_compressor, - self.ln_events[0], - self.ln_events[1], - aux_streams[1] if aux_streams is not None else None, - ) - run_indexer() - else: - # 3-way overlap (matches TRT-LLM PR #14142 Level 1): default runs - # wq_b+kv_insert; slot [0] runs the full indexer; slot [1] runs the - # MLA compressor. Slot [2] is reserved for the indexer's inner - # overlap. ROCm (aux_streams is None) falls back to sequential. - q, _ = execute_in_parallel( - wq_b_kv_insert, - [run_indexer, run_compressor], - self.ln_events[0], - [self.ln_events[1], self.ln_events[2]], - [aux_streams[0], aux_streams[1]] - if aux_streams is not None - else None, - enable=aux_streams is not None, - ) elif self.compressor is not None: # wq_b + kv_insert on default, compressor on aux. aux_stream = ( diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py index c292c2b6cb4..ed441b380f0 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -5,6 +5,7 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_pp_group +from vllm.lora.layers.base import BaseLayerWithLoRA from vllm.model_executor.model_loader import get_model @@ -48,6 +49,13 @@ def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Mod target_embed = getattr(target_inner, "embed_tokens", None) or getattr( target_inner, "embedding", None ) + # If the target's embedding is LoRA-wrapped, share the underlying base + # layer. The draft is not part of the LoRA adapter; sharing the wrapper + # would make the draft run the LoRA embedding kernel with the target's + # punica metadata (sized for the target's token count), causing an + # out-of-bounds GPU access during multi-step draft decode. + if isinstance(target_embed, BaseLayerWithLoRA): + target_embed = target_embed.base_layer draft_embed = getattr(draft_inner, "embed_tokens", None) if target_embed is not None and _should_share( eagle_model, "has_own_embed_tokens", draft_embed, target_embed