From 6d85b36a9fa2ac979ad53903358603d2820bf207 Mon Sep 17 00:00:00 2001 From: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Date: Tue, 21 Apr 2026 19:44:11 +0400 Subject: [PATCH 001/153] Revert #38730 and #38791 (#40032) Signed-off-by: Vadim Gimpelson Signed-off-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> --- .../kernels/attention/test_use_trtllm_attention.py | 4 ++-- .../pre_commit/generate_attention_backend_docs.py | 14 +++++--------- vllm/utils/flashinfer.py | 1 + 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/tests/kernels/attention/test_use_trtllm_attention.py b/tests/kernels/attention/test_use_trtllm_attention.py index a58a650fdda..fba18fe46e3 100644 --- a/tests/kernels/attention/test_use_trtllm_attention.py +++ b/tests/kernels/attention/test_use_trtllm_attention.py @@ -72,7 +72,7 @@ def test_supports_sm100_with_artifactory(_art, _cap): @patch("vllm.envs.VLLM_BATCH_INVARIANT", False) @patch( - "vllm.utils.flashinfer.current_platform.is_device_capability", + "vllm.utils.flashinfer.current_platform.is_device_capability_family", return_value=False, ) def test_supports_non_sm100_platform(_cap): @@ -81,7 +81,7 @@ def test_supports_non_sm100_platform(_cap): @patch("vllm.envs.VLLM_BATCH_INVARIANT", False) @patch( - "vllm.utils.flashinfer.current_platform.is_device_capability", + "vllm.utils.flashinfer.current_platform.is_device_capability_family", return_value=True, ) @patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=False) diff --git a/tools/pre_commit/generate_attention_backend_docs.py b/tools/pre_commit/generate_attention_backend_docs.py index 9e14f8739dc..bbbf4f4b64f 100644 --- a/tools/pre_commit/generate_attention_backend_docs.py +++ b/tools/pre_commit/generate_attention_backend_docs.py @@ -235,11 +235,10 @@ def _resolve_import_to_file( def _find_cc_in_function(tree: ast.AST, func_name: str) -> str | None: - """Find a compute capability from is_device_capability*() calls in a function. + """Find a compute capability from is_device_capability_family() calls in a function. - Handles two patterns: - - is_device_capability_family(N): "M.x" (e.g. 100 -> "10.x") - - is_device_capability(N): "M.m" (e.g. 100 -> "10.0") + Looks for the pattern: current_platform.is_device_capability_family(N) + and converts N (e.g. 100) to a CC string (e.g. "10.x"). """ for node in ast.walk(tree): if not isinstance(node, ast.FunctionDef) or node.name != func_name: @@ -248,15 +247,12 @@ def _find_cc_in_function(tree: ast.AST, func_name: str) -> str | None: if ( isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) + and n.func.attr == "is_device_capability_family" and n.args and isinstance(n.args[0], ast.Constant) and isinstance(n.args[0].value, int) ): - val = n.args[0].value - if n.func.attr == "is_device_capability_family": - return f"{val // 10}.x" - elif n.func.attr == "is_device_capability": - return f"{val // 10}.{val % 10}" + return f"{n.args[0].value // 10}.x" return None diff --git a/vllm/utils/flashinfer.py b/vllm/utils/flashinfer.py index cd54a06c5ab..316816d9658 100644 --- a/vllm/utils/flashinfer.py +++ b/vllm/utils/flashinfer.py @@ -305,6 +305,7 @@ def supports_trtllm_attention() -> bool: if envs.VLLM_BATCH_INVARIANT: return False + # Requires SM100 and NVIDIA artifactory to be accessible to download cubins return ( current_platform.is_device_capability_family(100) and has_nvidia_artifactory() ) From 66cc3fa559d71bb4ae2335c26c01115791968099 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 21 Apr 2026 11:49:05 -0400 Subject: [PATCH 002/153] [Model Runner V2] Multiple prompt logprobs support (#39937) Signed-off-by: yewentao256 Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill --- vllm/v1/worker/gpu/sample/prompt_logprob.py | 66 +++++++++++++++------ 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/vllm/v1/worker/gpu/sample/prompt_logprob.py b/vllm/v1/worker/gpu/sample/prompt_logprob.py index 1915a053979..11dbf698527 100644 --- a/vllm/v1/worker/gpu/sample/prompt_logprob.py +++ b/vllm/v1/worker/gpu/sample/prompt_logprob.py @@ -17,13 +17,14 @@ class PromptLogprobsWorker: self.max_num_reqs = max_num_reqs self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool) + self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32) # req_idx -> list of in-progress LogprobsTensors self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {} def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams): - # For now, only support prompt logprobs for the prompt tokens (not top-k). uses_prompt_logprobs = sampling_params.prompt_logprobs is not None self.uses_prompt_logprobs[req_idx] = uses_prompt_logprobs + self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0 if uses_prompt_logprobs: self.in_progress_prompt_logprobs[req_id] = [] @@ -52,6 +53,7 @@ class PromptLogprobsWorker: # Common case: No request asks for prompt logprobs. return {} + num_prompt_logprobs = self.num_prompt_logprobs[idx_mapping_np] prompt_lens = prompt_lens[idx_mapping_np] # NOTE(woosuk): -1 because the last prompt token's hidden state is not # needed for prompt logprobs. @@ -64,6 +66,14 @@ class PromptLogprobsWorker: if not np.any(needs_prompt_logprobs): return {} + # get the maximum number in this batch + requested_num_prompt_logprobs = num_prompt_logprobs[needs_prompt_logprobs] + max_num_prompt_logprobs = ( + -1 + if np.any(requested_num_prompt_logprobs == -1) + else int(requested_num_prompt_logprobs.max()) + ) + # Get the prompt logprobs token_ids. prompt_logprobs_token_ids = get_prompt_logprobs_token_ids( input_batch.num_tokens, @@ -72,45 +82,53 @@ class PromptLogprobsWorker: num_computed_tokens, all_token_ids, ) - # Compute the prompt logprobs. - prompt_logprobs, prompt_ranks = compute_prompt_logprobs_with_chunking( - prompt_logprobs_token_ids, - hidden_states[: input_batch.num_tokens], - logits_fn, + prompt_token_ids, prompt_logprobs, prompt_ranks = ( + compute_prompt_logprobs_with_chunking( + prompt_logprobs_token_ids, + hidden_states[: input_batch.num_tokens], + logits_fn, + max_num_prompt_logprobs, + ) ) pos_after_step = computed_prefill + input_batch.num_scheduled_tokens is_prompt_chunked = pos_after_step < prompt_lens query_start_loc_np = input_batch.query_start_loc_np - prompt_token_ids = prompt_logprobs_token_ids.unsqueeze(-1) prompt_logprobs_dict: dict[str, LogprobsTensors] = {} for i, req_id in enumerate(input_batch.req_ids): if not needs_prompt_logprobs[i]: continue + req_is_prompt_chunked = is_prompt_chunked[i] start_idx = query_start_loc_np[i] end_idx = query_start_loc_np[i + 1] assert start_idx < end_idx, ( f"start_idx ({start_idx}) >= end_idx ({end_idx})" ) - if not is_prompt_chunked[i]: + if not req_is_prompt_chunked: end_idx -= 1 - logprobs = LogprobsTensors( - logprob_token_ids=prompt_token_ids[start_idx:end_idx], - logprobs=prompt_logprobs[start_idx:end_idx], - selected_token_ranks=prompt_ranks[start_idx:end_idx], + + # no logprobs if start_idx >= end_idx + logprobs = ( + None + if start_idx >= end_idx + else LogprobsTensors( + logprob_token_ids=prompt_token_ids[start_idx:end_idx], + logprobs=prompt_logprobs[start_idx:end_idx], + selected_token_ranks=prompt_ranks[start_idx:end_idx], + ) ) prompt_logprobs_list = self.in_progress_prompt_logprobs[req_id] - if is_prompt_chunked[i]: - # Prompt is chunked. Do not return the logprobs yet. + if logprobs is not None and (req_is_prompt_chunked or prompt_logprobs_list): prompt_logprobs_list.append(logprobs) + if req_is_prompt_chunked: + # Prompt is chunked. Do not return the logprobs yet. continue if prompt_logprobs_list: # Merge the in-progress logprobs. - prompt_logprobs_list.append(logprobs) logprobs = LogprobsTensors( logprob_token_ids=torch.cat( [x.logprob_token_ids for x in prompt_logprobs_list] @@ -122,6 +140,9 @@ class PromptLogprobsWorker: ) prompt_logprobs_list.clear() + if logprobs is None: + continue + prompt_logprobs_dict[req_id] = logprobs return prompt_logprobs_dict @@ -184,10 +205,12 @@ def compute_prompt_logprobs_with_chunking( prompt_token_ids: torch.Tensor, prompt_hidden_states: torch.Tensor, logits_fn: Callable[[torch.Tensor], torch.Tensor], -) -> tuple[torch.Tensor, torch.Tensor]: + num_prompt_logprobs: int, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: # Since materializing the full prompt logits can take too much memory, # we compute it in chunks. CHUNK_SIZE = 1024 + token_ids = [] logprobs = [] ranks = [] prompt_token_ids = prompt_token_ids.to(torch.int64) @@ -195,14 +218,21 @@ def compute_prompt_logprobs_with_chunking( end_idx = start_idx + CHUNK_SIZE # NOTE(woosuk): logits_fn can be slow because it involves all-gather. prompt_logits = logits_fn(prompt_hidden_states[start_idx:end_idx]) + requested_num_prompt_logprobs = ( + prompt_logits.shape[-1] + if num_prompt_logprobs == -1 + else num_prompt_logprobs + ) prompt_logprobs = compute_topk_logprobs( prompt_logits, - 0, # num_logprobs + requested_num_prompt_logprobs, prompt_token_ids[start_idx:end_idx], ) + token_ids.append(prompt_logprobs.logprob_token_ids) logprobs.append(prompt_logprobs.logprobs) ranks.append(prompt_logprobs.selected_token_ranks) + token_ids = torch.cat(token_ids, dim=0) if len(token_ids) > 1 else token_ids[0] logprobs = torch.cat(logprobs, dim=0) if len(logprobs) > 1 else logprobs[0] ranks = torch.cat(ranks, dim=0) if len(ranks) > 1 else ranks[0] - return logprobs, ranks + return token_ids, logprobs, ranks From 6ee081d1d07bbcf7472b7bf76a34ae23310a36af Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Tue, 21 Apr 2026 16:51:30 +0100 Subject: [PATCH 003/153] Add new tp plan styles to the Transformers modelling backend (#40467) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- .../models/transformers/utils.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/models/transformers/utils.py b/vllm/model_executor/models/transformers/utils.py index e47f3bba5cf..04d6de28efd 100644 --- a/vllm/model_executor/models/transformers/utils.py +++ b/vllm/model_executor/models/transformers/utils.py @@ -94,7 +94,15 @@ def init_on_device_without_buffers(device: torch.device): setattr(torch, torch_function_name, old_torch_function) -Style = Literal["colwise", "colwise_rep", "rowwise", "rowwise_rep", "replicate"] +Style = Literal[ + "colwise", + "rowwise", + "replicate", + "colwise_gather_output", + "rowwise_split_input", + "colwise_rep", + "rowwise_rep", +] def replace_linear_class( @@ -120,10 +128,14 @@ def replace_linear_class( vllm_linear_cls, vllm_linear_kwargs = { "colwise": (ColumnParallelLinear, {}), - "colwise_rep": (ColumnParallelLinear, {"gather_output": True}), "rowwise": (RowParallelLinear, {}), - "rowwise_rep": (RowParallelLinear, {"input_is_parallel": False}), "replicate": (ReplicatedLinear, {}), + # Transformers v5 + "colwise_gather_output": (ColumnParallelLinear, {"gather_output": True}), + "rowwise_split_input": (RowParallelLinear, {"input_is_parallel": False}), + # Transformers v4 + "colwise_rep": (ColumnParallelLinear, {"gather_output": True}), + "rowwise_rep": (RowParallelLinear, {"input_is_parallel": False}), }.get(style, (ReplicatedLinear, {})) return vllm_linear_cls( From 67eb6083e38d1a65ae41cd00a573e6e95859751a Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Wed, 22 Apr 2026 00:08:06 +0800 Subject: [PATCH 004/153] Revert "[Misc] Move `pyav` and `soundfile` to common requirements" (#40276) Co-authored-by: Roger Wang --- requirements/common.txt | 2 -- requirements/test/rocm.txt | 5 +---- setup.py | 2 ++ 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/requirements/common.txt b/requirements/common.txt index 6a183e09acf..6e7fd90d023 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -32,9 +32,7 @@ pyzmq >= 25.0.0 msgspec gguf >= 0.17.0 mistral_common[image] >= 1.11.0 -av # required for audio in video IO opencv-python-headless >= 4.13.0 # required for video IO -soundfile # required for audio IO pyyaml six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12 setuptools>=77.0.3,<81.0.0; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 5abec4dce4c..33ea3a3f0db 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -76,9 +76,7 @@ attrs==26.1.0 audioread==3.0.1 # via librosa av==16.1.0 - # via - # -r requirements/test/../common.txt - # -r requirements/test/rocm.in + # via -r requirements/test/rocm.in azure-core==1.39.0 # via # azure-identity @@ -1333,7 +1331,6 @@ sortedcontainers==2.4.0 # via hypothesis soundfile==0.13.1 # via - # -r requirements/test/../common.txt # -r requirements/test/rocm.in # genai-perf # librosa diff --git a/setup.py b/setup.py index f6276616a19..bb2d6ac545d 100644 --- a/setup.py +++ b/setup.py @@ -1094,7 +1094,9 @@ setup( "instanttensor": ["instanttensor >= 0.1.5"], "runai": ["runai-model-streamer[s3,gcs,azure] >= 0.15.7"], "audio": [ + "av", "scipy", + "soundfile", "mistral_common[audio]", ], # Required for audio processing "video": [], # Kept for backwards compatibility From 9a6a66f3b837bd3565471dc09ce3e23831e0e3f7 Mon Sep 17 00:00:00 2001 From: Zijing Liu Date: Tue, 21 Apr 2026 09:30:32 -0700 Subject: [PATCH 005/153] [MRv2]fix: model accuracy regression caused by reusing the stale last_sampled_tokens and draft_tokens (#39833) Signed-off-by: Zijing Liu --- vllm/v1/worker/gpu/states.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index 24d22588610..cc371d32a91 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -102,6 +102,18 @@ class RequestState: self.num_computed_prefill_tokens[req_idx] = num_computed_tokens self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens) + if num_computed_tokens > 0 and num_computed_tokens <= prefill_len: + # For PD disagg or resumed requests: set last_sampled to the last + # computed token so the first decode step gets the right input_id. + # For fresh prefill requests (num_computed_tokens == 0) the tensor + # is not read by combine_sampled_and_draft_tokens so we skip the + # write. Use a slice assignment rather than scalar indexing so the + # write is dispatched through fill_ without a host/device sync. + self.last_sampled_tokens[req_idx : req_idx + 1] = all_token_ids[ + num_computed_tokens - 1 + ] + self.draft_tokens[req_idx].zero_() + def apply_staged_writes(self) -> None: self.prompt_len.copy_to_uva() self.prefill_len.copy_to_uva() From 9f39b380d070d2f60d28a152b9cbd05fea91a821 Mon Sep 17 00:00:00 2001 From: Rishi Puri Date: Tue, 21 Apr 2026 13:21:19 -0500 Subject: [PATCH 006/153] [Bugfix] Fix spec decode test failures on Blackwell (SM100+) (#39546) Signed-off-by: Stefano Castagnetta Signed-off-by: Rishi Puri Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Stefano Castagnetta Co-authored-by: Matthew Bonanni Co-authored-by: Benjamin Chislett --- .buildkite/test_areas/spec_decode.yaml | 34 ++++++++++++++++++++++++ vllm/v1/attention/backends/flashinfer.py | 23 ++++++++++++---- 2 files changed, 52 insertions(+), 5 deletions(-) diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 76cc887ed0a..05925da0da0 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -12,6 +12,17 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" +- label: Spec Decode Eagle Nightly B200 + timeout_in_minutes: 30 + device: b200 + optional: true + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" + - label: Spec Decode Speculators + MTP timeout_in_minutes: 30 device: h200_18gb @@ -23,6 +34,18 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" +- label: Spec Decode Speculators + MTP Nightly B200 + timeout_in_minutes: 30 + device: b200 + optional: true + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" + - label: Spec Decode Ngram + Suffix timeout_in_minutes: 30 device: h200_18gb @@ -43,6 +66,17 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" +- label: Spec Decode Draft Model Nightly B200 + timeout_in_minutes: 30 + device: b200 + optional: true + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - tests/v1/e2e/spec_decode/ + commands: + - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" + - label: DFlash Speculators Correctness timeout_in_minutes: 30 device: h100 diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index 78706e40c11..662ead1d1d0 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -920,9 +920,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): all_uses_trtllm = (num_prefills == 0 or prefill_use_trtllm) and ( num_decodes == 0 or decode_use_trtllm ) - is_only_trtllm_decode = num_prefills == 0 and ( - num_decodes > 0 and decode_use_trtllm - ) if not all_uses_trtllm: if self.has_sinks: @@ -968,7 +965,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): # Guard access to seq_lens_cpu, which may not always be needed # and can be expensive to retrieve in async mode. - needs_seq_lens_cpu = self.use_dcp or use_cascade or not is_only_trtllm_decode + # When all attention (both prefill and decode) uses TRTLLM, + # seq_lens_cpu is not needed since TRTLLM paths use GPU tensors + # (block_tables, seq_lens) directly. + needs_seq_lens_cpu = self.use_dcp or use_cascade or not all_uses_trtllm seq_lens_cpu = common_attn_metadata.seq_lens_cpu if needs_seq_lens_cpu else None seq_lens_np = seq_lens_cpu.numpy() if seq_lens_cpu is not None else None num_blocks_np = ( @@ -1006,7 +1006,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): num_blocks_np -= num_common_kv_blocks # Compute paged_kv_indices if necessary - needs_paged_kv_indices = use_cascade or not is_only_trtllm_decode + # paged_kv_indices is only needed for FlashInfer native paths; + # TRTLLM paths use block_tables directly on GPU. + needs_paged_kv_indices = use_cascade or not all_uses_trtllm if needs_paged_kv_indices: assert num_blocks_np is not None assert seq_lens_np is not None @@ -1083,9 +1085,20 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): qo_indptr_prefill_gpu = ( qo_indptr[prefill_start:] - qo_indptr[prefill_start] ) + # Compute cum_seq_lens_kv on GPU to avoid CPU sync. + # This is the cumulative sum of the number of KV cache + # blocks per prefill request. + prefill_seq_lens = seq_lens[prefill_start:] + num_blocks_per_req = (prefill_seq_lens + page_size - 1) // page_size paged_kv_indptr_prefill_gpu = self.paged_kv_indptr.gpu[ prefill_start : num_reqs + 1 ] + paged_kv_indptr_prefill_gpu[0] = 0 + torch.cumsum( + num_blocks_per_req, + dim=0, + out=paged_kv_indptr_prefill_gpu[1:], + ) # Compute max_q_len for prefill requests query_lens_prefill_cpu = ( qo_indptr_prefill_cpu[1:] - qo_indptr_prefill_cpu[:-1] From 5544f8c18b9e9ae20e41e019d23d58260940f225 Mon Sep 17 00:00:00 2001 From: Fergus Date: Tue, 21 Apr 2026 19:31:27 +0100 Subject: [PATCH 007/153] [Performance] Add is_reasoning_end_streaming() override to GptOssReasoningParser (#35745) Signed-off-by: Fergus Signed-off-by: fergus barratt Co-authored-by: Chauncey --- .../reasoning/test_gptoss_reasoning_parser.py | 69 +++++++++++++++++++ vllm/reasoning/gptoss_reasoning_parser.py | 21 +++++- 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/tests/reasoning/test_gptoss_reasoning_parser.py b/tests/reasoning/test_gptoss_reasoning_parser.py index 3b1327acb68..a6f815b6ae5 100644 --- a/tests/reasoning/test_gptoss_reasoning_parser.py +++ b/tests/reasoning/test_gptoss_reasoning_parser.py @@ -280,3 +280,72 @@ class TestGptOssStructuralTags: assert tag["content"]["type"] == "any_text" assert tag["end"] == "<|end|>" assert tag["begin"].startswith("<|channel|>") + + +@pytest.mark.parametrize( + "output, is_reasoning_end", + [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], +) +def test_gptoss_is_reasoning_end_streaming( + output, + is_reasoning_end, + gpt_oss_tokenizer, +): + """Streaming override must agree with is_reasoning_end for all cases.""" + tokens = gpt_oss_tokenizer.tokenize(output) + parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) + output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens) + delta_ids = output_ids[-1:] if output_ids else [] + actual = parser.is_reasoning_end_streaming(output_ids, delta_ids) + assert is_reasoning_end == actual + + +@pytest.mark.parametrize( + "output, is_reasoning_end", + [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], +) +def test_gptoss_is_reasoning_end_streaming_long_prefix( + output, + is_reasoning_end, + gpt_oss_tokenizer, +): + """Windowing must produce correct results even with a long prefix.""" + tokens = gpt_oss_tokenizer.tokenize(output) + parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) + output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens) + # Prepend 10k dummy reasoning tokens to simulate a long generation + long_prefix = [1] * 10_000 + padded_ids = long_prefix + list(output_ids) + delta_ids = output_ids[-1:] if output_ids else [] + actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids) + assert is_reasoning_end == actual + + +@pytest.mark.parametrize( + "output, is_reasoning_end", + [(t["output"], t["is_reasoning_end"]) for t in TEST_CASES], +) +def test_gptoss_is_reasoning_end_streaming_large_delta( + output, + is_reasoning_end, + gpt_oss_tokenizer, +): + """Simulate speculative decoding where the entire test sequence arrives + as a single large delta appended after a long prefix. The window must + expand to cover delta_ids so the end pattern is never missed.""" + tokens = gpt_oss_tokenizer.tokenize(output) + parser: ReasoningParser = GptOssReasoningParser(gpt_oss_tokenizer) + output_ids = gpt_oss_tokenizer.convert_tokens_to_ids(tokens) + long_prefix = [1] * 10_000 + padded_ids = long_prefix + list(output_ids) + # delta_ids = the entire test sequence (as if accepted in one spec step) + delta_ids = list(output_ids) + actual = parser.is_reasoning_end_streaming(padded_ids, delta_ids) + assert is_reasoning_end == actual + + +def test_gptoss_is_reasoning_end_streaming_signature(gpt_oss_tokenizer): + """Verify the method is callable with the expected signature.""" + parser = GptOssReasoningParser(gpt_oss_tokenizer) + result = parser.is_reasoning_end_streaming([], []) + assert result is False diff --git a/vllm/reasoning/gptoss_reasoning_parser.py b/vllm/reasoning/gptoss_reasoning_parser.py index 89299d4b12b..1ba933cca31 100644 --- a/vllm/reasoning/gptoss_reasoning_parser.py +++ b/vllm/reasoning/gptoss_reasoning_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from typing import TYPE_CHECKING from transformers import PreTrainedTokenizerBase @@ -112,6 +112,25 @@ class GptOssReasoningParser(ReasoningParser): return True return False + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + # The pattern window covers the end-of-reasoning marker itself. + # We add len(delta_ids) so that under speculative decoding (where + # a single step can accept many tokens) the entire accepted chunk + # is always inside the scan region. + delta_ids = tuple(delta_ids) + pattern_len = ( + len(self.reasoning_end_token_ids_prefix) + + self.reasoning_max_num_between_tokens + + len(self.reasoning_end_token_ids_suffix) + ) + window = pattern_len + len(delta_ids) + n = len(input_ids) + if n <= window: + return self.is_reasoning_end(input_ids) + return self.is_reasoning_end(input_ids[n - window :]) + def extract_content_ids(self, input_ids: list[int]) -> list[int]: _, content, _ = parse_chat_output(input_ids) if content is None: From 6fbec8ed473f418a9c20d5b6a4d56486544139da Mon Sep 17 00:00:00 2001 From: Jakub Zakrzewski Date: Tue, 21 Apr 2026 21:06:09 +0200 Subject: [PATCH 008/153] [Bugfix][Kernel] nvfp4 cutlass MoE: fix nvfp4 experts quant out-of-bounds read for expert counts not divisible by 4 or 16 (#40351) Signed-off-by: Jakub Zakrzewski --- .../quantization/fp4/nvfp4_experts_quant.cu | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/csrc/libtorch_stable/quantization/fp4/nvfp4_experts_quant.cu b/csrc/libtorch_stable/quantization/fp4/nvfp4_experts_quant.cu index f90bd543ab9..744ae4f7311 100644 --- a/csrc/libtorch_stable/quantization/fp4/nvfp4_experts_quant.cu +++ b/csrc/libtorch_stable/quantization/fp4/nvfp4_experts_quant.cu @@ -277,7 +277,9 @@ void quant_impl(void* output, void* output_scale, void* input, (totalWorkSize + block.x * grid.x - 1) / (block.x * grid.x); if (blockRepeat > 1) { size_t shared_mem_size = (n_experts + 1) * sizeof(uint32_t); - if (n_experts >= 4) { + // The shared-memory vectorized offset load only handles full 4-expert + // chunks. Use the scalar specialization for the remainder cases. + if (n_experts >= 4 && n_experts % 4 == 0) { cvt_fp16_to_fp4 <<>>( m_topk, k, reinterpret_cast(input), @@ -299,7 +301,9 @@ void quant_impl(void* output, void* output_scale, void* input, n_experts); } } else { - if (n_experts >= 16) { + // The low-latency vectorized expert lookup only handles full 16-expert + // chunks. Fall back to the scalar lookup path for the remainder cases. + if (n_experts >= 16 && n_experts % 16 == 0) { cvt_fp16_to_fp4 <<>>( m_topk, k, reinterpret_cast(input), From 16688b26a6fbabe0100c440462874ad4e4c78b16 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 21 Apr 2026 15:51:03 -0400 Subject: [PATCH 009/153] [Perf] Optimize batch invariant with fused rms norm, 2.1% E2E latency improvement (#40413) Signed-off-by: yewentao256 --- .../test_rms_norm_batch_invariant.py | 89 ++++++++++++++++++- vllm/_custom_ops.py | 1 + vllm/model_executor/layers/layernorm.py | 4 - 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/tests/v1/determinism/test_rms_norm_batch_invariant.py b/tests/v1/determinism/test_rms_norm_batch_invariant.py index 7d3b8437a93..5c036c1b380 100644 --- a/tests/v1/determinism/test_rms_norm_batch_invariant.py +++ b/tests/v1/determinism/test_rms_norm_batch_invariant.py @@ -12,7 +12,7 @@ import torch from utils import skip_unsupported from vllm.model_executor.layers.batch_invariant import rms_norm as triton_rms_norm -from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.layernorm import RMSNorm, fused_add_rms_norm from vllm.platforms import current_platform DEVICE_TYPE = current_platform.device_type @@ -71,6 +71,93 @@ def test_rms_norm_batch_invariant_vs_standard( ) +@skip_unsupported +@pytest.mark.parametrize("hidden_size", [512, 4096]) +@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16]) +@pytest.mark.parametrize("eps", [1e-6]) +def test_fused_add_rms_norm_batch_invariant_residual_path( + hidden_size: int, + dtype: torch.dtype, + eps: float, +): + """ + Test the batch-invariant fused residual-add + RMSNorm helper directly. + """ + device = torch.device(DEVICE_TYPE) + + torch.manual_seed(42) + x_single = torch.randn(1, hidden_size, dtype=dtype, device=device) + residual_single = torch.randn(1, hidden_size, dtype=dtype, device=device) + weight = torch.randn(hidden_size, dtype=dtype, device=device) + + x_batch = torch.cat( + [ + x_single, + torch.randn(3, hidden_size, dtype=dtype, device=device), + ], + dim=0, + ) + residual_batch = torch.cat( + [ + residual_single, + torch.randn(3, hidden_size, dtype=dtype, device=device), + ], + dim=0, + ) + + out_single, residual_out_single = fused_add_rms_norm( + x_single.clone(), + residual_single.clone(), + weight, + eps, + ) + out_batch, residual_out_batch = fused_add_rms_norm( + x_batch.clone(), + residual_batch.clone(), + weight, + eps, + ) + + merged_single = x_single + residual_single + ref_out = triton_rms_norm(merged_single, weight, eps=eps) + + torch.testing.assert_close( + residual_out_single, + merged_single, + rtol=0.0, + atol=0.0, + msg="Residual output should equal x + residual exactly", + ) + torch.testing.assert_close( + residual_out_batch[:1], + merged_single, + rtol=0.0, + atol=0.0, + msg="Residual output should be batch invariant", + ) + torch.testing.assert_close( + out_single, + out_batch[:1], + rtol=0.0, + atol=0.0, + msg="Fused add RMSNorm output should be batch invariant", + ) + + if dtype == torch.bfloat16: + rtol, atol = 1e-1, 1e-1 + else: + rtol, atol = 1e-2, 1e-2 + + torch.testing.assert_close( + out_single, + ref_out, + rtol=rtol, + atol=atol, + msg="Fused add RMSNorm output should stay numerically close to the " + "batch-invariant RMSNorm reference", + ) + + @skip_unsupported @pytest.mark.parametrize("batch_size", [1, 16, 128]) @pytest.mark.parametrize("seq_len", [1, 32, 512]) diff --git a/vllm/_custom_ops.py b/vllm/_custom_ops.py index a7b6a7059b0..d9be6a4c433 100644 --- a/vllm/_custom_ops.py +++ b/vllm/_custom_ops.py @@ -420,6 +420,7 @@ def rms_norm( def fused_add_rms_norm( input: torch.Tensor, residual: torch.Tensor, weight: torch.Tensor, epsilon: float ) -> None: + # Note: this func is batch invariant torch.ops._C.fused_add_rms_norm(input, residual, weight, epsilon) diff --git a/vllm/model_executor/layers/layernorm.py b/vllm/model_executor/layers/layernorm.py index e4d2d2be090..ac2423ce0e0 100644 --- a/vllm/model_executor/layers/layernorm.py +++ b/vllm/model_executor/layers/layernorm.py @@ -61,10 +61,6 @@ def fused_add_rms_norm( ) -> tuple[torch.Tensor, torch.Tensor]: from vllm import _custom_ops as ops - if envs.VLLM_BATCH_INVARIANT: - return rms_norm_batch_invariant( - x + residual, weight, variance_epsilon - ), x + residual ops.fused_add_rms_norm( x, residual, From 1842447c09224d9161857a03e1cfac33b7701c50 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Tue, 21 Apr 2026 17:59:20 -0400 Subject: [PATCH 010/153] [Refactor] Remove unused param (#39750) Signed-off-by: yewentao256 --- vllm/model_executor/models/qwen2_5_omni_thinker.py | 4 ---- vllm/model_executor/models/qwen3_asr.py | 2 -- vllm/model_executor/models/qwen3_omni_moe_thinker.py | 2 -- vllm/v1/engine/async_llm.py | 2 -- vllm/v1/engine/llm_engine.py | 1 - 5 files changed, 11 deletions(-) diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 7ce7dc8319c..14f4c424bb3 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -952,8 +952,6 @@ class Qwen2_5OmniConditionalGenerationMixin: def _process_audio_input( self, audio_input: Qwen2_5OmniAudioFeatureInputs, - audio_hashes: list[str] | None = None, - cached_audio_features: torch.Tensor | None = None, ) -> torch.Tensor: input_features = audio_input["input_features"] audio_feature_lengths = audio_input["audio_feature_lengths"] @@ -990,8 +988,6 @@ class Qwen2_5OmniConditionalGenerationMixin: def _process_video_input( self, video_input: Qwen2_5_VLVideoInputs, - video_hashes: list[str] = None, - cached_video_embeds: torch.Tensor = None, ) -> torch.Tensor: if video_input["type"] == "video_embeds": return video_input["video_embeds"].type(self.visual.dtype) diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 1d8e1bbaa4c..37903462dd4 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -363,8 +363,6 @@ class Qwen3ASRForConditionalGeneration( def _process_audio_input( self, audio_input: Qwen2_5OmniAudioFeatureInputs, - audio_hashes: list[str] | None = None, - cached_audio_features: torch.Tensor | None = None, ) -> torch.Tensor: input_features = audio_input["input_features"] audio_feature_lengths = audio_input["audio_feature_lengths"] diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 8cee51a1269..b4842e06388 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -1646,8 +1646,6 @@ class Qwen3OmniMoeConditionalGenerationMixin(Qwen2_5OmniConditionalGenerationMix def _process_audio_input( self, audio_input: Qwen2_5OmniAudioFeatureInputs, - audio_hashes: list[str] | None = None, - cached_audio_features: torch.Tensor | None = None, ) -> tuple[torch.Tensor, ...]: input_features = audio_input["input_features"] audio_feature_lengths = audio_input["audio_feature_lengths"] diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 1c87d9ec094..45ae416529e 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -77,7 +77,6 @@ class AsyncLLM(EngineClient): log_stats: bool, usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, - use_cached_outputs: bool = False, log_requests: bool = True, start_engine_loop: bool = True, stat_loggers: list[StatLoggerFactory] | None = None, @@ -95,7 +94,6 @@ class AsyncLLM(EngineClient): log_stats: Whether to log stats. usage_context: Usage context of the LLM. mm_registry: Multi-modal registry. - use_cached_outputs: Whether to use cached outputs. log_requests: Whether to log requests. start_engine_loop: Whether to start the engine loop. stat_loggers: customized stat loggers for the engine. diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index d0545651b96..62c7d0e2a81 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -56,7 +56,6 @@ class LLMEngine: usage_context: UsageContext = UsageContext.ENGINE_CONTEXT, stat_loggers: list[StatLoggerFactory] | None = None, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, - use_cached_outputs: bool = False, multiprocess_mode: bool = False, ) -> None: self.vllm_config = vllm_config From 5e584ce9ecb3cce63f1caab86177aef5c831690f Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:12:12 -0400 Subject: [PATCH 011/153] [MoE Refactor] Remove SharedFusedMoE class (#35782) Signed-off-by: Bill Nell --- tests/kernels/moe/test_moe_layer.py | 10 +++----- .../test_shared_fused_moe_routed_transform.py | 20 +++++++-------- .../base_device_communicator.py | 13 +++------- .../distributed/elastic_ep/elastic_execute.py | 6 ++--- vllm/lora/layers/fused_moe.py | 4 +-- .../layers/fused_moe/__init__.py | 2 -- .../layers/fused_moe/shared_fused_moe.py | 25 ------------------- vllm/model_executor/models/AXK1.py | 8 +++--- vllm/model_executor/models/afmoe.py | 10 ++++---- vllm/model_executor/models/aria.py | 4 +-- vllm/model_executor/models/bailing_moe.py | 6 ++--- .../models/bailing_moe_linear.py | 6 ++--- vllm/model_executor/models/deepseek_mtp.py | 4 +-- vllm/model_executor/models/deepseek_v2.py | 8 +++--- vllm/model_executor/models/dots1.py | 6 ++--- vllm/model_executor/models/ernie45_moe.py | 8 +++--- vllm/model_executor/models/ernie45_vl_moe.py | 8 +++--- vllm/model_executor/models/exaone_moe.py | 3 +-- vllm/model_executor/models/glm4_moe.py | 6 ++--- vllm/model_executor/models/glm4_moe_lite.py | 8 +++--- .../models/glm4_moe_lite_mtp.py | 4 +-- vllm/model_executor/models/hunyuan_v1.py | 6 ++--- vllm/model_executor/models/kimi_linear.py | 6 ++--- vllm/model_executor/models/llama4.py | 10 ++++---- vllm/model_executor/models/nemotron_h.py | 6 ++--- vllm/model_executor/models/openpangu.py | 6 ++--- vllm/model_executor/models/param2moe.py | 8 +++--- vllm/model_executor/models/qwen2_moe.py | 6 ++--- vllm/model_executor/models/qwen3_moe.py | 6 ++--- vllm/model_executor/models/qwen3_next.py | 6 ++--- vllm/model_executor/models/sarvam.py | 8 +++--- vllm/model_executor/models/step3p5.py | 3 +-- vllm/utils/__init__.py | 13 ++++++++++ 33 files changed, 112 insertions(+), 141 deletions(-) delete mode 100644 vllm/model_executor/layers/fused_moe/shared_fused_moe.py diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index 85619a91005..fca4096b086 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -37,7 +37,7 @@ from vllm.distributed.parallel_state import ( get_eplb_group, ) from vllm.forward_context import set_forward_context -from vllm.model_executor.layers.fused_moe import FusedMoE, SharedFusedMoE, fused_experts +from vllm.model_executor.layers.fused_moe import FusedMoE, fused_experts from vllm.model_executor.layers.fused_moe.activation import MoEActivation from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.router.router_factory import ( @@ -858,11 +858,7 @@ def make_fused_moe_layer( quant_config, qw = make_quant_config(quantization, w1, w2, global_num_experts) kwargs = dict() - if shared_experts is None: - builder = FusedMoE - else: - builder = SharedFusedMoE - kwargs["shared_experts"] = shared_experts + kwargs["shared_experts"] = shared_experts # Add gate and routed_input_transform if provided if gate is not None: @@ -872,7 +868,7 @@ def make_fused_moe_layer( kwargs["routed_input_transform"] = routed_input_transform kwargs["routed_output_transform"] = routed_output_transform - layer = builder( + layer = FusedMoE( num_experts=global_num_experts, top_k=top_k, hidden_size=hidden_size, diff --git a/tests/kernels/moe/test_shared_fused_moe_routed_transform.py b/tests/kernels/moe/test_shared_fused_moe_routed_transform.py index 464754c9f1b..4515021a4e9 100644 --- a/tests/kernels/moe/test_shared_fused_moe_routed_transform.py +++ b/tests/kernels/moe/test_shared_fused_moe_routed_transform.py @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Tests for SharedFusedMoE with routed_input_transform. +Tests for FusedMoE with routed_input_transform. -Verifies that applying routed_input_transform inside SharedFusedMoE +Verifies that applying routed_input_transform inside FusedMoE produces the same results as applying the transform manually outside. """ @@ -13,7 +13,7 @@ import torch.nn as nn from vllm.config import VllmConfig, set_current_vllm_config from vllm.forward_context import set_forward_context -from vllm.model_executor.layers.fused_moe.shared_fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.platforms import current_platform from vllm.utils.torch_utils import is_torch_equal_or_newer, set_random_seed @@ -133,9 +133,9 @@ def test_routed_input_transform_inside_vs_outside( workspace_init, monkeypatch, ): - """Compare SharedFusedMoE with transform inside vs manually applying outside. - Method A (inside): SharedFusedMoE with routed_input_transform - Method B (outside): Manually transform, then SharedFusedMoE without transform + """Compare FusedMoE with transform inside vs manually applying outside. + Method A (inside): FusedMoE with routed_input_transform + Method B (outside): Manually transform, then FusedMoE without transform """ if current_platform.is_rocm(): monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1" if use_rocm_aiter else "0") @@ -157,8 +157,8 @@ def test_routed_input_transform_inside_vs_outside( routed_transform = SimpleLinear(hidden_size, latent_size, dtype) with set_current_vllm_config(vllm_config): - # Method A: SharedFusedMoE WITH routed_input_transform - moe_with_transform = SharedFusedMoE( + # Method A: FusedMoE WITH routed_input_transform + moe_with_transform = FusedMoE( shared_experts=shared_experts, routed_input_transform=routed_transform, num_experts=num_experts, @@ -173,9 +173,9 @@ def test_routed_input_transform_inside_vs_outside( prefix="moe_with_transform", ) - # Method B: SharedFusedMoE WITHOUT routed_input_transform + # Method B: FusedMoE WITHOUT routed_input_transform # Note: shared_experts=None because when transform is done outside, - moe_without_transform = SharedFusedMoE( + moe_without_transform = FusedMoE( shared_experts=None, routed_input_transform=None, num_experts=num_experts, diff --git a/vllm/distributed/device_communicators/base_device_communicator.py b/vllm/distributed/device_communicators/base_device_communicator.py index 2125f7381fe..0b4b81f93bb 100644 --- a/vllm/distributed/device_communicators/base_device_communicator.py +++ b/vllm/distributed/device_communicators/base_device_communicator.py @@ -7,6 +7,8 @@ import torch import torch.distributed as dist from torch.distributed import ProcessGroup +from vllm.utils import is_moe_layer + class Cache: def __init__(self): @@ -317,16 +319,7 @@ class DeviceCommunicatorBase: if not self.is_ep_communicator: return - moe_modules = [ - module - for module in model.modules() - # TODO(bnell): Should use isinstance but can't. Maybe search for - # presence of quant_method.maybe_init_modular_kernel? - if ( - module.__class__.__name__ == "FusedMoE" - or module.__class__.__name__ == "SharedFusedMoE" - ) - ] + moe_modules = [module for module in model.modules() if is_moe_layer(module)] for module in moe_modules: module.maybe_init_modular_kernel() diff --git a/vllm/distributed/elastic_ep/elastic_execute.py b/vllm/distributed/elastic_ep/elastic_execute.py index a316a54bd51..24979b62af6 100644 --- a/vllm/distributed/elastic_ep/elastic_execute.py +++ b/vllm/distributed/elastic_ep/elastic_execute.py @@ -38,6 +38,7 @@ from vllm.distributed.parallel_state import ( from vllm.distributed.stateless_coordinator import StatelessGroupCoordinator from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.layer import FusedMoEParallelConfig +from vllm.utils import is_moe_layer from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper from vllm.v1.worker.workspace import lock_workspace, unlock_workspace @@ -319,10 +320,7 @@ class ElasticEPScalingExecutor: moe_modules = [ module for module in self.worker.model_runner.model.modules() - if ( - module.__class__.__name__ == "FusedMoE" - or module.__class__.__name__ == "SharedFusedMoE" - ) + if is_moe_layer(module) ] num_local_experts = moe_modules[0].moe_config.num_local_experts assert all( diff --git a/vllm/lora/layers/fused_moe.py b/vllm/lora/layers/fused_moe.py index d6eec675c6d..b07b471c4b8 100644 --- a/vllm/lora/layers/fused_moe.py +++ b/vllm/lora/layers/fused_moe.py @@ -610,7 +610,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA): ) -> bool: """Returns True if the layer can be replaced by this LoRA layer.""" - # source_layer is FusedMoE or SharedFusedMoE + # source_layer is FusedMoE return isinstance(source_layer, FusedMoE) and len(packed_modules_list) == 2 @@ -772,5 +772,5 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA): model_config: PretrainedConfig | None = None, ) -> bool: """Returns True if the layer can be replaced by this LoRA layer.""" - # source_layer is FusedMoE or SharedFusedMoE + # source_layer is FusedMoE return isinstance(source_layer, FusedMoE) and len(packed_modules_list) == 1 diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 926f0d1d015..1b2ce61f7c8 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -29,7 +29,6 @@ from vllm.model_executor.layers.fused_moe.router.fused_moe_router import ( FusedMoERouter, ) from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear -from vllm.model_executor.layers.fused_moe.shared_fused_moe import SharedFusedMoE from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import ( UnquantizedFusedMoEMethod, ) @@ -64,7 +63,6 @@ __all__ = [ "FusedMoEPrepareAndFinalizeModular", "GateLinear", "RoutingMethodType", - "SharedFusedMoE", "activation_without_mul", "apply_moe_activation", "override_config", diff --git a/vllm/model_executor/layers/fused_moe/shared_fused_moe.py b/vllm/model_executor/layers/fused_moe/shared_fused_moe.py deleted file mode 100644 index 9cfcb1baa9b..00000000000 --- a/vllm/model_executor/layers/fused_moe/shared_fused_moe.py +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.model_executor.layers.fused_moe.layer import FusedMoE - - -# TODO(bnell): Remove this entirely -class SharedFusedMoE(FusedMoE): - """ - A FusedMoE operation that also computes the results of shared experts. - If an all2all communicator is being used the shared expert computation - can be interleaved with the fused all2all dispatch communication step. - """ - - def forward( - self, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - ) -> torch.Tensor: - return super().forward( - hidden_states=hidden_states, - router_logits=router_logits, - ) diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index d42fbed42ae..c33d5b97372 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -42,7 +42,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -163,7 +163,7 @@ class AXK1MoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, @@ -916,7 +916,7 @@ class AXK1ForCausalLM( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -950,7 +950,7 @@ class AXK1ForCausalLM( # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/afmoe.py b/vllm/model_executor/models/afmoe.py index 5bad52a0c49..e34a418c981 100644 --- a/vllm/model_executor/models/afmoe.py +++ b/vllm/model_executor/models/afmoe.py @@ -18,7 +18,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe.shared_fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -124,8 +124,8 @@ class AfmoeMoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - # Routed experts using SharedFusedMoE - self.experts = SharedFusedMoE( + # Routed experts using FusedMoE + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.num_experts, top_k=config.num_experts_per_tok, @@ -479,7 +479,7 @@ class AfmoeModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -637,7 +637,7 @@ class AfmoeForCausalLM(nn.Module, SupportsPP, SupportsEagle3, SupportsLoRA): self.num_moe_layers = config.num_hidden_layers - config.num_dense_layers self.num_expert_groups = config.n_group - self.moe_layers: list[SharedFusedMoE] = [] + self.moe_layers: list[FusedMoE] = [] example_moe = None for layer in self.model.layers: if isinstance(layer, PPMissingLayer): diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index 7a079c56540..9696dec6d87 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -14,7 +14,7 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.distributed import get_tensor_model_parallel_rank from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -214,7 +214,7 @@ class AriaProjector(nn.Module): return out -class AriaFusedMoE(SharedFusedMoE): +class AriaFusedMoE(FusedMoE): def weight_loader( self, param: nn.Parameter, loaded_weight: torch.Tensor, shard_id: str ) -> None: diff --git a/vllm/model_executor/models/bailing_moe.py b/vllm/model_executor/models/bailing_moe.py index 510d605f804..ef4f66614a3 100644 --- a/vllm/model_executor/models/bailing_moe.py +++ b/vllm/model_executor/models/bailing_moe.py @@ -41,7 +41,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -285,7 +285,7 @@ class BailingMoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, @@ -461,7 +461,7 @@ class BailingMoeModel(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/bailing_moe_linear.py b/vllm/model_executor/models/bailing_moe_linear.py index a63ad83f45b..df36659b10c 100644 --- a/vllm/model_executor/models/bailing_moe_linear.py +++ b/vllm/model_executor/models/bailing_moe_linear.py @@ -21,7 +21,7 @@ from vllm.model_executor.layers.fla.ops.layernorm_guard import ( RMSNormGated, layernorm_fn, ) -from vllm.model_executor.layers.fused_moe import FusedMoE, SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -351,8 +351,8 @@ class BailingMoeV25(nn.Module): else: self.shared_experts = None - # Routed experts using SharedFusedMoE - self.experts = SharedFusedMoE( + # Routed experts using FusedMoE + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index a66ec7aa3e6..898e4333409 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -11,7 +11,7 @@ from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -252,7 +252,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): ] stacked_params_mapping.extend(indexer_fused_mapping) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 1b01caded94..53bcf87c6cc 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -48,9 +48,9 @@ from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.fused_moe import ( + FusedMoE, GateLinear, RoutingMethodType, - SharedFusedMoE, ) from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm from vllm.model_executor.layers.linear import ( @@ -311,7 +311,7 @@ class DeepseekV2MoE(nn.Module): prefix=f"{prefix}.shared_experts", ) - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, gate=self.gate, num_experts=config.n_routed_experts, @@ -1432,7 +1432,7 @@ class DeepseekV2ForCausalLM( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -1474,7 +1474,7 @@ class DeepseekV2ForCausalLM( # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/dots1.py b/vllm/model_executor/models/dots1.py index c176b736568..181bd598e8e 100644 --- a/vllm/model_executor/models/dots1.py +++ b/vllm/model_executor/models/dots1.py @@ -40,7 +40,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -155,7 +155,7 @@ class Dots1MoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, @@ -413,7 +413,7 @@ class Dots1Model(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/ernie45_moe.py b/vllm/model_executor/models/ernie45_moe.py index c92e230bcd2..58dd61e9d92 100644 --- a/vllm/model_executor/models/ernie45_moe.py +++ b/vllm/model_executor/models/ernie45_moe.py @@ -42,7 +42,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -188,7 +188,7 @@ class Ernie4_5_MoeMoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.moe_num_experts, top_k=config.moe_k, @@ -485,7 +485,7 @@ class Ernie4_5_MoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -667,7 +667,7 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe self.num_moe_layers = len(moe_layers_indices) self.num_expert_groups = 1 - self.moe_layers: list[SharedFusedMoE] = [] + self.moe_layers: list[FusedMoE] = [] example_moe = None for layer in self.model.layers: if isinstance(layer, PPMissingLayer): diff --git a/vllm/model_executor/models/ernie45_vl_moe.py b/vllm/model_executor/models/ernie45_vl_moe.py index e4b7ac6fb00..b4e7af9304b 100644 --- a/vllm/model_executor/models/ernie45_vl_moe.py +++ b/vllm/model_executor/models/ernie45_vl_moe.py @@ -36,7 +36,7 @@ from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -257,7 +257,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): prefix=f"{prefix}.text_experts_gate", ) - self.text_experts = SharedFusedMoE( + self.text_experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.moe_num_experts[0], top_k=config.moe_k, @@ -294,7 +294,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): prefix=f"{prefix}.vision_experts_gate", ) - self.vision_experts = SharedFusedMoE( + self.vision_experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.moe_num_experts[1], top_k=config.moe_k, @@ -649,7 +649,7 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index a46cadf007e..dd91a189628 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -31,7 +31,6 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.fused_moe import FusedMoE -from vllm.model_executor.layers.fused_moe.shared_fused_moe import SharedFusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -130,7 +129,7 @@ class ExaoneMoe(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, gate=self.gate, num_experts=self.n_routed_experts, diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index 671e868da0a..680e7460992 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -42,7 +42,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -178,7 +178,7 @@ class Glm4MoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, @@ -466,7 +466,7 @@ class Glm4MoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/glm4_moe_lite.py b/vllm/model_executor/models/glm4_moe_lite.py index 6d96f748e3e..5dc33ec18bf 100644 --- a/vllm/model_executor/models/glm4_moe_lite.py +++ b/vllm/model_executor/models/glm4_moe_lite.py @@ -41,7 +41,7 @@ from vllm.distributed import ( get_pp_group, ) from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -308,7 +308,7 @@ class Glm4MoeLiteModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -334,7 +334,7 @@ class Glm4MoeLiteModel(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -616,7 +616,7 @@ class Glm4MoeLiteForCausalLM( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/glm4_moe_lite_mtp.py b/vllm/model_executor/models/glm4_moe_lite_mtp.py index efa96c40d04..e00476abac6 100644 --- a/vllm/model_executor/models/glm4_moe_lite_mtp.py +++ b/vllm/model_executor/models/glm4_moe_lite_mtp.py @@ -32,7 +32,7 @@ from transformers import PretrainedConfig from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig -from vllm.model_executor.layers.fused_moe import FusedMoE, SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -260,7 +260,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), ] - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/hunyuan_v1.py b/vllm/model_executor/models/hunyuan_v1.py index 35d30006a66..9d3ebe4ed9c 100644 --- a/vllm/model_executor/models/hunyuan_v1.py +++ b/vllm/model_executor/models/hunyuan_v1.py @@ -42,7 +42,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -438,7 +438,7 @@ class HunYuanSparseMoeBlock(nn.Module): else: self.shared_mlp = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_mlp, num_experts=self.n_routed_experts, top_k=top_k, @@ -712,7 +712,7 @@ class HunYuanModel(nn.Module, EagleModelMixin): if _is_moe(self.config): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/kimi_linear.py b/vllm/model_executor/models/kimi_linear.py index e586a3ac346..21940fb2e1f 100644 --- a/vllm/model_executor/models/kimi_linear.py +++ b/vllm/model_executor/models/kimi_linear.py @@ -14,7 +14,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.kda import KimiDeltaAttention from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -144,7 +144,7 @@ class KimiMoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=num_experts, top_k=config.num_experts_per_token, @@ -476,7 +476,7 @@ class KimiLinearModel(nn.Module): if self.config.is_moe: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index a1c0ac89605..c9495a743b7 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -36,7 +36,7 @@ from vllm.model_executor.layers.attention import ( Attention, ChunkedLocalAttention, ) -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -127,7 +127,7 @@ class Llama4MoE(nn.Module): self.n_physical_experts = self.n_local_experts + self.n_redundant_experts self.n_local_physical_experts = self.n_physical_experts // self.ep_size - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_expert, num_experts=config.num_local_experts, top_k=config.num_experts_per_tok, @@ -414,7 +414,7 @@ class Llama4Model(LlamaModel): params_dict: The dictionary of module parameters. loaded_params: The set of already loaded parameters. expert_params_mapping: The mapping of expert parameters. Must be - generated by SharedFusedMoE.make_expert_params_mapping(). + generated by FusedMoE.make_expert_params_mapping(). fused: Whether the expert weights are fused into a single weight tensor or are separate weight tensors for each expert. When fused is True, loaded_weight should have shape of: @@ -554,7 +554,7 @@ class Llama4Model(LlamaModel): fused_experts_params = False # Expert parameter mapping for the case where the expert weights are # not fused into a single weight tensor. - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -564,7 +564,7 @@ class Llama4Model(LlamaModel): ) # Expert parameter mapping for the case where the expert weights are # fused into a single weight tensor. - expert_params_mapping_fused = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping_fused = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_up_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index fa068639648..9b8ed68560c 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -34,8 +34,8 @@ from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.layers.activation import ReLUSquaredActivation from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import ( + FusedMoE, GateLinear, - SharedFusedMoE, activation_without_mul, ) from vllm.model_executor.layers.layernorm import RMSNorm @@ -210,7 +210,7 @@ class NemotronHMoE(nn.Module): self.fc1_latent_proj = None self.fc2_latent_proj = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, @@ -652,7 +652,7 @@ class NemotronHModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: if self.has_moe: # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_params_mapping = FusedMoE.make_expert_params_mapping( # - FusedMoe.w1 (aka gate_proj) should be up_proj since that's # what the activation is applied to # - FusedMoe.w3 (aka up_proj) should be ignored since we're diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index 7de84da5193..96b837e42a8 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -44,7 +44,7 @@ from vllm.model_executor.layers.attention import ( Attention, StaticSinkAttention, ) -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -200,7 +200,7 @@ class OpenPanguMoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=config.n_routed_experts, top_k=config.num_experts_per_tok, @@ -1149,7 +1149,7 @@ class OpenPanguModel(nn.Module): ] has_experts = hasattr(self.config, "n_routed_experts") if has_experts: - expert_merge_mapping = SharedFusedMoE.make_expert_params_mapping( + expert_merge_mapping = FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/param2moe.py b/vllm/model_executor/models/param2moe.py index fddd1a8f173..4d1b3ff1b99 100644 --- a/vllm/model_executor/models/param2moe.py +++ b/vllm/model_executor/models/param2moe.py @@ -32,7 +32,7 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -353,7 +353,7 @@ class Param2MoEMoEBlock(nn.Module): else: self.shared_experts = None # type: ignore[assignment] - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, @@ -370,7 +370,7 @@ class Param2MoEMoEBlock(nn.Module): routed_scaling_factor=self.routed_scaling_factor, ) - def maybe_get_fused_moe(self) -> SharedFusedMoE: + def maybe_get_fused_moe(self) -> FusedMoE: return self.experts def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -690,7 +690,7 @@ class Param2MoEModel(nn.Module): return loaded_params def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index b5d13e926d7..7fc3c6a7dde 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -40,7 +40,7 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -164,7 +164,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): else: self.shared_expert = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_expert, num_experts=config.num_experts, top_k=config.num_experts_per_tok, @@ -418,7 +418,7 @@ class Qwen2MoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index f0f69d43537..6f080d07795 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -43,7 +43,7 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -205,7 +205,7 @@ class Qwen3MoeSparseMoeBlock(nn.Module): self.shared_expert_gate = None self.shared_expert = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_expert, gate=self.gate, num_experts=self.n_routed_experts, @@ -508,7 +508,7 @@ class Qwen3MoeModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 50d44dbbf63..2a4021be6e4 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -23,7 +23,7 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import ( GemmaRMSNorm as Qwen3NextRMSNorm, ) @@ -146,7 +146,7 @@ class Qwen3NextSparseMoeBlock(nn.Module): else: self.shared_expert = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_expert, gate=self.gate, num_experts=self.n_routed_experts, @@ -533,7 +533,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/sarvam.py b/vllm/model_executor/models/sarvam.py index 3656fc921b2..c770e203200 100644 --- a/vllm/model_executor/models/sarvam.py +++ b/vllm/model_executor/models/sarvam.py @@ -35,7 +35,7 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import SharedFusedMoE +from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -335,7 +335,7 @@ class SarvamMLAMoE(nn.Module): else: self.shared_experts = None - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.shared_experts, num_experts=self.num_experts, top_k=self.top_k, @@ -352,7 +352,7 @@ class SarvamMLAMoE(nn.Module): routed_scaling_factor=self.routed_scaling_factor, ) - def maybe_get_fused_moe(self) -> SharedFusedMoE: + def maybe_get_fused_moe(self) -> FusedMoE: return self.experts def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: @@ -529,7 +529,7 @@ class SarvamMLAModel(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return SharedFusedMoE.make_expert_params_mapping( + return FusedMoE.make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/step3p5.py b/vllm/model_executor/models/step3p5.py index 8b53c657b1e..a0bc1211bfe 100644 --- a/vllm/model_executor/models/step3p5.py +++ b/vllm/model_executor/models/step3p5.py @@ -24,7 +24,6 @@ from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.fused_moe import FusedMoE -from vllm.model_executor.layers.fused_moe.shared_fused_moe import SharedFusedMoE from vllm.model_executor.layers.layernorm import GemmaRMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -372,7 +371,7 @@ class FusedMoEBlock(nn.Module): quant_config=quant_config, prefix=f"{prefix}.share_expert", ) - self.experts = SharedFusedMoE( + self.experts = FusedMoE( shared_experts=self.share_expert, gate=self.gate, num_experts=config.moe_num_experts, diff --git a/vllm/utils/__init__.py b/vllm/utils/__init__.py index 9b481d63990..bf455c261f4 100644 --- a/vllm/utils/__init__.py +++ b/vllm/utils/__init__.py @@ -34,3 +34,16 @@ def length_from_prompt_token_ids_or_embeds( f" prompt_embeds={prompt_embeds_len}" ) return prompt_token_len + + +def is_moe_layer(module: torch.nn.Module) -> bool: + # TODO(bnell): Should use isinstance but can't due to circular dependencies. + def _check_bases(cls): + if cls.__name__ == "FusedMoE": + return True + + for b in cls.__bases__: + if _check_bases(b): + return True + + return _check_bases(module.__class__) From 9db4650e5e4c726eb5ae29330cd55e796567469c Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Tue, 21 Apr 2026 18:12:36 -0400 Subject: [PATCH 012/153] [MoE Refactor] Add more MoE layer tests (#39349) Signed-off-by: Bill Nell Signed-off-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- .../modular_kernel_tools/parallel_utils.py | 2 +- tests/kernels/moe/test_moe_layer.py | 137 ++++++++++++------ .../model_executor/layers/fused_moe/config.py | 3 +- 3 files changed, 98 insertions(+), 44 deletions(-) diff --git a/tests/kernels/moe/modular_kernel_tools/parallel_utils.py b/tests/kernels/moe/modular_kernel_tools/parallel_utils.py index 95004fa0ab4..07f244451b4 100644 --- a/tests/kernels/moe/modular_kernel_tools/parallel_utils.py +++ b/tests/kernels/moe/modular_kernel_tools/parallel_utils.py @@ -77,8 +77,8 @@ def _worker_parallel_launch( *args: Any, ) -> None: rank = node_rank * world_local_size + local_rank - torch.accelerator.set_device_index(local_rank) device = torch.device("cuda", local_rank) + torch.accelerator.set_device_index(device) torch.distributed.init_process_group( backend="cpu:gloo,cuda:nccl", init_method=init_method, diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index fca4096b086..838674db580 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -65,8 +65,8 @@ fp8_dtype = torch.float8_e4m3fn # current_platform.fp8_dtype SHAPE_COMBOS = [ (1, 128, 256), - (32, 1024, 512), - (222, 2048, 2048), + (32, 512, 512), + (222, 1024, 2048), ] MAX_M = max([x[0] for x in SHAPE_COMBOS]) @@ -95,7 +95,7 @@ if has_flashinfer_nvlink_one_sided(): BACKENDS += ["flashinfer_nvlink_one_sided"] if has_deep_ep(): - BACKENDS += ["deepep_low_latency", "deepep_high_throughput"] + BACKENDS += ["deepep_high_throughput", "deepep_low_latency"] if has_nixl_ep(): BACKENDS += ["nixl_ep"] @@ -103,6 +103,7 @@ if has_nixl_ep(): QUANT_METHODS = [ None, "fp8", + "fp8_blocked", "modelopt_fp8", "modelopt_fp4", ] @@ -114,10 +115,21 @@ BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = { "mori": {None, "fp8", "modelopt_fp8"}, "flashinfer_nvlink_two_sided": {None, "modelopt_fp8", "modelopt_fp4"}, "flashinfer_nvlink_one_sided": {None, "modelopt_fp8", "modelopt_fp4"}, - "deepep_low_latency": {None, "modelopt_fp8", "modelopt_fp4"}, - "deepep_high_throughput": {None, "fp8", "modelopt_fp8", "modelopt_fp4"}, + "deepep_low_latency": {None, "fp8_blocked", "modelopt_fp4"}, + "deepep_high_throughput": {None, "fp8_blocked", "modelopt_fp8", "modelopt_fp4"}, # noqa: E501 "nixl_ep": {None, "fp8", "modelopt_fp8"}, } + +# Map from backend -> (DP/EP support, DP support, TP support) +BACKEND_EP_DP_TP_SUPPORT: dict[str, tuple[bool, bool, bool]] = { + "allgather_reducescatter": (True, True, True), + "mori": (True, False, False), + "flashinfer_nvlink_two_sided": (False, True, False), + "flashinfer_nvlink_one_sided": (False, True, False), + "deepep_low_latency": (True, False, False), + "deepep_high_throughput": (True, False, False), + "nixl_ep": (True, False, False), +} # fmt: on # Which quantization methods support EPLB. @@ -424,27 +436,35 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]: f"Skipping unsupported K {config.k} in {config.backend} w/o EP.", ) - if config.enable_eplb and config.ep_size == 1: - return False, "EPLB requires EP." + if config.backend is not None: + supports_ep_dp, supports_dp, supports_tp = BACKEND_EP_DP_TP_SUPPORT[ + config.backend + ] - if config.enable_eplb and config.quantization not in EPLB_SUPPORTED_QUANTS: - return False, f"EPLB not supported with {config.quantization} quantization." + if config.tp_size > 1 and not supports_tp: + return False, f"{config.backend} does not support TP." - if config.enable_eplb and config.backend not in EPLB_SUPPORTED_BACKENDS: - return False, f"EPLB not supported with {config.backend}." + if config.dp_size > 1 and config.ep_size == 1 and not supports_dp: + return False, f"{config.backend} does not support DP." - if ( - config.backend is not None - and config.backend.startswith("flashinfer_nvlink") - and config.ep_size > 1 - ): - return False, "flashinfer_nvlink EP not yet supported." + if config.dp_size > 1 and config.ep_size > 1 and not supports_ep_dp: + return False, f"{config.backend} does not support EP/DP." + else: + if config.tp_size > 1 or config.ep_size > 1 or config.dp_size > 1: + return False, "An all2all backend is required for parallelism." - if config.enable_eplb and config.num_experts % config.dp_size != 0: - return False, "EPLB requires num_experts divisible by ep_size" + if config.enable_eplb: + if config.ep_size == 1: + return False, "EPLB requires EP." - if config.enable_eplb and config.ep_size == 1: - return False, "EPLB only works with EP+DP" + if config.quantization not in EPLB_SUPPORTED_QUANTS: + return False, f"EPLB not supported with {config.quantization} quantization." + + if config.backend not in EPLB_SUPPORTED_BACKENDS: + return False, f"EPLB not supported with {config.backend}." + + if config.num_experts % config.dp_size != 0: + return False, "EPLB requires num_experts divisible by ep_size" # Disable fp4 tests until flashinfer is updated or the Dockerfile is # modified to install cublasLt.h. See #39525. @@ -507,27 +527,48 @@ class QuantizedWeights: def _quantize_fp8_halves( w1: torch.Tensor, w2: torch.Tensor, + block_shape: list[int] | None = None, ) -> QuantizedWeights: """Quantize w13 gate/up halves separately to FP8, producing per-shard scales.""" half = w1.shape[1] // 2 w1q_a, w1s_a, _ = moe_quantize_weights( - w1[:, :half, :], None, fp8_dtype, False, None + w1[:, :half, :], + None, + fp8_dtype, + False, + block_shape, ) w1q_b, w1s_b, _ = moe_quantize_weights( - w1[:, half:, :], None, fp8_dtype, False, None + w1[:, half:, :], + None, + fp8_dtype, + False, + block_shape, ) assert w1s_a is not None and w1s_b is not None - w2q, w2s, _ = moe_quantize_weights(w2, None, fp8_dtype, False, None) + w2q, w2s, _ = moe_quantize_weights(w2, None, fp8_dtype, False, block_shape) assert w2s is not None + if block_shape is not None: + # Blocked quantization: scales have shape (E, n_tiles, k_tiles) + # Concatenate gate and up scales along the n_tiles dimension (dim=1) + # to match the concatenation of gate and up weights + w13_weight_scale = torch.cat([w1s_a, w1s_b], dim=1) + # w2 scales keep their blocked shape (E, k_tiles, n_tiles) + w2_weight_scale = w2s + else: + # Non-blocked quantization: scales have shape (E, 1, 1) + # Each w1s_x is (E, 1, 1) -> reshape to (E, 1), cat to (E, 2) + w13_weight_scale = torch.cat([w1s_a.view(-1, 1), w1s_b.view(-1, 1)], dim=1) + # w2s is (E, 1, 1) -> reshape to (E,) + w2_weight_scale = w2s.view(-1) + return QuantizedWeights( w13_weight=torch.cat([w1q_a, w1q_b], dim=1), w2_weight=w2q, - # Each w1s_x is (E, 1, 1) -> reshape to (E, 1), cat to (E, 2) - w13_weight_scale=torch.cat([w1s_a.view(-1, 1), w1s_b.view(-1, 1)], dim=1), - # w2s is (E, 1, 1) -> reshape to (E,) - w2_weight_scale=w2s.view(-1), + w13_weight_scale=w13_weight_scale, + w2_weight_scale=w2_weight_scale, ) @@ -536,7 +577,7 @@ def quantization_to_quant_dtype( ) -> torch.dtype | str | None: if quantization is None: return None - elif quantization in ["fp8", "modelopt_fp8"]: + elif quantization in ["fp8", "fp8_blocked", "modelopt_fp8"]: return fp8_dtype elif quantization in ["modelopt_fp4"]: return "nvfp4" @@ -558,6 +599,12 @@ def make_quant_config( if quantization == "fp8": return Fp8Config(True), _quantize_fp8_halves(w1, w2) + if quantization == "fp8_blocked": + block_shape = [128, 128] + return Fp8Config(True, weight_block_size=block_shape), _quantize_fp8_halves( + w1, w2, block_shape + ) + if quantization == "modelopt_fp8": qw = _quantize_fp8_halves(w1, w2) # why? @@ -896,11 +943,13 @@ def make_fused_moe_layer( **kwargs, ) + weight_scale_name = getattr(layer.quant_method, "weight_scale_name", "weight_scale") + for name, value in [ ("w13_weight", qw.w13_weight), ("w2_weight", qw.w2_weight), - ("w13_weight_scale", qw.w13_weight_scale), - ("w2_weight_scale", qw.w2_weight_scale), + (f"w13_{weight_scale_name}", qw.w13_weight_scale), + (f"w2_{weight_scale_name}", qw.w2_weight_scale), ("w13_weight_scale_2", qw.w13_weight_scale_2), ("w2_weight_scale_2", qw.w2_weight_scale_2), ("w13_input_scale", qw.w13_input_scale), @@ -922,7 +971,7 @@ def make_fake_moe_layer( top_k: int, global_num_experts: int, in_dtype: torch.dtype, - quant_dtype: torch.dtype | None, + quantization: str | None, renormalize: bool = False, shared_experts_config: SharedExpertsConfig | None = None, use_grouped_topk: bool = False, @@ -948,6 +997,7 @@ def make_fake_moe_layer( dp_size: int = 1, ep_size: int = 1, ) -> Callable: + quant_dtype = None activation = MoEActivation.from_str(activation) router = create_fused_moe_router( @@ -1139,7 +1189,6 @@ def _test_body_eplb( routed_output_transform=routed_output_transform, ) - # Necessary? if eplb_moe_layer._expert_map is not None: eplb_moe_layer._expert_map = eplb_moe_layer._expert_map.to(device) @@ -1267,6 +1316,7 @@ def _run_one_config( gate = test_data.gate routed_input_transform = test_data.routed_input_transform routed_output_transform = test_data.routed_output_transform + activation = "silu" baseline_layer = make_fake_moe_layer( w1=w1, @@ -1274,7 +1324,7 @@ def _run_one_config( top_k=top_k, global_num_experts=num_experts, in_dtype=in_dtype, - quant_dtype=None, # quantization_to_quant_dtype(quantization), + quantization=quantization, renormalize=False, shared_experts_config=shared_experts_config, gate=gate, @@ -1284,6 +1334,7 @@ def _run_one_config( tp_size=tp_size, ep_size=ep_size, dp_size=dp_size, + activation=activation, ) baseline_output = baseline_layer(hidden_states, router_logits) @@ -1328,9 +1379,9 @@ def _run_one_config( gate=gate, routed_input_transform=routed_input_transform, routed_output_transform=routed_output_transform, + activation=activation, ) - # Necessary? if moe_layer._expert_map is not None: moe_layer._expert_map = moe_layer._expert_map.to(device) @@ -1377,13 +1428,17 @@ def _run_one_config( atol, rtol = 7.6e-2, 7.6e-2 else: atol, rtol = 3.5e-2, 3.5e-2 - elif quantization in ("fp8", "modelopt_fp8"): - if k >= 2048: - atol, rtol = 7.6e-2, 7.6e-2 - else: - atol, rtol = 6e-2, 6e-2 + elif quantization in ("fp8", "fp8_blocked", "modelopt_fp8"): + atol, rtol = 6e-2, 6e-2 elif quantization == "modelopt_fp4": - atol = rtol = 1e-1 + k * 5e-4 + if k >= 2048: + atol = rtol = 1e-1 + (k * 1e-4) + else: + atol = rtol = 1e-1 + + if backend == "allgather_reducescatter" and tp_size > 1: + atol += 2e-1 + rtol += 2e-1 else: atol, rtol = 6e-2, 6e-2 diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 231c5652e45..00d7d8e7890 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -990,8 +990,7 @@ class FusedMoEParallelConfig: @property def use_batched_activation_format(self): - # TODO(bnell): nixl also uses batched format - return self.use_deepep_ll_kernels + return self.use_deepep_ll_kernels or self.use_nixl_ep_kernels @property def use_ag_rs_all2all_kernels(self): From 96a85c57501fe12592efc1c601a8fa2fd8214b81 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Tue, 21 Apr 2026 18:16:59 -0400 Subject: [PATCH 013/153] [Startup][UX] Enable CUDAGraph memory profiling by default (#38284) Signed-off-by: Matthew Bonanni Co-authored-by: Tyler Michael Smith --- tests/distributed/test_torchrun_example.py | 2 +- .../distributed/test_torchrun_example_moe.py | 2 +- tests/v1/determinism/test_batch_invariance.py | 2 +- tests/v1/e2e/spec_decode/test_spec_decode.py | 4 +-- vllm/config/cache.py | 4 +-- vllm/entrypoints/llm.py | 2 +- vllm/envs.py | 6 ++-- vllm/v1/worker/gpu_worker.py | 31 +++++++++---------- 8 files changed, 26 insertions(+), 27 deletions(-) diff --git a/tests/distributed/test_torchrun_example.py b/tests/distributed/test_torchrun_example.py index 8c9898ca20f..af9c76d9c7e 100644 --- a/tests/distributed/test_torchrun_example.py +++ b/tests/distributed/test_torchrun_example.py @@ -29,7 +29,7 @@ llm = LLM( tensor_parallel_size=2, pipeline_parallel_size=int(os.getenv("PP_SIZE", 1)), distributed_executor_backend="external_launcher", - gpu_memory_utilization=random.uniform(0.7, 0.9), + gpu_memory_utilization=random.uniform(0.8, 0.92), seed=0, ) diff --git a/tests/distributed/test_torchrun_example_moe.py b/tests/distributed/test_torchrun_example_moe.py index 7b20a23f5be..c0437d9b930 100644 --- a/tests/distributed/test_torchrun_example_moe.py +++ b/tests/distributed/test_torchrun_example_moe.py @@ -36,7 +36,7 @@ llm = LLM( pipeline_parallel_size=int(os.getenv("PP_SIZE", "1")), enable_expert_parallel=int(os.getenv("ENABLE_EP", "0")) == 1, distributed_executor_backend="external_launcher", - gpu_memory_utilization=random.uniform(0.7, 0.9), + gpu_memory_utilization=random.uniform(0.8, 0.92), seed=0, max_model_len=1024, max_num_seqs=16, diff --git a/tests/v1/determinism/test_batch_invariance.py b/tests/v1/determinism/test_batch_invariance.py index eb5dec3e215..41242da5c22 100644 --- a/tests/v1/determinism/test_batch_invariance.py +++ b/tests/v1/determinism/test_batch_invariance.py @@ -65,7 +65,7 @@ def test_v1_generation_is_deterministic_across_batch_sizes_with_needle( assert max_batch_size >= 2, "Batch size should be >= 2 to mix needle." # Keep GPU memory usage low to avoid startup allocation failures. - gpu_mem_util = float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.4")) + gpu_mem_util = float(os.getenv("VLLM_GPU_MEMORY_UTILIZATION", "0.5")) max_model_len = int(os.getenv("VLLM_MAX_MODEL_LEN", "5120")) # Sampling parameters: longer outputs with a more random-sounding diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index a8fed766528..03448e9bb3e 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -321,7 +321,7 @@ def test_speculators_model_integration( test_prompts = get_test_prompts(mm_enabled=False) # First run: Direct speculator model (simplified integration) - spec_llm = LLM(model=model_path, max_model_len=4096) + spec_llm = LLM(model=model_path, max_model_len=4096, gpu_memory_utilization=0.92) evaluate_llm_for_gsm8k( spec_llm, expected_accuracy_threshold=expected_accuracy_threshold ) @@ -351,7 +351,7 @@ def test_speculators_model_integration( cleanup_dist_env_and_memory() # Second run: Reference without speculative decoding - ref_llm = LLM(model=verifier_model, max_model_len=4096) + ref_llm = LLM(model=verifier_model, max_model_len=4096, gpu_memory_utilization=0.92) ref_outputs = ref_llm.chat(test_prompts, sampling_config) del ref_llm torch.accelerator.empty_cache() diff --git a/vllm/config/cache.py b/vllm/config/cache.py index e34f57dea47..48ff1a32ec0 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -51,10 +51,10 @@ class CacheConfig: """Whether block_size was explicitly provided. Derived automatically.""" user_specified_mamba_block_size: bool = field(default=False, init=False) """Whether mamba_block_size was explicitly provided. Derived automatically.""" - gpu_memory_utilization: float = Field(default=0.9, gt=0, le=1) + gpu_memory_utilization: float = Field(default=0.92, gt=0, le=1) """The fraction of GPU memory to be used for the model executor, which can range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory - utilization. If unspecified, will use the default value of 0.9. This is a + utilization. If unspecified, will use the default value of 0.92. This is a per-instance limit, and only applies to the current vLLM instance. It does not matter if you have another vLLM instance running on the same GPU. For example, if you have two vLLM instances running on the same GPU, you can diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 61f32a0d098..0146cb83aa6 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -228,7 +228,7 @@ class LLM: tokenizer_revision: str | None = None, chat_template: Path | str | None = None, seed: int = 0, - gpu_memory_utilization: float = 0.9, + gpu_memory_utilization: float = 0.92, cpu_offload_gb: float = 0, offload_group_size: int = 0, offload_num_in_group: int = 1, diff --git a/vllm/envs.py b/vllm/envs.py index 8ed1d33434c..71566f4c2d4 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -253,7 +253,7 @@ if TYPE_CHECKING: VLLM_CUDA_COMPATIBILITY_PATH: str | None = None VLLM_ELASTIC_EP_SCALE_UP_LAUNCH: bool = False VLLM_ELASTIC_EP_DRAIN_REQUESTS: bool = False - VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = False + VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS: bool = True VLLM_NIXL_EP_MAX_NUM_RANKS: int = 32 VLLM_XPU_ENABLE_XPU_GRAPH: bool = False VLLM_LORA_ENABLE_DUAL_STREAM: bool = False @@ -1687,9 +1687,9 @@ environment_variables: dict[str, Callable[[], Any]] = { ), # If set to 1, enable CUDA graph memory estimation during memory profiling. # This profiles CUDA graph memory usage to provide more accurate KV cache - # memory allocation. Disabled by default to preserve existing behavior. + # memory allocation. Enabled by default as of v0.21.0 "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS": lambda: bool( - int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "0")) + int(os.getenv("VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS", "1")) ), # NIXL EP environment variables "VLLM_NIXL_EP_MAX_NUM_RANKS": lambda: int( diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index ec8f9c6dd31..98f3212bae0 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -454,14 +454,13 @@ class Worker(WorkerBase): 1.0, ) logger.info( - "CUDA graph memory profiling is enabled " - "(VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1). " - "This will become the default in v0.21. " - "The current --gpu-memory-utilization=%.4f is equivalent " - "to --gpu-memory-utilization=%.4f without CUDA graph " - "memory profiling. To maintain the same effective KV " - "cache size as before, increase " - "--gpu-memory-utilization to %.4f.", + "CUDA graph memory profiling is enabled (default since " + "v0.21.0). The current --gpu-memory-utilization=%.4f is " + "equivalent to --gpu-memory-utilization=%.4f without " + "CUDA graph memory profiling. To maintain the same " + "effective KV cache size as before, increase " + "--gpu-memory-utilization to %.4f. To disable, set " + "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0.", current_util, equiv_util, suggested_util, @@ -471,14 +470,14 @@ class Worker(WorkerBase): round(current_util + cg_util_delta, 4), 1.0, ) - logger.info( - "In v0.21, CUDA graph memory profiling will be enabled " - "by default (VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1), " - "which more accurately accounts for CUDA graph memory " - "during KV cache allocation. To try it now, set " - "VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=1 and increase " - "--gpu-memory-utilization from %.4f to %.4f to maintain " - "the same effective KV cache size.", + logger.warning( + "CUDA graph memory profiling is disabled " + "(VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0). " + "Without it, CUDA graph memory is not accounted for " + "during KV cache allocation, which may require lowering " + "--gpu-memory-utilization to avoid OOM. Consider " + "re-enabling it (the default as of v0.21.0) and increasing " + "--gpu-memory-utilization from %.4f to %.4f.", current_util, suggested_util, ) From 583e6f22269c15ff3f69431b756486c64bf3ea0c Mon Sep 17 00:00:00 2001 From: TJian Date: Wed, 22 Apr 2026 09:18:07 +0900 Subject: [PATCH 014/153] [ROCm] [Wheel] [Bugfix] [Critical] Remove any packages installed from github from rocm.txt e.g `fastsafetensors` as it is incompatible with `uv pip` (#40461) Signed-off-by: tjtanaa --- docker/Dockerfile.rocm | 24 ++++++++++++++++++++++++ requirements/rocm.txt | 4 +--- requirements/test/rocm.txt | 4 +--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 43be4e669d9..1c6cdb74d6e 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -307,6 +307,30 @@ RUN --mount=type=bind,source=.git,target=vllm/.git \ && echo "Detected vLLM version: ${VLLM_VERSION}" \ && echo "${VLLM_VERSION}" > /tmp/vllm_version.txt +# Fail if git-based package dependencies are found in requirements files +# (uv doesn't handle git+ URLs well, and packages should be distributed on PyPI) +# Extra notes: pip install is able to handle git+ URLs, but uv doesn't. +RUN echo "Checking for git-based packages in requirements files..." \ + && echo "Checking common.txt for git-based packages:" \ + && if grep -q 'git+' ${COMMON_WORKDIR}/vllm/requirements/common.txt; then \ + echo "ERROR: Git-based packages found in common.txt:"; \ + grep 'git+' ${COMMON_WORKDIR}/vllm/requirements/common.txt; \ + echo "Please publish these packages to PyPI instead of using git dependencies."; \ + exit 1; \ + else \ + echo " ✓ No git-based packages found in common.txt"; \ + fi \ + && echo "Checking rocm.txt for git-based packages:" \ + && if grep -q 'git+' ${COMMON_WORKDIR}/vllm/requirements/rocm.txt; then \ + echo "ERROR: Git-based packages found in rocm.txt:"; \ + grep 'git+' ${COMMON_WORKDIR}/vllm/requirements/rocm.txt; \ + echo "Please publish these packages to PyPI instead of using git dependencies."; \ + exit 1; \ + else \ + echo " ✓ No git-based packages found in rocm.txt"; \ + fi \ + && echo "All requirements files are clean - no git-based packages found" + # Pin vLLM dependencies to exact versions of custom ROCm wheels # This ensures 'pip install vllm' automatically installs correct torch/triton/torchvision/amdsmi COPY tools/vllm-rocm/pin_rocm_dependencies.py /tmp/pin_rocm_dependencies.py diff --git a/requirements/rocm.txt b/requirements/rocm.txt index deaeae2d5e2..6639e71a4b9 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -20,6 +20,4 @@ conch-triton-kernels==1.2.1 timm>=1.0.17 # amd-quark: required for Quark quantization on ROCm # To be consistent with test_quark.py -amd-quark>=0.8.99 -# Required for faster safetensors model loading -fastsafetensors @ git+https://github.com/foundation-model-stack/fastsafetensors.git@0.2.2 \ No newline at end of file +amd-quark>=0.8.99 \ No newline at end of file diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 33ea3a3f0db..6558c2f3811 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -276,9 +276,7 @@ fastar==0.10.0 fastparquet==2026.3.0 # via genai-perf fastsafetensors @ git+https://github.com/foundation-model-stack/fastsafetensors.git@65d80088fca7a8f567fba30415fbcc80f7d2259c - # via - # -c requirements/rocm.txt - # -r requirements/test/rocm.in + # via -r requirements/test/rocm.in filelock==3.25.2 # via # -c requirements/common.txt From 6ff8dea0756d32f2487b8981ec2626d0402ef0ba Mon Sep 17 00:00:00 2001 From: Khushali Desai Date: Tue, 21 Apr 2026 17:19:50 -0700 Subject: [PATCH 015/153] [Bugfix] avoid warmup if text only expectation in multi_modal run (#40409) Signed-off-by: khushali9 --- tests/renderers/test_warmup.py | 111 +++++++++++++++++++++++++++++++++ vllm/renderers/base.py | 4 +- 2 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 tests/renderers/test_warmup.py diff --git a/tests/renderers/test_warmup.py b/tests/renderers/test_warmup.py new file mode 100644 index 00000000000..90cbe2bef67 --- /dev/null +++ b/tests/renderers/test_warmup.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for BaseRenderer.warmup MM-warmup behavior. + +These tests exercise: + - Zero-limit modalities are filtered from mm_counts passed to + get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...) + - MM warmup is skipped entirely when mm_processor is None + +No model weights are required: warmup() is called directly on a MagicMock +that acts as the renderer instance. +""" + +from unittest.mock import MagicMock, patch + +from vllm.renderers.base import BaseRenderer +from vllm.renderers.params import ChatParams + + +def _make_renderer_mock(mm_limits: dict[str, int]) -> MagicMock: + """Return a MagicMock that quacks like a BaseRenderer instance. + + render_chat is mocked to raise ChatTemplateResolutionError so the chat + warmup block is skipped cleanly, keeping the test focused on MM warmup. + """ + from vllm.entrypoints.chat_utils import ChatTemplateResolutionError + + renderer = MagicMock() + + # chat warmup: make render_chat raise so we skip past it cleanly + renderer.render_chat.side_effect = ChatTemplateResolutionError("no template") + + # MM processor with configurable limits + mm_processor = MagicMock() + mm_processor.info.allowed_mm_limits = mm_limits + renderer.mm_processor = mm_processor + + return renderer + + +class TestMmWarmupZeroLimitFiltering: + """Zero-limit modalities must be excluded from mm_counts.""" + + def test_zero_limit_modality_excluded_from_mm_counts(self): + """A modality with limit=0 must not appear in mm_counts.""" + renderer = _make_renderer_mock({"image": 1, "video": 0}) + + with patch("vllm.multimodal.processing.TimingContext", autospec=True): + BaseRenderer.warmup(renderer, ChatParams()) + + get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs.assert_called_once() + _, kwargs = get_inputs.call_args + assert "video" not in kwargs["mm_counts"] + assert kwargs["mm_counts"]["image"] == 1 + + def test_all_zero_limits_passes_empty_mm_counts(self): + """When all limits are 0, mm_counts must be empty.""" + renderer = _make_renderer_mock({"image": 0, "video": 0}) + + with patch("vllm.multimodal.processing.TimingContext", autospec=True): + BaseRenderer.warmup(renderer, ChatParams()) + + get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs.assert_called_once() + _, kwargs = get_inputs.call_args + assert kwargs["mm_counts"] == {} + + def test_positive_limits_all_included_in_mm_counts(self): + """All modalities with limit > 0 must be present in mm_counts.""" + renderer = _make_renderer_mock({"image": 2, "video": 1}) + + with patch("vllm.multimodal.processing.TimingContext", autospec=True): + BaseRenderer.warmup(renderer, ChatParams()) + + get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs + get_inputs.assert_called_once() + _, kwargs = get_inputs.call_args + assert kwargs["mm_counts"] == {"image": 1, "video": 1} + + +class TestMmWarmupRunsNormally: + """MM warmup must run when mm_processor is set and limits > 0.""" + + def test_processor_apply_called(self): + renderer = _make_renderer_mock({"image": 1}) + + with patch("vllm.multimodal.processing.TimingContext", autospec=True): + BaseRenderer.warmup(renderer, ChatParams()) + + renderer.mm_processor.apply.assert_called_once() + + def test_mm_cache_cleared_after_warmup(self): + renderer = _make_renderer_mock({"image": 1}) + + with patch("vllm.multimodal.processing.TimingContext", autospec=True): + BaseRenderer.warmup(renderer, ChatParams()) + + renderer.clear_mm_cache.assert_called_once() + + +class TestMmWarmupSkippedWhenNoProcessor: + """MM warmup must be skipped when mm_processor is None (text-only model).""" + + def test_no_warmup_without_processor(self): + renderer = _make_renderer_mock({}) + renderer.mm_processor = None # override to None + + BaseRenderer.warmup(renderer, ChatParams()) + + renderer.model_config.get_multimodal_config.assert_not_called() diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 02c3a4f35c9..cf13d74bd1f 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -226,7 +226,9 @@ class BaseRenderer(ABC, Generic[_T]): model_config = self.model_config mm_config = model_config.get_multimodal_config() processor = self.mm_processor - mm_limits = processor.info.allowed_mm_limits + mm_limits = { + k: v for k, v in processor.info.allowed_mm_limits.items() if v > 0 + } try: logger.debug("Warming up multi-modal processing...") From 46794958f0c60bc3a4f30562032e991222ab5d56 Mon Sep 17 00:00:00 2001 From: Jhao-Ting Chen Date: Tue, 21 Apr 2026 17:46:53 -0700 Subject: [PATCH 016/153] test: add nan/inf clamp regression test for fused_topk_bias (#40553) Signed-off-by: Jhao-Ting Chen --- tests/kernels/moe/test_fused_topk.py | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tests/kernels/moe/test_fused_topk.py b/tests/kernels/moe/test_fused_topk.py index a0e3580ee5a..825cd20263d 100644 --- a/tests/kernels/moe/test_fused_topk.py +++ b/tests/kernels/moe/test_fused_topk.py @@ -202,3 +202,72 @@ def test_fused_topk_nan_inf_clamp( f"Row {row} has non-finite weights {topk_weights[row].tolist()} " f"(bad_value={bad_value}, scoring_func={scoring_func})" ) + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="This test is skipped on non-CUDA platform." +) +@pytest.mark.parametrize("num_experts", [6, 8, 16]) +@pytest.mark.parametrize("topk", [3, 4]) +@pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"]) +@pytest.mark.parametrize("bad_value", [float("nan"), float("inf")]) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.half, torch.float32]) +def test_fused_topk_bias_nan_inf_clamp( + num_experts: int, + topk: int, + scoring_func: str, + bad_value: float, + dtype: torch.dtype, +): + """Regression test: NaN/Inf in gating logits must not produce duplicate + expert IDs or non-finite weights when e_score_correction_bias is present. + + Same scenario as test_fused_topk_nan_inf_clamp but exercising the bias + path (fused_topk_bias) so the fix in topk_softmax_kernels.cu is covered + for that entry point as well. + """ + torch.manual_seed(0) + num_tokens = 4 + hidden_size = 1024 + hidden_states = torch.randn((num_tokens, hidden_size), dtype=dtype, device="cuda") + e_score_correction_bias = torch.randn( + (num_experts,), dtype=torch.float32, device="cuda" + ) + + gating_output = torch.randn((num_tokens, num_experts), dtype=dtype, device="cuda") + gating_output[1:, :] = bad_value + + topk_weights, topk_ids = fused_topk_bias( + hidden_states=hidden_states, + gating_output=gating_output, + e_score_correction_bias=e_score_correction_bias, + topk=topk, + renormalize=False, + scoring_func=scoring_func, + ) + + # Normal row must still match the torch reference. + ref_weights, ref_ids = torch_topk( + gating_output=gating_output[:1], + topk=topk, + renormalize=False, + e_score_correction_bias=e_score_correction_bias, + scoring_func=scoring_func, + ) + torch.testing.assert_close( + ref_weights.to(torch.float32), topk_weights[:1], atol=1e-2, rtol=1e-2 + ) + torch.testing.assert_close(ref_ids.to(torch.int32), topk_ids[:1], atol=0, rtol=0) + + # Poisoned rows: IDs must be unique (no duplicates) and weights must be + # finite (no NaN/Inf propagation into downstream MoE kernels). + for row in range(1, num_tokens): + row_ids = topk_ids[row] + assert row_ids.unique().numel() == topk, ( + f"Row {row} has duplicate expert IDs {row_ids.tolist()} " + f"(bad_value={bad_value}, scoring_func={scoring_func})" + ) + assert torch.isfinite(topk_weights[row]).all(), ( + f"Row {row} has non-finite weights {topk_weights[row].tolist()} " + f"(bad_value={bad_value}, scoring_func={scoring_func})" + ) From cefa5281a752068aed17208506054b03322e4d37 Mon Sep 17 00:00:00 2001 From: rasmith Date: Tue, 21 Apr 2026 19:48:25 -0500 Subject: [PATCH 017/153] [ROCm][P/D][MORI][BugFix] Ensure correct api is used when making requests to prefill / decode nodes (#39835) Signed-off-by: Randall Smith --- .../moriio_toy_proxy_server.py | 39 +++++++++++++------ .../v1/moriio/moriio_connector.py | 2 +- 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py b/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py index 33fb56c8802..e2a0bfc7c9b 100644 --- a/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py +++ b/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py @@ -12,7 +12,7 @@ import aiohttp import msgpack import regex as re import zmq -from quart import Quart, make_response, request +from quart import Quart, Request, make_response, request from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOConstants, @@ -139,10 +139,13 @@ async def send_request_to_prefill( return await response.json() else: - raise RuntimeError( - "send_request_to_prefill response.status != 200response.status = ", - response.status, + error_message = ( + f"send_request_to_prefill response ={response}," + f"reason={response.reason}, status={response.status}," + f"method={response.method}, url={response.url}," + f"real_url={response.real_url}" ) + raise RuntimeError(error_message) async def start_decode_request(endpoint, req_data, request_id): @@ -163,9 +166,13 @@ async def stream_decode_response(session, response, request_id): async for chunk_bytes in response.content.iter_chunked(1024): yield chunk_bytes else: - raise RuntimeError( - f"decode response.status != 200, status = {response.status}" + error_message = ( + f"stream_decode_response response ={response}," + f"reason={response.reason}, status={response.status}," + f"method={response.method}, url={response.url}," + f"real_url={response.real_url}" ) + raise RuntimeError(error_message) finally: await session.close() @@ -175,8 +182,16 @@ def example_round_robin_dp_loader(request_number, dp_size): @app.route("/v1/completions", methods=["POST"]) +async def handle_completions_request(): + return await handle_request("/completions", request) + + @app.route("/v1/chat/completions", methods=["POST"]) -async def handle_request(): +async def handle_chat_completions_request(): + return await handle_request("/chat/completions", request) + + +async def handle_request(api: str, request: Request): try: with _list_lock: global request_nums @@ -230,9 +245,10 @@ async def handle_request(): ) req_data_to_prefill["kv_transfer_params"]["transfer_id"] = transfer_id + prefill_request_url = prefill_instance_endpoint["request_address"] + api send_prefill_task = asyncio.create_task( send_request_to_prefill( - prefill_instance_endpoint["request_address"], + prefill_request_url, req_data_to_prefill, request_id, decode_instance_endpoint, @@ -241,7 +257,7 @@ async def handle_request(): selected_prefill_dp_rank, ) ) - ip, port = extract_ip_port_fast(prefill_instance_endpoint["request_address"]) + ip, port = extract_ip_port_fast(prefill_request_url) req_data["max_tokens"] -= 1 @@ -276,10 +292,9 @@ async def handle_request(): req_data["kv_transfer_params"]["remote_dp_rank"] = selected_prefill_dp_rank req_data["kv_transfer_params"]["transfer_id"] = transfer_id + decode_request_url = decode_instance_endpoint["request_address"] + api decode_request_task = asyncio.create_task( - start_decode_request( - decode_instance_endpoint["request_address"], req_data, request_id - ) + start_decode_request(decode_request_url, req_data, request_id) ) session, decode_response = await decode_request_task diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index dcde7665f34..0fd6d81f23e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -846,7 +846,7 @@ class MoRIIOConnectorWorker: ] def _ping(self, zmq_context): - http_request_address = f"http://{self.request_address}/v1/completions" + http_request_address = f"http://{self.request_address}/v1" role = "P" if self.is_producer else "D" retry_count = 0 From f90aa446629f74d4d7049771fe938e04b5c2d3ef Mon Sep 17 00:00:00 2001 From: Soila Kavulya Date: Tue, 21 Apr 2026 18:26:33 -0700 Subject: [PATCH 018/153] [NIXL][XPU]Fix nixl import on XPU (#40430) Signed-off-by: Soila Kavulya --- vllm/distributed/nixl_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/distributed/nixl_utils.py b/vllm/distributed/nixl_utils.py index b2b433339be..2da37017a37 100644 --- a/vllm/distributed/nixl_utils.py +++ b/vllm/distributed/nixl_utils.py @@ -24,7 +24,7 @@ if "UCX_RCACHE_MAX_UNRELEASED" not in os.environ: os.environ["UCX_RCACHE_MAX_UNRELEASED"] = "1024" try: - if current_platform.is_cuda(): + if not current_platform.is_rocm(): from nixl._api import nixl_agent as NixlWrapper else: from rixl._api import nixl_agent as NixlWrapper @@ -35,7 +35,7 @@ except ImportError: NixlWrapper = None # type: ignore[assignment, misc] try: - if current_platform.is_cuda(): + if not current_platform.is_rocm(): from nixl._api import nixl_agent_config else: from rixl._api import nixl_agent_config @@ -44,7 +44,7 @@ except ImportError: logger.warning_once("NIXL agent config is not available") try: - if current_platform.is_cuda(): + if not current_platform.is_rocm(): from nixl._bindings import nixlXferTelemetry else: from rixl._bindings import nixlXferTelemetry From f946659fff3c4be2105a4ebfdd4dd0dbec0d6b8a Mon Sep 17 00:00:00 2001 From: EdalatiAli Date: Tue, 21 Apr 2026 21:58:33 -0400 Subject: [PATCH 019/153] [Bugfix] Fix W4A8_FP8 MoE tp>1 correctness and view() TypeError (#40310) Signed-off-by: EdalatiAli Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py | 4 ++++ vllm/model_executor/layers/quantization/utils/quant_utils.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py index 74cb0b4f6e1..ab805591dee 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py @@ -198,11 +198,15 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): # encode and reorder weight tensors, and get the layout to pass to # the grouped gemm kernel. `b_strides1/2` specifies the entire layout convert_packed_uint4b8_to_signed_int4_inplace(layer.w13_weight_packed) + # mirror the sync in CutlassW4A8LinearKernel; required for tp>1 correctness + torch.accelerator.synchronize() w13_weight_shuffled, self.b_strides1 = ( ops.cutlass_encode_and_reorder_int4b_grouped(layer.w13_weight_packed) ) replace_parameter(layer, "w13_weight_packed", w13_weight_shuffled) convert_packed_uint4b8_to_signed_int4_inplace(layer.w2_weight_packed) + # mirror the sync in CutlassW4A8LinearKernel; required for tp>1 correctness + torch.accelerator.synchronize() w2_weight_shuffled, self.b_strides2 = ( ops.cutlass_encode_and_reorder_int4b_grouped(layer.w2_weight_packed) ) diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index d1b1b77988c..de76deb191d 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -818,7 +818,7 @@ def convert_bf16_scales_to_fp8( # restore original shape fp8_scales = fp8_scales.view(orig_shape) - chan_scales = chan_scales.view(orig_shape[:-1], -1) + chan_scales = chan_scales.view(*orig_shape[:-1], -1) return fp8_scales, chan_scales From 2463f00fb690a7b182050285c0179da03aad66fe Mon Sep 17 00:00:00 2001 From: rasmith Date: Tue, 21 Apr 2026 21:21:02 -0500 Subject: [PATCH 020/153] [AMD][CI][BugFix] Override normalize_e4m3fn_to_e4m3fnuz for fnuz machines in test_moe_layer_no_parallel (#40550) Signed-off-by: Randall Smith --- tests/kernels/moe/test_moe_layer.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index 838674db580..07c04a16802 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -17,6 +17,7 @@ from typing import get_args import pytest import torch +import vllm.model_executor.layers.quantization.utils.w8a8_utils from tests.kernels.moe.modular_kernel_tools.parallel_utils import ( ProcessGroupInfo, _set_vllm_config, @@ -144,6 +145,24 @@ EPLB_SUPPORTED_QUANTS: list[str | None] = [None, "fp8"] EPLB_SUPPORTED_BACKENDS: list[str] = ["allgather_reducescatter"] +def mock_normalize_e4m3fn_to_e4m3fnuz( + weight: torch.Tensor, + weight_scale: torch.Tensor, + input_scale: torch.Tensor | None = None, +): + return weight, weight_scale, input_scale + + +# Needed since weights will already be in e4m3fnuz format on platforms that +# use the fnuz fp8 format and the normalize_e4m3fn_to_e4m3fnuz() function +# is not being tested here. +# NOTE: The weights are quantized by moe_quantize_weights_2d in +# _quantize_fp8_halves. +# NOTE: Not able to use monkeypatch because of the spawned parallel workers. +def override_normalize_e4m3fn_to_e4m3fnuz(): + vllm.model_executor.layers.quantization.utils.w8a8_utils.normalize_e4m3fn_to_e4m3fnuz = mock_normalize_e4m3fn_to_e4m3fnuz # noqa: E501 + + def maybe_roundup_layer_hidden_size( hidden_size: int, act_dtype: torch.dtype, @@ -1471,6 +1490,11 @@ def test_moe_layer_no_parallel( if os.environ.get("VLLM_LOGGING_LEVEL") is None: monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR") + # Needed since weights will already be in e4m3fnuz format and the + # normalize_e4m3fn_to_e4m3fnuz() function is not being tested here. + if current_platform.is_fp8_fnuz(): + override_normalize_e4m3fn_to_e4m3fnuz() + test_config = MoETestConfig( m, n, @@ -1546,6 +1570,9 @@ def _parallel_worker( dp_rank = vllm_config.parallel_config.data_parallel_rank + if current_platform.is_fp8_fnuz(): + override_normalize_e4m3fn_to_e4m3fnuz() + for test_config in test_configs: cc = vllm_config.compilation_config if "from_forward_context" in cc.static_forward_context: From 6f2c71be8ffd65648bbf99d571c5353ff5b77f24 Mon Sep 17 00:00:00 2001 From: Jaseel Muhammad Date: Wed, 22 Apr 2026 07:14:57 +0400 Subject: [PATCH 021/153] [Multimodal] Add PyAV video backend for concurrent video decoding (#39986) Signed-off-by: Jaseel Muhammad Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py --- .../multimodal/processing/test_glm4_1v.py | 12 +- tests/multimodal/test_video.py | 121 ++++++-- vllm/envs.py | 7 +- vllm/multimodal/video.py | 272 +++++++++++------- 4 files changed, 292 insertions(+), 120 deletions(-) diff --git a/tests/models/multimodal/processing/test_glm4_1v.py b/tests/models/multimodal/processing/test_glm4_1v.py index f70d0052427..5798c566347 100644 --- a/tests/models/multimodal/processing/test_glm4_1v.py +++ b/tests/models/multimodal/processing/test_glm4_1v.py @@ -6,7 +6,7 @@ import pytest from vllm.assets.video import VideoAsset from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import batched_tensors_equal -from vllm.multimodal.video import OpenCVDynamicVideoBackend, OpenCVVideoBackend +from vllm.multimodal.video import DynamicVideoBackend, VideoBackend from ...utils import build_model_context @@ -70,9 +70,11 @@ def test_processor_override( @pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"]) @pytest.mark.parametrize("fps", [2]) +@pytest.mark.parametrize("backend", ["opencv", "pyav"]) def test_video_loader_consistency( model_id: str, fps: int, + backend: str, ): """ Ensure dynamic video loader (pre-sampled by loader) and normal video @@ -93,9 +95,11 @@ def test_video_loader_consistency( with open(video_path, "rb") as f: video_bytes = f.read() - static_video, static_metadata = OpenCVVideoBackend.load_bytes(video_bytes) - dynamic_video, dynamic_metadata = OpenCVDynamicVideoBackend.load_bytes( - video_bytes, fps=fps + static_video, static_metadata = VideoBackend.load_bytes( + video_bytes, backend=backend + ) + dynamic_video, dynamic_metadata = DynamicVideoBackend.load_bytes( + video_bytes, fps=fps, backend=backend ) # pre-sampled loader shouldn't read all frames diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 3ece384348b..e82883ece33 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -71,7 +71,9 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch): video_data = f.read() loader = VIDEO_LOADER_REGISTRY.load("opencv") - frames, metadata = loader.load_bytes(video_data, num_frames=-1) + frames, metadata = loader.load_bytes( + video_data, num_frames=-1, backend="opencv" + ) # Verify metadata consistency: # frames_indices must match actual loaded frames @@ -158,12 +160,12 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch): # Test WITHOUT recovery - should have fewer frames due to failures frames_no_recovery, meta_no = loader.load_bytes( - video_data, num_frames=8, frame_recovery=False + video_data, num_frames=8, frame_recovery=False, backend="opencv" ) # Test WITH recovery - should recover using next valid frames frames_with_recovery, meta_yes = loader.load_bytes( - video_data, num_frames=8, frame_recovery=True + video_data, num_frames=8, frame_recovery=True, backend="opencv" ) # With recovery should have MORE frames than without @@ -214,12 +216,12 @@ def test_video_recovery_with_corrupted_file(monkeypatch: pytest.MonkeyPatch): # Test without recovery - frame 17 will be skipped frames_no_recovery, meta_no_recovery = loader.load_bytes( - video_data, num_frames=8, frame_recovery=False + video_data, num_frames=8, frame_recovery=False, backend="opencv" ) # Test with recovery - frame 18 should fill in for frame 17 frames_with_recovery, meta_with_recovery = loader.load_bytes( - video_data, num_frames=8, frame_recovery=True + video_data, num_frames=8, frame_recovery=True, backend="opencv" ) # Verify metadata consistency for both modes @@ -271,12 +273,16 @@ def test_video_recovery_dynamic_backend(monkeypatch: pytest.MonkeyPatch): # Test without recovery frames_no_recovery, meta_no = loader.load_bytes( - video_data, fps=2, max_duration=10, frame_recovery=False + video_data, + fps=2, + max_duration=10, + frame_recovery=False, + backend="opencv", ) # Test with frame_recovery enabled frames_with_recovery, meta_with = loader.load_bytes( - video_data, fps=2, max_duration=10, frame_recovery=True + video_data, fps=2, max_duration=10, frame_recovery=True, backend="opencv" ) # Verify basic properties @@ -310,27 +316,81 @@ def dummy_video_path(tmp_path): return video_path +# ============================================================================ +# PyAV Backend Tests +# ============================================================================ + + +def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyPatch): + """Test that the pyav codec backend can load frames from a valid video.""" + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") + + with open(dummy_video_path, "rb") as f: + video_data = f.read() + + loader = VIDEO_LOADER_REGISTRY.load("opencv") + frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav") + + assert frames.ndim == 4 + assert frames.shape[3] == 3 # RGB + assert frames.shape[0] == 8 + assert frames.shape[0] == len(metadata["frames_indices"]) + assert metadata["video_backend"] == "pyav" + assert "total_num_frames" in metadata + assert "fps" in metadata + assert "duration" in metadata + + +def test_pyav_dynamic_backend_loads_frames( + dummy_video_path, monkeypatch: pytest.MonkeyPatch +): + """Test that the pyav codec with dynamic sampling can load frames.""" + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic") + + with open(dummy_video_path, "rb") as f: + video_data = f.read() + + loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") + frames, metadata = loader.load_bytes( + video_data, fps=2, max_duration=10, backend="pyav" + ) + + assert frames.ndim == 4 + assert frames.shape[3] == 3 # RGB + assert frames.shape[0] > 0 + assert frames.shape[0] == len(metadata["frames_indices"]) + assert metadata["video_backend"] == "pyav_dynamic" + + @pytest.mark.parametrize( - "backend, kwargs, expected_num_frames", + "loader_key, kwargs, expected_num_frames", [ - # opencv: num_frames directly controls count - pytest.param("opencv", {"num_frames": 32}, 32, id="opencv-num_frames"), - pytest.param("opencv", {"fps": 2}, 120, id="opencv-fps"), + # uniform sampling + opencv codec pytest.param( "opencv", - {"num_frames": 500, "fps": 2}, + {"num_frames": 32, "backend": "opencv"}, + 32, + id="opencv-num_frames", + ), + pytest.param("opencv", {"fps": 2, "backend": "opencv"}, 120, id="opencv-fps"), + pytest.param( + "opencv", + {"num_frames": 500, "fps": 2, "backend": "opencv"}, 120, id="opencv-num_frames_wins_fps", ), + # dynamic sampling + opencv codec pytest.param( "opencv_dynamic", - {"fps": 1, "max_duration": 60}, + {"fps": 1, "max_duration": 60, "backend": "opencv"}, 60, id="opencv_dynamic-within_max_duration", ), pytest.param( "opencv_dynamic", - {"fps": 2, "max_duration": 30}, + {"fps": 2, "max_duration": 30, "backend": "opencv"}, 60, id="opencv_dynamic-exceeds_max_duration", ), @@ -349,18 +409,45 @@ def dummy_video_path(tmp_path): 119, id="molmo2-fps", ), + # uniform sampling + pyav codec (same frame counts as opencv) + pytest.param( + "opencv", + {"num_frames": 32, "backend": "pyav"}, + 32, + id="pyav-num_frames", + ), + pytest.param("opencv", {"fps": 2, "backend": "pyav"}, 120, id="pyav-fps"), + pytest.param( + "opencv", + {"num_frames": 500, "fps": 2, "backend": "pyav"}, + 120, + id="pyav-num_frames_wins_fps", + ), + # dynamic sampling + pyav codec + pytest.param( + "opencv_dynamic", + {"fps": 1, "max_duration": 60, "backend": "pyav"}, + 60, + id="pyav_dynamic-within_max_duration", + ), + pytest.param( + "opencv_dynamic", + {"fps": 2, "max_duration": 30, "backend": "pyav"}, + 60, + id="pyav_dynamic-exceeds_max_duration", + ), ], ) def test_video_loader_frames_sampling( dummy_video_path, monkeypatch: pytest.MonkeyPatch, - backend: str, + loader_key: str, kwargs: dict, expected_num_frames: int, ): """Test video loader frames sampling functionality.""" - monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", backend) - loader = VIDEO_LOADER_REGISTRY.load(backend) + monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", loader_key) + loader = VIDEO_LOADER_REGISTRY.load(loader_key) with open(dummy_video_path, "rb") as f: long_video_bytes = f.read() diff --git a/vllm/envs.py b/vllm/envs.py index 71566f4c2d4..faafe93ac5f 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -829,9 +829,10 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MAX_AUDIO_CLIP_FILESIZE_MB": lambda: int( os.getenv("VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "25") ), - # Backend for Video IO - # - "opencv": Default backend that uses OpenCV stream buffered backend. - # - "identity": Returns raw video bytes for model processor to handle. + # Backend for Video IO — selects the frame-sampling algorithm. + # - "opencv": uniform sampling. + # - "opencv_dynamic": duration-aware dynamic sampling. + # - "identity": returns raw video bytes for model processor to handle. # # Custom backend implementations can be registered # via `@VIDEO_LOADER_REGISTRY.register("my_custom_video_loader")` and diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 90102151423..5b118af8fc5 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -3,7 +3,7 @@ import math from abc import abstractmethod from io import BytesIO -from typing import Any, NamedTuple, cast +from typing import Any, ClassVar, Literal, NamedTuple, cast import numpy as np import numpy.typing as npt @@ -19,6 +19,11 @@ except ImportError: cv2 = PlaceholderModule("cv2") vr = PlaceholderModule("cv2").placeholder_attr("videoio_registry") +try: + import av +except ImportError: + av = PlaceholderModule("av") # type: ignore[assignment] + logger = init_logger(__name__) @@ -355,8 +360,75 @@ class OpenCVVideoBackendMixin: return frames, valid_frame_indices +class PyAVVideoBackendMixin: + """PyAV (in-process FFmpeg bindings) codec utilities. + + Reads stream metadata and decodes target frames via per-frame + ``container.seek()``. The seek releases the GIL between frames and + scales with the number of sampled frames rather than the video + length, enabling concurrent decoding under serving load. + """ + + @staticmethod + def get_metadata( + container: "av.container.InputContainer", + ) -> VideoSourceMetadata: + if not container.streams.video: + raise ValueError("No video streams found in container") + stream = container.streams.video[0] + total_frames = stream.frames or 0 + fps = float(stream.average_rate) if stream.average_rate else 0.0 + duration = float(stream.duration * stream.time_base) if stream.duration else 0.0 + if total_frames == 0 and duration > 0 and fps > 0: + total_frames = int(duration * fps) + return VideoSourceMetadata(total_frames, fps, duration) + + @staticmethod + def decode_frames( + container: "av.container.InputContainer", + frame_indices: list[int], + fps: float, + duration: float, + ) -> tuple[npt.NDArray, list[int]]: + """Decode target frames via per-frame seek + keyframe decode.""" + stream = container.streams.video[0] + # SLICE parallelizes within a single frame without the + # one-frame-per-thread latency penalty of FRAME threading. + stream.thread_type = "SLICE" + time_base = stream.time_base + + frames_list: list[npt.NDArray] = [] + valid_indices: list[int] = [] + frame_interval = 1.0 / fps if fps > 0 else 0.1 + max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf") + + for idx in frame_indices: + ts = min(idx / fps, max_ts) if fps > 0 else 0.0 + pts = int(ts / time_base) + container.seek(pts, stream=stream) + frame = next(container.decode(video=0), None) + if frame is not None: + frames_list.append(frame.to_ndarray(format="rgb24")) + valid_indices.append(idx) + + if not frames_list: + return np.empty((0,), dtype=np.uint8), valid_indices + return np.stack(frames_list), valid_indices + + @VIDEO_LOADER_REGISTRY.register("opencv") -class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin): +class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): + """Uniform-sampling video backend. + + Samples ``num_frames`` uniformly across the video (or one frame every + ``1/fps`` seconds, whichever produces fewer frames). The decoding codec + is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``), + which can be passed through ``--media-io-kwargs``. Defaults to + ``"pyav"`` for concurrent decoding. + """ + + _sampling_suffix: ClassVar[str] = "" + @classmethod def compute_frames_index_to_sample( cls, @@ -366,7 +438,6 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin): ) -> list[int]: total_frames_num = source.total_frames_num duration = source.duration - num_frames = target.num_frames fps = target.fps # resample video to target num_frames and fps @@ -376,16 +447,18 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin): num_frames_to_sample = min(num_frames, total_frames_num) if fps > 0: num_frames_to_sample = min(num_frames_to_sample, math.floor(duration * fps)) - num_frames_to_sample = max(1, num_frames_to_sample) # at least one sample + num_frames_to_sample = max(1, num_frames_to_sample) if num_frames_to_sample == total_frames_num: - frame_idx = list(range(0, num_frames_to_sample)) - else: - uniform_sampled_frames = np.linspace( - 0, total_frames_num - 1, num_frames_to_sample, dtype=int - ) - frame_idx = uniform_sampled_frames.tolist() - return frame_idx + return list(range(num_frames_to_sample)) + return np.linspace( + 0, total_frames_num - 1, num_frames_to_sample, dtype=int + ).tolist() + + @classmethod + def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: + """Sampling-algorithm-specific metadata adjustment hook.""" + return source @classmethod def load_bytes( @@ -395,55 +468,101 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin): fps: int = -1, max_duration: int = 300, frame_recovery: bool = False, + *, + backend: Literal["opencv", "pyav"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: - """ - Load video frames from bytes. + """Load sampled frames from raw video bytes. Args: - data: Raw video bytes - num_frames: Target number of frames to sample (-1 for all) - fps: Target FPS for sampling (-1 for original) - max_duration: Maximum duration (unused in base backend) - frame_recovery: Enable forward-scan recovery for failed frames + data: Raw video bytes. + num_frames: Target number of frames to sample (``-1`` for all). + fps: Target FPS for sampling (``-1`` for original). + max_duration: Maximum duration in seconds — only used by the + dynamic subclass; ignored here. + frame_recovery: Enable forward-scan recovery for failed frames. + Only honored by the OpenCV codec. + backend: Decoding codec — ``"opencv"`` or ``"pyav"`` . Returns: - Tuple of (frames_array, metadata_dict) + Tuple of ``(frames_array, metadata_dict)``. """ - cap = cls.open_video_capture(data) - - source = OpenCVVideoBackendMixin.get_video_metadata(cap) target = VideoTargetMetadata( - num_frames=num_frames, - fps=fps, - max_duration=max_duration, + num_frames=num_frames, fps=fps, max_duration=max_duration ) - # resample video to target num_frames and fps - # - the minimum of the two will be used - frame_idx = cls.compute_frames_index_to_sample( + if backend == "opencv": + cap = cls.open_video_capture(data) + source = cls._prepare_source(cls.get_video_metadata(cap)) + frame_idx = cls.compute_frames_index_to_sample( + source=source, target=target, **kwargs + ) + frames, valid = cls.read_frames( + cap, + frame_idx, + total_frames_num=source.total_frames_num, + frame_recovery=frame_recovery, + ) + elif backend == "pyav": + assert not frame_recovery, ( + "frame_recovery is only available for `opencv` backend" + ) + with av.open(BytesIO(data)) as container: + source = cls._prepare_source(cls.get_metadata(container)) + frame_idx = cls.compute_frames_index_to_sample( + source=source, target=target, **kwargs + ) + frames, valid = cls.decode_frames( + container, frame_idx, source.original_fps, source.duration + ) + else: + raise ValueError( + f"Unknown video codec backend {backend!r}; " + "valid options: 'opencv', 'pyav'." + ) + + if len(valid) < len(frame_idx): + logger.warning( + "%s video loading: expected %d frames but got %d.", + backend, + len(frame_idx), + len(valid), + ) + + return frames, cls.create_hf_metadata( source=source, - target=target, + video_backend=f"{backend}{cls._sampling_suffix}", + valid_frame_indices=valid, ) - frames, valid_frame_indices = cls.read_frames( - cap, - frame_idx, - total_frames_num=source.total_frames_num, - frame_recovery=frame_recovery, - ) - - metadata = cls.create_hf_metadata( - source=source, - video_backend="opencv", - valid_frame_indices=valid_frame_indices, - ) - - return frames, metadata - @VIDEO_LOADER_REGISTRY.register("opencv_dynamic") -class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin): +class DynamicVideoBackend(VideoBackend): + """Duration-aware dynamic-sampling video backend. + + Samples at ``fps`` up to ``max_duration`` seconds, falling back to + uniform sampling across the full duration when the video is longer + than ``max_duration``. Codec is selectable the same way as + :class:`VideoBackend`. + """ + + _sampling_suffix: ClassVar[str] = "_dynamic" + + @classmethod + def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: + # Estimate duration from frame count and fps when the container + # does not report it (common for WebM/streaming inputs). + if source.duration: + return source + if source.original_fps > 0: + max_frame_idx = source.total_frames_num - 1 + duration = round(max_frame_idx / source.original_fps) + 1 + else: + duration = 0 + return VideoSourceMetadata( + source.total_frames_num, source.original_fps, duration + ) + @classmethod def compute_frames_index_to_sample( cls, @@ -456,8 +575,8 @@ class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin): original_fps = source.original_fps max_duration = target.max_duration fps = target.fps - max_frame_idx = source.total_frames_num - 1 + # Refer to: # https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140 frame_indices_list: list[int] @@ -491,62 +610,20 @@ class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin): fps: int = 2, max_duration: int = 300, frame_recovery: bool = False, + *, + backend: Literal["opencv", "pyav"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: - """ - Load video frames with dynamic sampling based on duration. - - Args: - data: Raw video bytes - num_frames: Not used in dynamic backend - fps: Target FPS for sampling (default: 2) - max_duration: Maximum video duration to process (default: 300s) - frame_recovery: Enable forward-scan recovery for failed frames - - Returns: - Tuple of (frames_array, metadata_dict) - """ - cap = cls.open_video_capture(data) - - orig_source = OpenCVVideoBackendMixin.get_video_metadata(cap) - max_frame_idx = orig_source.total_frames_num - 1 - duration = ( - orig_source.duration or round(max_frame_idx / orig_source.original_fps) + 1 - ) - - # recompute source metadata with adjusted duration to ensure correct - # sampling indices computation - source = VideoSourceMetadata( - total_frames_num=orig_source.total_frames_num, - original_fps=orig_source.original_fps, - duration=duration, - ) - target = VideoTargetMetadata( + return super().load_bytes( + data, num_frames=num_frames, fps=fps, max_duration=max_duration, - ) - - frame_indices_list = cls.compute_frames_index_to_sample( - source=source, - target=target, - ) - - frames, valid_frame_indices = cls.read_frames( - cap, - frame_indices_list, - total_frames_num=source.total_frames_num, frame_recovery=frame_recovery, + backend=backend, + **kwargs, ) - metadata = cls.create_hf_metadata( - source=source, - video_backend="opencv_dynamic", - valid_frame_indices=valid_frame_indices, - ) - - return frames, metadata - @VIDEO_LOADER_REGISTRY.register("molmo2") class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin): @@ -835,7 +912,7 @@ class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin): @VIDEO_LOADER_REGISTRY.register("nemotron_vl") -class NemotronVLVideoBackend(OpenCVVideoBackend): +class NemotronVLVideoBackend(VideoBackend): @classmethod def load_bytes( cls, @@ -844,14 +921,17 @@ class NemotronVLVideoBackend(OpenCVVideoBackend): fps: int = -1, max_duration: int = 300, frame_recovery: bool = False, + *, + backend: Literal["opencv", "pyav"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: - frames, metadata = OpenCVVideoBackend.load_bytes( + frames, metadata = super().load_bytes( data, num_frames=num_frames, fps=fps, max_duration=max_duration, frame_recovery=frame_recovery, + backend=backend, **kwargs, ) From 3951d3eacde3a3addb81e09f7569972f6d6150cb Mon Sep 17 00:00:00 2001 From: Martin Hickey Date: Wed, 22 Apr 2026 04:15:02 +0100 Subject: [PATCH 022/153] [MyPy] Enable mypy for `vllm/model_executor/layers/` (#40159) Signed-off-by: Martin Hickey --- tools/pre_commit/mypy.py | 1 - vllm/model_executor/layers/activation.py | 27 ++++---- .../layers/attention/attention.py | 21 ++++-- .../attention/chunked_local_attention.py | 2 +- .../layers/attention/cross_attention.py | 13 ++-- .../attention/encoder_only_attention.py | 4 +- .../layers/attention/mla_attention.py | 27 +++++--- .../layers/fused_moe/all2all_utils.py | 11 ++-- .../model_executor/layers/fused_moe/config.py | 3 +- .../experts/batched_deep_gemm_moe.py | 6 +- .../layers/fused_moe/oracle/mxfp4.py | 13 ++-- .../flashinfer_nvlink_one_sided.py | 24 +++++-- .../flashinfer_nvlink_two_sided.py | 25 ++++--- .../fused_moe/prepare_finalize/naive_dp_ep.py | 4 ++ .../fused_moe/runner/default_moe_runner.py | 4 +- vllm/model_executor/layers/kda.py | 43 +++++++----- vllm/model_executor/layers/layernorm.py | 2 +- vllm/model_executor/layers/mamba/abstract.py | 3 +- .../layers/mamba/gdn_linear_attn.py | 66 ++++++++++--------- .../layers/mamba/linear_attn.py | 9 +-- .../layers/mamba/mamba_mixer.py | 13 ++-- .../layers/mamba/mamba_mixer2.py | 21 ++++-- .../model_executor/layers/mamba/short_conv.py | 9 +-- .../layers/pooler/seqwise/poolers.py | 1 + .../layers/pooler/tokwise/poolers.py | 1 + .../layers/quantization/fp_quant.py | 8 ++- .../layers/quantization/quark/quark.py | 2 +- .../layers/sparse_attn_indexer.py | 26 ++++---- 28 files changed, 243 insertions(+), 146 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 41c05efd201..7c7b0ada60d 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -29,7 +29,6 @@ SEPARATE_GROUPS = [ "tests", # v0 related "vllm/lora", - "vllm/model_executor/layers", ] # TODO(woosuk): Include the code from Megatron and HuggingFace. diff --git a/vllm/model_executor/layers/activation.py b/vllm/model_executor/layers/activation.py index 26a771cb750..e2b70b771a1 100644 --- a/vllm/model_executor/layers/activation.py +++ b/vllm/model_executor/layers/activation.py @@ -666,16 +666,7 @@ _ACTIVATION_REGISTRY = LazyDict( "gelu": lambda: GELU(), "gelu_fast": lambda: FastGELU(), "gelu_new": lambda: NewGELU(), - "gelu_pytorch_tanh": lambda: ( - # TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile - logger.warning_once( - "[ROCm] PyTorch's native GELU with tanh approximation is unstable. " - "Falling back to GELU(approximate='none')." - ), - nn.GELU(approximate="none"), - )[1] - if current_platform.is_rocm() - else nn.GELU(approximate="tanh"), + "gelu_pytorch_tanh": lambda: _get_gelu_pytorch_tanh(), "relu": lambda: nn.ReLU(), "relu2": lambda: ReLUSquaredActivation(), "silu": lambda: nn.SiLU(), @@ -687,6 +678,18 @@ _ACTIVATION_REGISTRY = LazyDict( ) +def _get_gelu_pytorch_tanh() -> nn.Module: + """Get PyTorch GELU with tanh approximation, with ROCm fallback.""" + if current_platform.is_rocm(): + # TODO:[ROCm] PyTorch native GELU with tanh is unstable with torch.compile + logger.warning_once( + "[ROCm] PyTorch's native GELU with tanh approximation is unstable. " + "Falling back to GELU(approximate='none')." + ) + return nn.GELU(approximate="none") + return nn.GELU(approximate="tanh") + + def get_act_fn(act_fn_name: str) -> nn.Module: """Get an activation function by name.""" act_fn_name = act_fn_name.lower() @@ -703,12 +706,12 @@ def get_act_fn(act_fn_name: str) -> nn.Module: return _ACTIVATION_REGISTRY[act_fn_name] -_ACTIVATION_AND_MUL_REGISTRY = LazyDict( +_ACTIVATION_AND_MUL_REGISTRY: LazyDict[nn.Module] = LazyDict( { "gelu": lambda: GeluAndMul(), "silu": lambda: SiluAndMul(), "geglu": lambda: GeluAndMul(), - "swigluoai": lambda *args, **kwargs: SwigluOAIAndMul(*args, **kwargs), + "swigluoai": lambda: SwigluOAIAndMul(), } ) diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index 54f0e1ce5fe..9d2e29d02de 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -33,6 +33,7 @@ from vllm.utils.torch_utils import ( ) from vllm.v1.attention.backend import ( AttentionBackend, + AttentionMetadata, AttentionType, ) from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -209,6 +210,7 @@ class Attention(nn.Module, AttentionLayerBase): `self.kv_cache`. """ super().__init__() + sliding_window: int | None if per_layer_sliding_window is not None: # per-layer sliding window sliding_window = per_layer_sliding_window @@ -335,7 +337,7 @@ class Attention(nn.Module, AttentionLayerBase): cache_config.enable_prefix_caching = False impl_cls = self.attn_backend.get_impl_cls() - self.impl = impl_cls( + self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass num_heads, head_size, scale, @@ -576,7 +578,7 @@ class Attention(nn.Module, AttentionLayerBase): def get_attn_backend(self) -> type[AttentionBackend]: return self.attn_backend - def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: # Block size may get updated after model loading, refresh it block_size = vllm_config.cache_config.block_size # Should not be called for enc-dec or encoder-only attention. @@ -680,9 +682,16 @@ def get_attention_context( extracted from the forward context. """ forward_context: ForwardContext = get_forward_context() - attn_metadata = forward_context.attn_metadata - if isinstance(attn_metadata, dict): - attn_metadata = attn_metadata[layer_name] + attn_metadata_raw = forward_context.attn_metadata + attn_metadata: AttentionMetadata + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw[layer_name] + elif isinstance(attn_metadata_raw, list): + # list[dict[str, AttentionMetadata]]: used in speculative decoding + # where [0] is the base-model (non-speculative) metadata dict. + attn_metadata = attn_metadata_raw[0][layer_name] + else: + attn_metadata = attn_metadata_raw attn_layer: Attention | MLAAttention = forward_context.no_compile_layers[layer_name] kv_cache = attn_layer.kv_cache slot_mapping = forward_context.slot_mapping @@ -708,7 +717,7 @@ def unified_kv_cache_update( assert hasattr(attn_layer.impl, "do_kv_cache_update"), ( f"{attn_layer.impl.__class__.__name__} does not support kv cache update" ) - attn_layer.impl.do_kv_cache_update( + attn_layer.impl.do_kv_cache_update( # type: ignore[attr-defined] attn_layer, key, value, diff --git a/vllm/model_executor/layers/attention/chunked_local_attention.py b/vllm/model_executor/layers/attention/chunked_local_attention.py index 136574d9752..cb595438ade 100644 --- a/vllm/model_executor/layers/attention/chunked_local_attention.py +++ b/vllm/model_executor/layers/attention/chunked_local_attention.py @@ -29,7 +29,7 @@ from vllm.v1.kv_cache_interface import ( @functools.lru_cache def create_chunked_local_attention_backend( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], attention_chunk_size: int, ) -> type[AttentionBackend]: prefix = f"ChunkedLocalAttention_{attention_chunk_size}_" diff --git a/vllm/model_executor/layers/attention/cross_attention.py b/vllm/model_executor/layers/attention/cross_attention.py index 61699832a62..312f906abac 100644 --- a/vllm/model_executor/layers/attention/cross_attention.py +++ b/vllm/model_executor/layers/attention/cross_attention.py @@ -72,7 +72,7 @@ def _get_cross_slot_mapping( @functools.lru_cache def create_cross_attention_backend( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], ) -> type[AttentionBackend]: prefix = "CrossAttention_" underlying_builder = underlying_attn_backend.get_builder_cls() @@ -87,6 +87,7 @@ def create_cross_attention_backend( ) -> AttentionMetadata: new_metadata = copy(common_attn_metadata) new_metadata.causal = False + assert new_metadata.encoder_seq_lens_cpu is not None max_encoder_len = int(new_metadata.encoder_seq_lens_cpu.max()) new_metadata.max_seq_len = max_encoder_len # Any computed tokens indicated decode step>1 (no chunked prefill) @@ -118,7 +119,7 @@ def create_cross_attention_backend( self.device, ) attn_metadata = super().build(common_prefix_len, new_metadata, fast_build) - attn_metadata.slot_mapping = slot_mapping + attn_metadata.slot_mapping = slot_mapping # type: ignore[attr-defined] return attn_metadata # NOTE(Lucas): we need a custom impl so we can use the slot-mapping computed by @@ -144,8 +145,12 @@ def create_cross_attention_backend( and key is not None and value is not None ): - self.do_kv_cache_update( - layer, key, value, kv_cache, attn_metadata.slot_mapping + self.do_kv_cache_update( # type: ignore[attr-defined] + layer, + key, + value, + kv_cache, + attn_metadata.slot_mapping, # type: ignore[attr-defined] ) return super().forward( diff --git a/vllm/model_executor/layers/attention/encoder_only_attention.py b/vllm/model_executor/layers/attention/encoder_only_attention.py index 0897ee45b84..5805fe2ae1c 100644 --- a/vllm/model_executor/layers/attention/encoder_only_attention.py +++ b/vllm/model_executor/layers/attention/encoder_only_attention.py @@ -21,7 +21,7 @@ from vllm.v1.kv_cache_interface import KVCacheSpec @functools.lru_cache def create_encoder_only_attention_backend( - underlying_attn_backend: AttentionBackend, + underlying_attn_backend: type[AttentionBackend], ) -> type[AttentionBackend]: prefix = "EncoderOnlyAttention_" underlying_builder = underlying_attn_backend.get_builder_cls() @@ -93,6 +93,6 @@ class EncoderOnlyAttention(Attention): **kwargs, ) - def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec: + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: # Does not need KV cache return None diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index cbbf5f3c3ca..a1e3921b0c5 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -389,7 +389,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): cache_config.enable_prefix_caching = False impl_cls = cast(type[MLAAttentionImpl], self.attn_backend.get_impl_cls()) - self.impl = impl_cls( + self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an MLAAttentionImpl subclass num_heads=self.num_heads, head_size=self.head_size, scale=self.scale, @@ -485,16 +485,23 @@ class MLAAttention(nn.Module, AttentionLayerBase): if self.use_direct_call: forward_context: ForwardContext = get_forward_context() - attn_metadata = forward_context.attn_metadata - if isinstance(attn_metadata, dict): - attn_metadata = attn_metadata[self.layer_name] + attn_metadata_raw = forward_context.attn_metadata + attn_metadata: MLACommonMetadata + if isinstance(attn_metadata_raw, dict): + attn_metadata = attn_metadata_raw[self.layer_name] # type: ignore[assignment] + elif isinstance(attn_metadata_raw, list): + # list[dict[str, AttentionMetadata]]: used in speculative decoding + # where [0] is the base-model (non-speculative) metadata dict. + attn_metadata = attn_metadata_raw[0][self.layer_name] # type: ignore[assignment] + else: + attn_metadata = attn_metadata_raw self_kv_cache = self.kv_cache slot_mapping = forward_context.slot_mapping assert isinstance(slot_mapping, dict), ( f"Expected slot_mapping to be a dict, got {type(slot_mapping)}. " ) - self.impl.do_kv_cache_update( + self.impl.do_kv_cache_update( # type: ignore[attr-defined] kv_c_normed, k_pe, self_kv_cache, @@ -612,7 +619,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): num_mha_tokens = q.size(0) - num_mqa_tokens if num_mha_tokens > 0: - self.impl.forward_mha( + self.impl.forward_mha( # type: ignore[attr-defined] q[num_mqa_tokens:], k_c_normed[num_mqa_tokens:], k_pe[num_mqa_tokens:], @@ -695,7 +702,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): # call decode attn if not is_sparse_impl: assert attn_metadata.decode is not None - attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) + attn_out, lse = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) # type: ignore[attr-defined] # correct dcp attn_out with lse. if self.impl.dcp_world_size > 1: @@ -1053,9 +1060,9 @@ except ImportError: "AITER_MLA backends use aiter kernels instead." ) elif current_platform.is_xpu(): - from vllm._xpu_ops import xpu_ops as ops + from vllm._xpu_ops import xpu_ops - flash_attn_varlen_func = ops.flash_attn_varlen_func # type: ignore[no-redef] + flash_attn_varlen_func = xpu_ops.flash_attn_varlen_func # type: ignore[no-redef,attr-defined,assignment] def dynamic_per_batched_tensor_quant( @@ -1988,7 +1995,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): assert isinstance(attn_metadata.prefill, FlashInferPrefillMetadata) self._build_fi_prefill_wrappers(attn_metadata.prefill) - return attn_metadata + return attn_metadata # type: ignore[return-value] def reorg_kvcache( diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index 62b2602928f..e8034113983 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -117,17 +117,20 @@ def maybe_make_prepare_finalize( "Detected DP deployment with no --enable-expert-parallel. " "Falling back to AllGather+ReduceScatter dispatch/combine." ) + device_communicator = get_ep_group().device_communicator + assert device_communicator is not None + assert device_communicator.all2all_manager is not None return make_moe_prepare_and_finalize_naive_dp_ep( is_sequence_parallel=moe.moe_parallel_config.is_sequence_parallel, - num_dispatchers=( - get_ep_group().device_communicator.all2all_manager.world_size - ), + num_dispatchers=(device_communicator.all2all_manager.world_size), use_monolithic=use_monolithic, ) else: return make_moe_prepare_and_finalize_no_dp_ep(use_monolithic) - all2all_manager = get_ep_group().device_communicator.all2all_manager + device_communicator = get_ep_group().device_communicator + assert device_communicator is not None + all2all_manager = device_communicator.all2all_manager assert all2all_manager is not None prepare_finalize: FusedMoEPrepareAndFinalize | None = None diff --git a/vllm/model_executor/layers/fused_moe/config.py b/vllm/model_executor/layers/fused_moe/config.py index 00d7d8e7890..e3a91966777 100644 --- a/vllm/model_executor/layers/fused_moe/config.py +++ b/vllm/model_executor/layers/fused_moe/config.py @@ -7,6 +7,7 @@ from typing import Union import torch from vllm.config import ParallelConfig, SchedulerConfig +from vllm.config.kernel import MoEBackend from vllm.distributed import get_dp_group, get_pcp_group, get_tensor_model_parallel_rank from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.activation import MoEActivation @@ -1192,7 +1193,7 @@ class FusedMoEConfig: # Defaults to intermediate_size_per_partition if not specified. intermediate_size_per_partition_unpadded: int | None = None - moe_backend: str = "auto" + moe_backend: MoEBackend = "auto" max_num_tokens: int = SchedulerConfig.DEFAULT_MAX_NUM_BATCHED_TOKENS_FOR_BATCHED_DP has_bias: bool = False is_act_and_mul: bool = True diff --git a/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py index 2cb0bd7649f..fad39b3e9d4 100644 --- a/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py @@ -210,9 +210,9 @@ def persistent_masked_m_silu_mul_quant( DeepGemmQuantScaleFMT.UE8M0, ] - cuda_arch = current_platform.get_device_capability( - device_id=y.device.index - ).to_int() + device_capability = current_platform.get_device_capability(device_id=y.device.index) + assert device_capability is not None + cuda_arch = device_capability.to_int() if current_platform.is_cuda() and cuda_arch >= 80: torch.ops._C.persistent_masked_m_silu_mul_quant( diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 917d474fc9b..587954d5267 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -7,6 +7,7 @@ import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm import envs +from vllm.config.kernel import MoEBackend from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoEConfig, @@ -146,7 +147,7 @@ def backend_to_kernel_cls( raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}") -def map_mxfp4_backend(runner_backend: str) -> Mxfp4MoeBackend: +def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend: """Map user's moe_backend string to Mxfp4MoeBackend.""" mapping: dict[str, Mxfp4MoeBackend] = { "flashinfer_trtllm": Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16, @@ -201,10 +202,12 @@ def select_gpt_oss_mxfp4_moe_backend( Select the primary MXFP4 MoE backend. Note: Shape-specific fallbacks may still occur at runtime. """ - triton_kernels_supported = has_triton_kernels() and ( - 9, - 0, - ) <= current_platform.get_device_capability() < (11, 0) + device_capability = current_platform.get_device_capability() + triton_kernels_supported = ( + has_triton_kernels() + and device_capability is not None + and (9, 0) <= device_capability < (11, 0) + ) # LoRA: separate experts backend path if config.is_lora_enabled: diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py index bdde3da6b3a..a04ff3b8b68 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_one_sided.py @@ -4,6 +4,9 @@ import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.distributed import get_ep_group +from vllm.distributed.device_communicators.base_device_communicator import ( + All2AllManagerBase, +) from vllm.forward_context import get_forward_context from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input @@ -11,12 +14,16 @@ from vllm.utils.flashinfer import nvfp4_block_scale_interleave def get_local_sizes(): - return get_forward_context().dp_metadata.get_chunk_sizes_across_dp_rank() + dp_metadata = get_forward_context().dp_metadata + assert dp_metadata is not None + return dp_metadata.get_chunk_sizes_across_dp_rank() class FlashInferNVLinkOneSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): """FlashInfer implementation using the Moe AlltoAll kernel.""" + all2all_manager: All2AllManagerBase + def __init__( self, max_num_tokens: int, @@ -32,8 +39,12 @@ class FlashInferNVLinkOneSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeMo self.hidden_size = hidden_size self.num_dispatchers_ = num_dispatchers - self.all2all_manager = get_ep_group().device_communicator.all2all_manager - self.all2all_manager.initialize( + device_communicator = get_ep_group().device_communicator + assert device_communicator is not None + all2all_manager = device_communicator.all2all_manager + assert all2all_manager is not None + self.all2all_manager = all2all_manager + self.all2all_manager.initialize( # type: ignore[attr-defined] max_num_tokens=self.max_num_tokens, top_k=self.top_k, num_experts=self.num_experts, @@ -97,7 +108,8 @@ class FlashInferNVLinkOneSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeMo payloads.append(topk_ids) payloads.append(topk_weights) - recv_payloads = self.all2all_manager.moe_alltoall.dispatch( + assert self.all2all_manager.moe_alltoall is not None # type: ignore[attr-defined] + recv_payloads = self.all2all_manager.moe_alltoall.dispatch( # type: ignore[attr-defined] token_selected_experts=topk_ids, input_payloads=payloads, runtime_max_tokens_per_rank=self.runtime_max_tokens_per_rank, @@ -131,7 +143,7 @@ class FlashInferNVLinkOneSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeMo apply_router_weight_on_input: bool, weight_and_reduce_impl: mk.TopKWeightAndReduce, ) -> None: - assert self.all2all_manager.moe_alltoall is not None + assert self.all2all_manager.moe_alltoall is not None # type: ignore[attr-defined] ep_size = self.all2all_manager.world_size hidden_size = fused_expert_output.shape[-1] @@ -139,7 +151,7 @@ class FlashInferNVLinkOneSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeMo ep_size, self.runtime_max_tokens_per_rank, hidden_size ) - combined_output = self.all2all_manager.moe_alltoall.combine( + combined_output = self.all2all_manager.moe_alltoall.combine( # type: ignore[attr-defined] payload=fused_expert_output, runtime_max_tokens_per_rank=self.runtime_max_tokens_per_rank, ) diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py index be63bd4e3f6..47fe293d511 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/flashinfer_nvlink_two_sided.py @@ -15,19 +15,26 @@ from vllm.utils.flashinfer import nvfp4_block_scale_interleave def get_local_sizes(): - return get_forward_context().dp_metadata.get_chunk_sizes_across_dp_rank() + dp_metadata = get_forward_context().dp_metadata + assert dp_metadata is not None + return dp_metadata.get_chunk_sizes_across_dp_rank() class FlashInferNVLinkTwoSidedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): """Base class for FlashInfer MoE prepare and finalize operations.""" + all2all_manager: All2AllManagerBase + def __init__( self, num_dispatchers: int = 1, ): super().__init__() self.num_dispatchers_ = num_dispatchers - self.all2all_manager = get_ep_group().device_communicator.all2all_manager + device_communicator = get_ep_group().device_communicator + assert device_communicator is not None + assert device_communicator.all2all_manager is not None + self.all2all_manager = device_communicator.all2all_manager @property def activation_format(self) -> mk.FusedMoEActivationFormat: @@ -129,7 +136,7 @@ def flashinfer_alltoall_dispatch( ): from flashinfer.comm.trtllm_alltoall import MnnvlMoe - assert all2all_manager.ensure_alltoall_workspace_initialized(), ( + assert all2all_manager.ensure_alltoall_workspace_initialized(), ( # type: ignore[attr-defined] "FlashInfer AllToAll workspace not available" ) @@ -144,7 +151,7 @@ def flashinfer_alltoall_dispatch( topk_ids, topk_weights, None, - all2all_manager.prepare_workspace_tensor, + all2all_manager.prepare_workspace_tensor, # type: ignore[attr-defined] max_num_token, ep_rank, ep_size, @@ -172,7 +179,7 @@ def flashinfer_alltoall_dispatch( x = MnnvlMoe.mnnvl_moe_alltoallv( x, alltoall_info, - all2all_manager.workspace_tensor, + all2all_manager.workspace_tensor, # type: ignore[attr-defined] ep_rank, ep_size, ) @@ -180,7 +187,7 @@ def flashinfer_alltoall_dispatch( x_sf = MnnvlMoe.mnnvl_moe_alltoallv( x_sf, alltoall_info, - all2all_manager.workspace_tensor, + all2all_manager.workspace_tensor, # type: ignore[attr-defined] ep_rank, ep_size, ) @@ -196,7 +203,7 @@ def flashinfer_alltoall_dispatch( x = MnnvlMoe.mnnvl_moe_alltoallv( x, alltoall_info, - all2all_manager.workspace_tensor, + all2all_manager.workspace_tensor, # type: ignore[attr-defined] ep_rank, ep_size, ) @@ -212,13 +219,13 @@ def flashinfer_alltoall_combine( ): from flashinfer.comm.trtllm_alltoall import MnnvlMoe - assert all2all_manager.ensure_alltoall_workspace_initialized(), ( + assert all2all_manager.ensure_alltoall_workspace_initialized(), ( # type: ignore[attr-defined] "FlashInfer AllToAll workspace not available" ) return MnnvlMoe.mnnvl_moe_alltoallv_combine( output, alltoall_info, - all2all_manager.workspace_tensor, + all2all_manager.workspace_tensor, # type: ignore[attr-defined] ep_rank=all2all_manager.rank, ep_size=all2all_manager.world_size, top_k=top_k, diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/naive_dp_ep.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/naive_dp_ep.py index 6dc9f695804..2b21e2db9f6 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/naive_dp_ep.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/naive_dp_ep.py @@ -132,9 +132,11 @@ class MoEPrepareAndFinalizeNaiveDPEPModular(mk.FusedMoEPrepareAndFinalizeModular ) if scales is None: + assert len(res) == 3 a1q, topk_weights, topk_ids = res a1q_scale = None else: + assert len(res) == 4 a1q, topk_weights, topk_ids, scales = res a1q_scale = _unwrap_scale_and_prepare_for_moe(scales, quant_config) @@ -217,9 +219,11 @@ class MoEPrepareAndFinalizeNaiveDPEPMonolithic(mk.FusedMoEPrepareAndFinalizeMono ) if scales is None: + assert len(res) == 2 a1q, router_logits = res a1q_scale = None else: + assert len(res) == 3 a1q, router_logits, scales = res a1q_scale = _unwrap_scale_and_prepare_for_moe(scales, quant_config) diff --git a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py index 8cd2fc65704..df4c0c86924 100644 --- a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py @@ -54,11 +54,13 @@ class DefaultMoERunner(MoERunnerBase): # NOTE: this will be removed once all kernels are migrated into the # MoEKernel framework. if self.do_naive_dispatch_combine: - hidden_states, router_logits = get_ep_group().dispatch_router_logits( + res = get_ep_group().dispatch_router_logits( hidden_states, router_logits, self.moe_config.is_sequence_parallel, ) + assert len(res) == 2 + hidden_states, router_logits = res # NOTE: Similar with DP, PCP also needs dispatch and combine. For # simplicity, AgRsAll2All was added separately for PCP here. Maybe diff --git a/vllm/model_executor/layers/kda.py b/vllm/model_executor/layers/kda.py index b09f980c7e6..70c67f33f0a 100644 --- a/vllm/model_executor/layers/kda.py +++ b/vllm/model_executor/layers/kda.py @@ -16,7 +16,6 @@ from vllm.logger import init_logger from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader from vllm.model_executor.utils import set_weight_attrs from vllm.utils.torch_utils import direct_register_custom_op -from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata from .fla.ops.kda import ( @@ -123,7 +122,7 @@ class KimiDeltaAttention(nn.Module, MambaBase): self.cache_config = cache_config if model_config is None: raise ValueError("model_config must be provided") - kda_config = model_config.linear_attn_config + kda_config = model_config.linear_attn_config # type: ignore[attr-defined] self.head_dim = kda_config["head_dim"] self.num_heads = kda_config["num_heads"] self.layer_idx = layer_idx @@ -297,19 +296,21 @@ class KimiDeltaAttention(nn.Module, MambaBase): core_attn_out: torch.Tensor, ) -> None: forward_context = get_forward_context() - attn_metadata: AttentionMetadata = forward_context.attn_metadata + attn_metadata_raw = forward_context.attn_metadata - if attn_metadata is None: + if attn_metadata_raw is None: # # V1 profile run return - assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] - assert isinstance(attn_metadata, GDNAttentionMetadata) - has_initial_state = attn_metadata.has_initial_state - non_spec_query_start_loc = attn_metadata.non_spec_query_start_loc - non_spec_state_indices_tensor = attn_metadata.non_spec_state_indices_tensor # noqa: E501 - num_actual_tokens = attn_metadata.num_actual_tokens + assert isinstance(attn_metadata_raw, dict) + attn_metadata_narrowed = attn_metadata_raw[self.prefix] + assert isinstance(attn_metadata_narrowed, GDNAttentionMetadata) + has_initial_state = attn_metadata_narrowed.has_initial_state + non_spec_query_start_loc = attn_metadata_narrowed.non_spec_query_start_loc + non_spec_state_indices_tensor = ( + attn_metadata_narrowed.non_spec_state_indices_tensor + ) # noqa: E501 + num_actual_tokens = attn_metadata_narrowed.num_actual_tokens constant_caches = self.kv_cache q_proj_states = q_proj_states[:num_actual_tokens] @@ -335,7 +336,7 @@ class KimiDeltaAttention(nn.Module, MambaBase): v_conv_weights = self.v_conv1d.weight.view( self.v_conv1d.weight.size(0), self.v_conv1d.weight.size(2) ) - if attn_metadata.num_prefills > 0: + if attn_metadata_narrowed.num_prefills > 0: q_proj_states = q_proj_states.transpose(0, 1) k_proj_states = k_proj_states.transpose(0, 1) v_proj_states = v_proj_states.transpose(0, 1) @@ -348,7 +349,7 @@ class KimiDeltaAttention(nn.Module, MambaBase): has_initial_state=has_initial_state, cache_indices=non_spec_state_indices_tensor, query_start_loc=non_spec_query_start_loc, - metadata=attn_metadata, + metadata=attn_metadata_narrowed, ).transpose(0, 1) k = causal_conv1d_fn( k_proj_states, @@ -359,7 +360,7 @@ class KimiDeltaAttention(nn.Module, MambaBase): has_initial_state=has_initial_state, cache_indices=non_spec_state_indices_tensor, query_start_loc=non_spec_query_start_loc, - metadata=attn_metadata, + metadata=attn_metadata_narrowed, ).transpose(0, 1) v = causal_conv1d_fn( v_proj_states, @@ -370,11 +371,12 @@ class KimiDeltaAttention(nn.Module, MambaBase): has_initial_state=has_initial_state, cache_indices=non_spec_state_indices_tensor, query_start_loc=non_spec_query_start_loc, - metadata=attn_metadata, + metadata=attn_metadata_narrowed, ).transpose(0, 1) else: + assert non_spec_state_indices_tensor is not None decode_conv_indices = non_spec_state_indices_tensor[ - : attn_metadata.num_actual_tokens + : attn_metadata_narrowed.num_actual_tokens ] q = causal_conv1d_update( q_proj_states, @@ -408,7 +410,9 @@ class KimiDeltaAttention(nn.Module, MambaBase): lambda x: rearrange(x, "n (h d) -> 1 n h d", d=self.head_dim), (q, k, v) ) - if attn_metadata.num_prefills > 0: + if attn_metadata_narrowed.num_prefills > 0: + assert non_spec_state_indices_tensor is not None + assert has_initial_state is not None zero_idx = non_spec_state_indices_tensor[~has_initial_state] recurrent_state[zero_idx] = 0 initial_state = recurrent_state[non_spec_state_indices_tensor].contiguous() @@ -429,6 +433,7 @@ class KimiDeltaAttention(nn.Module, MambaBase): # Init cache recurrent_state[non_spec_state_indices_tensor] = last_recurrent_state else: + assert non_spec_query_start_loc is not None ( core_attn_out_non_spec, last_recurrent_state, @@ -440,7 +445,9 @@ class KimiDeltaAttention(nn.Module, MambaBase): beta=beta, initial_state=recurrent_state, use_qk_l2norm_in_kernel=True, - cu_seqlens=non_spec_query_start_loc[: attn_metadata.num_decodes + 1], + cu_seqlens=non_spec_query_start_loc[ + : attn_metadata_narrowed.num_decodes + 1 + ], ssm_state_indices=non_spec_state_indices_tensor, ) core_attn_out[0, :num_actual_tokens] = core_attn_out_non_spec[ diff --git a/vllm/model_executor/layers/layernorm.py b/vllm/model_executor/layers/layernorm.py index ac2423ce0e0..d9184bb7707 100644 --- a/vllm/model_executor/layers/layernorm.py +++ b/vllm/model_executor/layers/layernorm.py @@ -76,7 +76,7 @@ def poly_norm( from vllm import _custom_ops as ops out = torch.empty_like(x) - ops.poly_norm( + ops.poly_norm( # type: ignore[attr-defined] out, x, weight, diff --git a/vllm/model_executor/layers/mamba/abstract.py b/vllm/model_executor/layers/mamba/abstract.py index 3c6b0139424..2c05880c0fe 100644 --- a/vllm/model_executor/layers/mamba/abstract.py +++ b/vllm/model_executor/layers/mamba/abstract.py @@ -42,9 +42,10 @@ class MambaBase(AttentionLayerBase): def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: mamba_block_size = vllm_config.cache_config.mamba_block_size + assert mamba_block_size is not None page_size_padded = vllm_config.cache_config.mamba_page_size_padded return MambaSpec( - shapes=self.get_state_shape(), + shapes=tuple(self.get_state_shape()), dtypes=self.get_state_dtype(), block_size=mamba_block_size, page_size_padded=page_size_padded, diff --git a/vllm/model_executor/layers/mamba/gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn_linear_attn.py index 70a4794ad54..c74ca13024a 100644 --- a/vllm/model_executor/layers/mamba/gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn_linear_attn.py @@ -62,7 +62,6 @@ from vllm.utils.torch_utils import ( _resolve_layer_name, direct_register_custom_op, ) -from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata logger = init_logger(__name__) @@ -121,9 +120,9 @@ def fi_chunk_gated_delta_rule( class ChunkGatedDeltaRule(CustomOp): def __init__(self) -> None: super().__init__() - backend_cfg = get_current_vllm_config().additional_config.get( - "gdn_prefill_backend", "auto" - ) + additional_config = get_current_vllm_config().additional_config + assert isinstance(additional_config, dict) + backend_cfg = additional_config.get("gdn_prefill_backend", "auto") backend = str(backend_cfg).strip().lower() supports_flashinfer = ( @@ -621,18 +620,19 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): # Part 2: Core Attention # ============================================================ forward_context = get_forward_context() - attn_metadata: AttentionMetadata = forward_context.attn_metadata + attn_metadata_raw = forward_context.attn_metadata core_attn_out = torch.zeros( (num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim), dtype=hidden_states.dtype, device=hidden_states.device, ) z = torch.empty_like(core_attn_out) - if attn_metadata is not None: - attn_metadata = attn_metadata[self.prefix] + if attn_metadata_raw is not None: + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] # TODO: xpu does not support this param yet - spec_sequence_masks = attn_metadata.spec_sequence_masks + spec_sequence_masks = attn_metadata.spec_sequence_masks # type: ignore[attr-defined] assert spec_sequence_masks is None conv_weights = self.conv1d.weight.view( @@ -658,12 +658,12 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): activation=self.activation, A_log=self.A_log, dt_bias=self.dt_bias, - num_prefills=attn_metadata.num_prefills, - num_decodes=attn_metadata.num_decodes, - has_initial_state=attn_metadata.has_initial_state, - non_spec_query_start_loc=attn_metadata.non_spec_query_start_loc, - non_spec_state_indices_tensor=attn_metadata.non_spec_state_indices_tensor, - num_actual_tokens=attn_metadata.num_actual_tokens, + num_prefills=attn_metadata.num_prefills, # type: ignore[attr-defined] + num_decodes=attn_metadata.num_decodes, # type: ignore[attr-defined] + has_initial_state=attn_metadata.has_initial_state, # type: ignore[attr-defined] + non_spec_query_start_loc=attn_metadata.non_spec_query_start_loc, # type: ignore[attr-defined] + non_spec_state_indices_tensor=attn_metadata.non_spec_state_indices_tensor, # type: ignore[attr-defined] + num_actual_tokens=attn_metadata.num_actual_tokens, # type: ignore[attr-defined] tp_size=self.tp_size, reorder_input=not self.gqa_interleaved_layout, ) @@ -792,16 +792,16 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): core_attn_out: torch.Tensor, ): forward_context = get_forward_context() - attn_metadata: AttentionMetadata = forward_context.attn_metadata + attn_metadata_raw = forward_context.attn_metadata - if attn_metadata is None: + if attn_metadata_raw is None: # V1 profile run — warm up prefill kernels so that # autotuning completes before KV cache allocation. self._warmup_prefill_kernels(mixed_qkv) return - assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] # type: ignore[index] assert isinstance(attn_metadata, GDNAttentionMetadata) if ( @@ -860,14 +860,16 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): # 1.1: Process the multi-query part if spec_sequence_masks is not None: + # spec_state_indices_tensor is always set when spec_sequence_masks is set + assert spec_state_indices_tensor is not None mixed_qkv_spec = causal_conv1d_update( mixed_qkv_spec, conv_state, conv_weights, self.conv1d.bias, self.activation, - conv_state_indices=spec_state_indices_tensor[:, 0][ - : attn_metadata.num_spec_decodes + conv_state_indices=spec_state_indices_tensor[:, 0][ # type: ignore[index] + : attn_metadata.num_spec_decodes # type: ignore[attr-defined] ], num_accepted_tokens=num_accepted_tokens, query_start_loc=spec_query_start_loc, @@ -900,8 +902,8 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): conv_weights, self.conv1d.bias, self.activation, - conv_state_indices=non_spec_state_indices_tensor[ - : attn_metadata.num_actual_tokens + conv_state_indices=non_spec_state_indices_tensor[ # type: ignore[index] + : attn_metadata.num_actual_tokens # type: ignore[attr-defined] ], validate_data=True, ) @@ -965,8 +967,9 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): v=value_spec, initial_state=ssm_state, inplace_final_state=True, - cu_seqlens=spec_query_start_loc[ - : attn_metadata.num_spec_decodes + 1 + cu_seqlens=spec_query_start_loc[ # type: ignore[index] + : attn_metadata.num_spec_decodes + + 1 # type: ignore[attr-defined] ], ssm_state_indices=spec_state_indices_tensor, num_accepted_tokens=num_accepted_tokens, @@ -978,8 +981,10 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): # 2.2: Process the remaining part if attn_metadata.num_prefills > 0: - initial_state = ssm_state[non_spec_state_indices_tensor].contiguous() - initial_state[~has_initial_state, ...] = 0 + assert non_spec_state_indices_tensor is not None + initial_state = ssm_state[non_spec_state_indices_tensor].contiguous() # type: ignore[index] + assert has_initial_state is not None + initial_state[~has_initial_state, ...] = 0 # type: ignore[operator] ( core_attn_out_non_spec, last_recurrent_state, @@ -1012,8 +1017,9 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): v=value_non_spec, initial_state=ssm_state, inplace_final_state=True, - cu_seqlens=non_spec_query_start_loc[ - : attn_metadata.num_decodes + 1 + cu_seqlens=non_spec_query_start_loc[ # type: ignore[index] + : attn_metadata.num_decodes + + 1 # type: ignore[attr-defined] ], ssm_state_indices=non_spec_state_indices_tensor, use_qk_l2norm_in_kernel=True, @@ -1073,7 +1079,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): conv_weights, self.conv1d.bias, self.activation, - conv_state_indices=non_spec_state_indices_tensor[:num_actual_tokens], + conv_state_indices=non_spec_state_indices_tensor[:num_actual_tokens], # type: ignore[index] validate_data=False, ) out_buf = core_attn_out[:num_actual_tokens].unsqueeze(1) @@ -1086,7 +1092,7 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): scale=self.head_k_dim**-0.5, initial_state=ssm_state, out=out_buf, - ssm_state_indices=non_spec_state_indices_tensor[:num_actual_tokens], + ssm_state_indices=non_spec_state_indices_tensor[:num_actual_tokens], # type: ignore[index] use_qk_l2norm_in_kernel=True, ) return diff --git a/vllm/model_executor/layers/mamba/linear_attn.py b/vllm/model_executor/layers/mamba/linear_attn.py index 18fcc1426cc..8e8527aed8a 100644 --- a/vllm/model_executor/layers/mamba/linear_attn.py +++ b/vllm/model_executor/layers/mamba/linear_attn.py @@ -396,10 +396,11 @@ class MiniMaxText01LinearAttention(nn.Module, MambaBase): self, hidden_states: torch.Tensor, output: torch.Tensor, positions: torch.Tensor ) -> None: forward_context = get_forward_context() - attn_metadata: AttentionMetadata = forward_context.attn_metadata - if attn_metadata is not None: - assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] + attn_metadata_raw = forward_context.attn_metadata + attn_metadata: AttentionMetadata | None = None + if attn_metadata_raw is not None: + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] assert isinstance(attn_metadata, LinearAttentionMetadata) num_actual_tokens = ( attn_metadata.num_prefill_tokens + attn_metadata.num_decode_tokens diff --git a/vllm/model_executor/layers/mamba/mamba_mixer.py b/vllm/model_executor/layers/mamba/mamba_mixer.py index 4509a095628..0e476755201 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer.py @@ -40,6 +40,7 @@ from vllm.utils.torch_utils import ( _resolve_layer_name, direct_register_custom_op, ) +from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.mamba1_attn import Mamba1AttentionMetadata @@ -258,15 +259,16 @@ class MambaMixer(MambaBase, PluggableLayer): """ forward_context: ForwardContext = get_forward_context() - attn_metadata = forward_context.attn_metadata + attn_metadata_raw = forward_context.attn_metadata assert self.cache_config is not None mamba_block_size = self.cache_config.mamba_block_size is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" - if attn_metadata is not None: - assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] + attn_metadata: AttentionMetadata | None = None + if attn_metadata_raw is not None: + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] assert isinstance(attn_metadata, Mamba1AttentionMetadata) query_start_loc_p = attn_metadata.query_start_loc_p state_indices_tensor_p = attn_metadata.state_indices_tensor_p @@ -391,6 +393,9 @@ class MambaMixer(MambaBase, PluggableLayer): ssm_outputs.append(scan_out_p) if has_decode: + # state_indices_tensor_d is assigned when attn_metadata is not None, + # and has_decode is only True when attn_metadata is not None + assert state_indices_tensor_d is not None if is_mamba_cache_all: state_indices_tensor_d_input = state_indices_tensor_d.gather( 1, block_idx_last_computed_token_d.unsqueeze(1) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index 0518bde2f42..2b4b1934f9b 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -572,14 +572,16 @@ class MambaMixer2(MambaBase, PluggableLayer): # kernels to operate in continuous batching and in chunked prefill # modes; they are computed at top-level model forward since they # stay the same and reused for all mamba layers in the same iteration - attn_metadata: AttentionMetadata = forward_context.attn_metadata + attn_metadata_raw = forward_context.attn_metadata assert self.cache_config is not None mamba_block_size = self.cache_config.mamba_block_size is_mamba_cache_all = self.cache_config.mamba_cache_mode == "all" - if attn_metadata is not None: - assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] + + attn_metadata: AttentionMetadata | None = None + if attn_metadata_raw is not None: + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] assert isinstance(attn_metadata, Mamba2AttentionMetadata) # conv_state must be (..., dim, width-1) for the conv kernels. # DS layout stores it that way directly; SD layout needs a @@ -708,6 +710,7 @@ class MambaMixer2(MambaBase, PluggableLayer): # 3. State Space Model sequence transformation initial_states = None if has_initial_states_p is not None and prep_initial_states: + assert state_indices_tensor_p is not None kernel_ssm_indices = state_indices_tensor_p if is_mamba_cache_all: kernel_ssm_indices = state_indices_tensor_p.gather( @@ -746,6 +749,13 @@ class MambaMixer2(MambaBase, PluggableLayer): ) if is_mamba_cache_all: + assert mamba_block_size is not None + assert state_indices_tensor_p is not None + assert block_idx_first_scheduled_token_p is not None + assert block_idx_last_scheduled_token_p is not None + assert last_chunk_indices_p is not None + assert num_computed_tokens_p is not None + # The chunk_stride is the number of chunks per mamba block # e.g., if mamba_block_size = 512 and chunk_size = 256, # then chunk_stride = 2 @@ -810,6 +820,7 @@ class MambaMixer2(MambaBase, PluggableLayer): ssm_state[cache_blocks_to_fill] = from_where # For all seqs, store the last state (note: might be partial): + assert state_indices_tensor_p is not None ssm_state[ state_indices_tensor_p.gather( 1, block_idx_last_scheduled_token_p.unsqueeze(1) @@ -820,10 +831,12 @@ class MambaMixer2(MambaBase, PluggableLayer): # update ssm states # - varlen state is a (num_prefills, nheads, headdim, dstate) # tensor + assert state_indices_tensor_p is not None ssm_state[state_indices_tensor_p] = varlen_states # Process decode requests if has_decode: + assert state_indices_tensor_d is not None if is_mamba_cache_all: state_indices_tensor_d_input = state_indices_tensor_d.gather( 1, block_idx_last_computed_token_d.unsqueeze(1) diff --git a/vllm/model_executor/layers/mamba/short_conv.py b/vllm/model_executor/layers/mamba/short_conv.py index 11e9b590f86..629167acfe5 100644 --- a/vllm/model_executor/layers/mamba/short_conv.py +++ b/vllm/model_executor/layers/mamba/short_conv.py @@ -113,10 +113,11 @@ class ShortConv(MambaBase, CustomOp): # chunked prefill modes; they are computed at top-level model forward # since they stay the same and reused for all mamba layers in the same # iteration. - attn_metadata: AttentionMetadata = forward_context.attn_metadata - if attn_metadata is not None: - assert isinstance(attn_metadata, dict) - attn_metadata = attn_metadata[self.prefix] + attn_metadata_raw = forward_context.attn_metadata + attn_metadata: AttentionMetadata | None = None + if attn_metadata_raw is not None: + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] assert isinstance(attn_metadata, ShortConvAttentionMetadata) conv_state = ( self.kv_cache[0] diff --git a/vllm/model_executor/layers/pooler/seqwise/poolers.py b/vllm/model_executor/layers/pooler/seqwise/poolers.py index 74fa4cdbbe4..762869c6782 100644 --- a/vllm/model_executor/layers/pooler/seqwise/poolers.py +++ b/vllm/model_executor/layers/pooler/seqwise/poolers.py @@ -115,6 +115,7 @@ def pooler_for_classify( vllm_config = get_current_vllm_config() model_config = vllm_config.model_config + assert model_config.pooler_config is not None head = ClassifierPoolerHead( head_dtype=model_config.head_dtype, classifier=classifier, diff --git a/vllm/model_executor/layers/pooler/tokwise/poolers.py b/vllm/model_executor/layers/pooler/tokwise/poolers.py index 6462a5056c5..131074a5556 100644 --- a/vllm/model_executor/layers/pooler/tokwise/poolers.py +++ b/vllm/model_executor/layers/pooler/tokwise/poolers.py @@ -124,6 +124,7 @@ def pooler_for_token_classify( vllm_config = get_current_vllm_config() model_config = vllm_config.model_config + assert model_config.pooler_config is not None head = TokenClassifierPoolerHead( head_dtype=model_config.head_dtype, classifier=classifier, diff --git a/vllm/model_executor/layers/quantization/fp_quant.py b/vllm/model_executor/layers/quantization/fp_quant.py index 4ed8d57dd43..7d0b6a974d7 100644 --- a/vllm/model_executor/layers/quantization/fp_quant.py +++ b/vllm/model_executor/layers/quantization/fp_quant.py @@ -3,7 +3,7 @@ # Supports FP-Quant compression, see https://arxiv.org/abs/2509.23202 -from typing import Any +from typing import Any, Literal, cast import torch from torch.nn.parameter import Parameter @@ -251,7 +251,11 @@ class FPQuantLinearMethod(LinearMethodBase): def fused_quantize_mx( x_flat: torch.Tensor, hadamard_matrix: torch.Tensor, forward_method: str ) -> tuple[torch.Tensor, torch.Tensor]: - return fusedQuantizeMx(x_flat, hadamard_matrix, method=forward_method) + return fusedQuantizeMx( + x_flat, + hadamard_matrix, + method=cast(Literal["quest", "abs_max"], forward_method), + ) def fused_quantize_mx_fake(x_flat, hadamard_matrix, forward_method): diff --git a/vllm/model_executor/layers/quantization/quark/quark.py b/vllm/model_executor/layers/quantization/quark/quark.py index 33bd0cfc22e..6aaf9a64588 100644 --- a/vllm/model_executor/layers/quantization/quark/quark.py +++ b/vllm/model_executor/layers/quantization/quark/quark.py @@ -114,7 +114,7 @@ class QuarkConfig(QuantizationConfig): :param hf_to_vllm_mapper: maps from hf model structure (the assumed structure of the qconfig) to vllm model structure """ - quant_config_with_hf_to_vllm_mapper = {} + quant_config_with_hf_to_vllm_mapper: dict[str, Any] = {} for k, v in self.quant_config.items(): if isinstance(v, list): diff --git a/vllm/model_executor/layers/sparse_attn_indexer.py b/vllm/model_executor/layers/sparse_attn_indexer.py index bdaa6af0945..1f1f7e7df89 100644 --- a/vllm/model_executor/layers/sparse_attn_indexer.py +++ b/vllm/model_executor/layers/sparse_attn_indexer.py @@ -26,7 +26,7 @@ from vllm.v1.worker.workspace import current_workspace_manager if current_platform.is_cuda_alike(): from vllm import _custom_ops as ops elif current_platform.is_xpu(): - from vllm._xpu_ops import xpu_ops as ops + from vllm._xpu_ops import xpu_ops logger = init_logger(__name__) @@ -84,12 +84,12 @@ def sparse_attn_indexer( total_seq_lens, topk_indices_buffer, ) - attn_metadata = attn_metadata[k_cache_prefix] - assert isinstance(attn_metadata, DeepseekV32IndexerMetadata) - slot_mapping = attn_metadata.slot_mapping - has_decode = attn_metadata.num_decodes > 0 - has_prefill = attn_metadata.num_prefills > 0 - num_decode_tokens = attn_metadata.num_decode_tokens + attn_metadata_narrowed = attn_metadata[k_cache_prefix] + assert isinstance(attn_metadata_narrowed, DeepseekV32IndexerMetadata) + slot_mapping = attn_metadata_narrowed.slot_mapping + has_decode = attn_metadata_narrowed.num_decodes > 0 + has_prefill = attn_metadata_narrowed.num_prefills > 0 + num_decode_tokens = attn_metadata_narrowed.num_decode_tokens # During speculative decoding, k may be padded to the CUDA graph batch # size while slot_mapping only covers actual tokens. Truncate k to avoid @@ -97,6 +97,8 @@ def sparse_attn_indexer( num_tokens = slot_mapping.shape[0] k = k[:num_tokens] + # scale_fmt can be None, but the function expects str + assert scale_fmt is not None ops.indexer_k_quant_and_cache( k, kv_cache, @@ -107,7 +109,7 @@ def sparse_attn_indexer( topk_indices_buffer[: hidden_states.shape[0]] = -1 if has_prefill: - prefill_metadata = attn_metadata.prefill + prefill_metadata = attn_metadata_narrowed.prefill assert prefill_metadata is not None # Get the full shared workspace buffers once (will allocate on first use) @@ -144,7 +146,7 @@ def sparse_attn_indexer( ] if current_platform.is_xpu(): - ops.top_k_per_row_prefill( + xpu_ops.top_k_per_row_prefill( # type: ignore[attr-defined] logits, chunk.cu_seqlen_ks, chunk.cu_seqlen_ke, @@ -167,7 +169,7 @@ def sparse_attn_indexer( ) if has_decode: - decode_metadata = attn_metadata.decode + decode_metadata = attn_metadata_narrowed.decode assert decode_metadata is not None # kv_cache shape [ # kv_cache size requirement [num_block, block_size, n_head, head_dim], @@ -217,11 +219,11 @@ def sparse_attn_indexer( topk_indices, topk_workspace, topk_tokens, - attn_metadata.max_seq_len, + attn_metadata_narrowed.max_seq_len, ) else: if current_platform.is_xpu(): - ops.top_k_per_row_decode( + xpu_ops.top_k_per_row_decode( # type: ignore[attr-defined] logits, next_n, seq_lens, From 9b60e2ffaae1571f07dd4739da943415be8f4da5 Mon Sep 17 00:00:00 2001 From: Rishapveer Singh Date: Wed, 22 Apr 2026 05:15:58 +0200 Subject: [PATCH 023/153] [Bugfix] Fix quantized model initialization failure with prefetch offloading (#40432) Signed-off-by: Rishapveer Singh Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/offloader/prefetch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/offloader/prefetch.py b/vllm/model_executor/offloader/prefetch.py index cc04367d54c..466d8c13ce7 100644 --- a/vllm/model_executor/offloader/prefetch.py +++ b/vllm/model_executor/offloader/prefetch.py @@ -21,6 +21,7 @@ import torch.nn as nn import vllm.model_executor.offloader.prefetch_ops # noqa: F401 from vllm.logger import init_logger from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory +from vllm.utils.torch_utils import get_dtype_size logger = init_logger(__name__) @@ -53,7 +54,7 @@ class ParamInfo: numel = 1 for dim in self.shape: numel *= dim - return numel * torch.finfo(self.dtype).bits // 8 + return numel * get_dtype_size(self.dtype) class StaticBufferPool: From 4506319a286272e0ec4c4938691490ce6b33e832 Mon Sep 17 00:00:00 2001 From: Carl Y <4531192+carlyou@users.noreply.github.com> Date: Tue, 21 Apr 2026 20:16:58 -0700 Subject: [PATCH 024/153] [compile] mla + group fp8 fusion (#38877) Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docs/design/fusions.md | 4 +- tests/compile/fusions_e2e/conftest.py | 16 + tests/compile/fusions_e2e/models.py | 24 +- tests/compile/fusions_e2e/test_tp1_quant.py | 5 +- tests/compile/fusions_e2e/test_tp2_ar_rms.py | 5 +- .../passes/test_mla_attn_quant_fusion.py | 117 ++++++-- .../passes/fusion/mla_attn_quant_fusion.py | 277 ++++++++++++++++-- .../layers/attention/mla_attention.py | 106 ++++++- vllm/v1/attention/backend.py | 16 +- 9 files changed, 511 insertions(+), 59 deletions(-) diff --git a/docs/design/fusions.md b/docs/design/fusions.md index afc7f351500..046c509d4b8 100644 --- a/docs/design/fusions.md +++ b/docs/design/fusions.md @@ -44,7 +44,7 @@ The table below lists the quantization schemes supported by each fusion on each | `fuse_allreduce_rms` | FP16/BF16, FP8 static, NVFP4 | FP16/BF16, FP8 static | — | — | — | | `fuse_minimax_qk_norm`\* | FP16/BF16 | FP16/BF16 | FP16/BF16 | FP16/BF16 | — | | `fuse_attn_quant`\* | FP8 static\*, NVFP4\* | FP8 static\* | FP8 static\* | — | FP8 static\* | -| `fuse_attn_quant` (MLA)\* | FP8 static\*, NVFP4\* | FP8 static\* | FP8 static\* | — | FP8 static(untested)\* | +| `fuse_attn_quant` (MLA)\* | FP8 static\*, FP8 per-group\*, NVFP4\* | FP8 static\*, FP8 per-group\* | FP8 static\*, FP8 per-group\* | — | FP8 static\* (untested) | | `fuse_rope_kvcache` | — | — | — | — | FP16/BF16 | | `enable_qk_norm_rope_fusion` | FP16/BF16 | FP16/BF16 | FP16/BF16† | FP16/BF16† | — | | `enable_sp` | FP16/BF16, FP8 static† | FP16/BF16, FP8 static | FP16/BF16† | FP16/BF16† | — | @@ -152,7 +152,7 @@ standard `Attention` and `MLAAttention` (used by DeepSeek-V2/V3/R1 models). Patt - `FLASHINFER`: CUDA sm100+ with FlashInfer installed -`MLAAttention → FP8 static quant` / `MLAAttention → NVFP4 dynamic quant`: +`MLAAttention → FP8 static, FP8 per-group, NVFP4 dynamic quant` The MLA fusion operates at the graph level on the `unified_mla_attention_with_output` op and works with all MLA decode and prefill backend combinations. Unlike standard `Attention` backends (where diff --git a/tests/compile/fusions_e2e/conftest.py b/tests/compile/fusions_e2e/conftest.py index adc569192d1..f7896728f9d 100644 --- a/tests/compile/fusions_e2e/conftest.py +++ b/tests/compile/fusions_e2e/conftest.py @@ -116,6 +116,22 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): model_kwargs["attention_config"] = {"backend": attn_backend.backend.name} model_kwargs["tensor_parallel_size"] = tp_size + # Sparse MLA models (DSv3.2) hit an over-strict inductor assertion in + # decompose_auto_functionalized when +rotary_embedding is forced into + # the compile graph. Disable qk_norm+rope fusion (which auto-enables + # +rotary_embedding) for this combo to avoid the known torch bug. + # TODO: remove once upstream torch fix lands. + if requires_sparse: + if "pass_config" in compilation_config: + compilation_config["pass_config"].enable_qk_norm_rope_fusion = False + matches_check = [m for m in matches_check if m != "norm_rope_fusion"] + # DSv3.2 sparse indexer uses persistent_topk with k=config.index_topk + # (2048 for the default config). max_model_len must be >= index_topk + # or the topk kernel raises "k out of range" at runtime. + model_kwargs["max_model_len"] = max( + model_kwargs.get("max_model_len", 0), 2048 + ) + # Always compile the full graph instead of piecewise if not compilation_config["use_inductor_graph_partition"]: compilation_config["splitting_ops"] = [] diff --git a/tests/compile/fusions_e2e/models.py b/tests/compile/fusions_e2e/models.py index 8d830e88406..3d73373cb91 100644 --- a/tests/compile/fusions_e2e/models.py +++ b/tests/compile/fusions_e2e/models.py @@ -59,7 +59,10 @@ TRITON_MLA_ATTN = pytest.param( ) FLASHMLA_SPARSE_ATTN = pytest.param( - AttentionBackendCase(backend=AttentionBackendEnum.FLASHMLA_SPARSE), + AttentionBackendCase( + backend=AttentionBackendEnum.FLASHMLA_SPARSE, + model_kwargs=dict(kv_cache_dtype="fp8_ds_mla"), + ), id="FLASHMLA_SPARSE", marks=pytest.mark.skipif( not is_blackwell(), @@ -173,9 +176,8 @@ deepseek_v3_fp8 = ModelFusionInfo( rms_quant_fusion=n_layers * 2 + min(3, n_layers), # add for 3 dense layers # silu+block quant act_quant_fusion=min(3, n_layers), # dense layers only - # MLA attn + per-group FP8 quant not supported yet: - # https://github.com/vllm-project/vllm/issues/35792 - attn_quant_fusion=0, + # MLA attn + per-group FP8 quant + attn_quant_fusion=n_layers, ar_rms_fusion=n_layers * 2 + 1, # TODO # sequence_parallel= n_layers * 2 + 1, @@ -183,11 +185,23 @@ deepseek_v3_fp8 = ModelFusionInfo( ), ) +deepseek_r1_fp4 = ModelFusionInfo( + model_name="nvidia/DeepSeek-R1-0528-NVFP4-v2", + matches=lambda n_layers: Matches( + rms_quant_fusion=0, + act_quant_fusion=min(3, n_layers), + attn_quant_fusion=n_layers, + ar_rms_fusion=n_layers * 2 + 1, + ), +) + deepseek_v32_fp4 = ModelFusionInfo( model_name="nvidia/DeepSeek-V3.2-NVFP4", matches=lambda n_layers: Matches( rms_quant_fusion=0, - act_quant_fusion=0, + # silu+quant on dense layers only; MoE hides the act+quant site + act_quant_fusion=min(3, n_layers), + # MLA attn + NVFP4 output quant fuses on sparse MLA output path attn_quant_fusion=n_layers, ar_rms_fusion=n_layers * 2 + 1, ), diff --git a/tests/compile/fusions_e2e/test_tp1_quant.py b/tests/compile/fusions_e2e/test_tp1_quant.py index ded39939e16..fbb382b4458 100644 --- a/tests/compile/fusions_e2e/test_tp1_quant.py +++ b/tests/compile/fusions_e2e/test_tp1_quant.py @@ -24,6 +24,7 @@ from .models import ( TRITON_ATTN, TRITON_MLA_ATTN, deepseek_coder_v2_lite_fp8, + deepseek_r1_fp4, deepseek_v3_fp8, deepseek_v32_fp4, llama3_8b_fp4, @@ -148,11 +149,11 @@ def test_tp1_fp8_fusions( @pytest.mark.parametrize( "model_name, matches_fn, model_kwargs, hf_overrides", - [llama3_8b_fp4, llama4_scout_fp4, deepseek_v32_fp4], + [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4], ) @pytest.mark.parametrize( "attn_backend", - [FLASHINFER_ATTN, FLASHMLA_SPARSE_ATTN], + [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN], ) @pytest.mark.parametrize("n_layers", [6]) @pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm")) diff --git a/tests/compile/fusions_e2e/test_tp2_ar_rms.py b/tests/compile/fusions_e2e/test_tp2_ar_rms.py index 4b0a0859b02..9156f6afa06 100644 --- a/tests/compile/fusions_e2e/test_tp2_ar_rms.py +++ b/tests/compile/fusions_e2e/test_tp2_ar_rms.py @@ -21,6 +21,7 @@ from .models import ( FLASHMLA_SPARSE_ATTN, TRITON_ATTN, deepseek_coder_v2_lite_fp8, + deepseek_r1_fp4, deepseek_v3_fp8, deepseek_v32_fp4, gpt_oss_20b, @@ -113,11 +114,11 @@ def test_tp2_ar_rms_fp8_fusions( @multi_gpu_test(num_gpus=2) @pytest.mark.parametrize( "model_name, matches_fn, model_kwargs, hf_overrides", - [llama3_8b_fp4, llama4_scout_fp4, deepseek_v32_fp4], + [llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4, deepseek_v32_fp4], ) @pytest.mark.parametrize( "attn_backend", - [FLASHINFER_ATTN, FLASHMLA_SPARSE_ATTN], + [FLASHINFER_ATTN, FLASHINFER_MLA_ATTN, FLASHMLA_SPARSE_ATTN], ) @pytest.mark.parametrize("n_layers", [4]) @pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm")) diff --git a/tests/compile/passes/test_mla_attn_quant_fusion.py b/tests/compile/passes/test_mla_attn_quant_fusion.py index a5875a6b396..8a575909612 100644 --- a/tests/compile/passes/test_mla_attn_quant_fusion.py +++ b/tests/compile/passes/test_mla_attn_quant_fusion.py @@ -29,12 +29,18 @@ from vllm.config import ( set_current_vllm_config, ) from vllm.forward_context import get_forward_context, set_forward_context +from vllm.model_executor.kernels.linear.scaled_mm.cutlass import ( + CutlassFp8BlockScaledMMKernel, +) from vllm.model_executor.layers.attention import MLAAttention from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.quantization.fp8 import Fp8Config from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, QuantKey, + create_fp8_quant_key, + kFp8Dynamic128Sym, kFp8StaticTensorSym, kNvfp4Dynamic, ) @@ -279,6 +285,67 @@ class TestMLAAttentionNvfp4QuantPatternModel(MLAAttentionQuantPatternModel): ) +class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel): + """Test model for MLA Attention + per-group FP8 (block quant) fusion.""" + + quant_key = kFp8Dynamic128Sym + quant_config = Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + weight_quant_key = create_fp8_quant_key( + static=True, group_shape=GroupShape(128, 128) + ) + device = kwargs.get("device", torch.device("cuda:0")) + + # Subclass to set weight_block_size before process_weights_after_loading + class _BlockFP8Layer(TestFP8Layer): + def __init__(self, *a, **kw): + self.weight_block_size = [128, 128] + super().__init__(*a, **kw) + + # Force CutlassFp8BlockScaledMMKernel to ensure the graph uses + # per_token_group_fp8_quant (not the deepgemm packed variant). + self.block_fp8_linear = _BlockFP8Layer( + weight_shape=(self.output_dim, self.output_dim), + activation_quant_key=self.quant_key, + weight_quant_key=weight_quant_key, + input_dtype=self.dtype, + device=device, + force_kernel=CutlassFp8BlockScaledMMKernel, + ) + + w = kwargs.get("w") + if w is not None: + self.block_fp8_linear.weight = w["weight"] + # Block-wise uses weight_scale_inv, not weight_scale + self.block_fp8_linear.weight_scale_inv = w["wscale"] + + self.w = { + "weight": self.block_fp8_linear.weight, + "wscale": self.block_fp8_linear.weight_scale_inv, + } + + def forward( + self, + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + ): + """Forward pass: MLA attention -> block FP8 linear (group quant).""" + attn_output = self.mla_attn( + q, + kv_c_normed, + k_pe, + output_shape=(q.shape[0], self.output_dim), + ) + return self.block_fp8_linear(attn_output) + + def is_nvfp4_supported(): return current_platform.has_device_capability(100) @@ -286,6 +353,7 @@ def is_nvfp4_supported(): # MLA test configuration MLA_DIMS: list[tuple[int, int, int, int, int]] = [] PATTERN_TEST_MODELS_MLA_FP8: list[tuple[str, type]] = [] +PATTERN_TEST_MODELS_MLA_GROUP_FP8: list[tuple[str, type]] = [] PATTERN_TEST_MODELS_MLA_FP4: list[tuple[str, type]] = [] BACKENDS_MLA_FP8: list[AttentionBackendEnum] = [] BACKENDS_MLA_FP4: list[AttentionBackendEnum] = [] @@ -299,6 +367,12 @@ if current_platform.is_cuda(): TestMLAAttentionFp8StaticQuantPatternModel, ) ] + PATTERN_TEST_MODELS_MLA_GROUP_FP8 = [ + ( + "deepseek-ai/DeepSeek-V3", + TestMLAAttentionFp8GroupQuantPatternModel, + ) + ] PATTERN_TEST_MODELS_MLA_FP4 = [ ( "deepseek-ai/DeepSeek-V2-Lite", @@ -324,6 +398,13 @@ if current_platform.is_cuda(): ["+quant_fp8", "-quant_fp8"], ) ) + + list( + flat_product( + BACKENDS_MLA_FP8, + PATTERN_TEST_MODELS_MLA_GROUP_FP8, + ["+quant_fp8"], + ) + ) + list(flat_product(BACKENDS_MLA_FP4, PATTERN_TEST_MODELS_MLA_FP4, [""])), ) @pytest.mark.skipif( @@ -470,12 +551,13 @@ def test_mla_attention_quant_pattern( ) # Check quantization ops in the graph + is_per_group = quant_key.scale.group_shape.is_per_group() quant_op = ( torch.ops.aten.reciprocal if "-quant_fp8" in custom_ops_list else QUANT_OPS[quant_key] ) - test_backend.check_before_ops([quant_op], fully_replaced=quant_key is kNvfp4Dynamic) + test_backend.check_before_ops([quant_op], fully_replaced=is_per_group) assert attn_pass.pass_.matched_count == sum(attn_fusion_supported) @@ -487,25 +569,24 @@ def test_mla_attention_quant_pattern( assert len(attn_nodes_pre) == len(attn_nodes_post), ( "Should have same number of MLA attention nodes before and after fusion" ) - assert attn_nodes_pre[0].kwargs.get("output_scale") is None, ( - "MLA attention should not have output_scale before fusion" - ) - assert attn_nodes_post[0].kwargs.get("output_scale") is not None, ( - "MLA attention should have output_scale after fusion" - ) - assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None, ( - "MLA attention should not have output_block_scale before fusion" - ) + # Before fusion: neither scale should be set + assert attn_nodes_pre[0].kwargs.get("output_scale") is None + assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None - if quant_key.dtype == FP8_DTYPE: - assert attn_nodes_post[0].kwargs.get("output_block_scale") is None, ( - "MLA attention should not have output_block_scale after FP8 fusion" - ) - elif quant_key.dtype == FP4_DTYPE: - assert attn_nodes_post[0].kwargs.get("output_block_scale") is not None, ( - "MLA attention should have output_block_scale after FP4 fusion" - ) + # After fusion: derive expected scale presence from quant_key properties. + # - output_scale: present for static quant or non-FP8 (NVFP4 carries input_scale) + # - output_block_scale: present when quant uses per-group/block scaling + has_output_scale = attn_nodes_post[0].kwargs.get("output_scale") is not None + has_block_scale = attn_nodes_post[0].kwargs.get("output_block_scale") is not None + + expects_output_scale = quant_key.scale.static or quant_key.dtype != FP8_DTYPE + assert has_output_scale == expects_output_scale, ( + f"output_scale: expected present={expects_output_scale}, got {has_output_scale}" + ) + assert has_block_scale == is_per_group, ( + f"output_block_scale: expected present={is_per_group}, got {has_block_scale}" + ) # Check numerical correctness torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2) diff --git a/vllm/compilation/passes/fusion/mla_attn_quant_fusion.py b/vllm/compilation/passes/fusion/mla_attn_quant_fusion.py index e36400ec8ec..84c24bc60e5 100644 --- a/vllm/compilation/passes/fusion/mla_attn_quant_fusion.py +++ b/vllm/compilation/passes/fusion/mla_attn_quant_fusion.py @@ -6,15 +6,18 @@ from collections.abc import Callable import torch from torch._higher_order_ops.auto_functionalize import auto_functionalized -from vllm._custom_ops import create_fp4_output_tensors from vllm.config import VllmConfig, get_layers_from_vllm_config from vllm.logger import init_logger from vllm.model_executor.layers.attention.mla_attention import MLAAttention from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Dynamic64Sym, + kFp8Dynamic128Sym, kFp8StaticTensorSym, kNvfp4Dynamic, ) from vllm.platforms import current_platform +from vllm.utils.math_utils import round_up from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement @@ -203,6 +206,8 @@ class MLAAttnNvfp4QuantPattern( kv_c_normed, k_pe, output_attn, + output_quant, + output_scale, input_scale, kv_cache_dummy_dep, layer_name, @@ -218,9 +223,6 @@ class MLAAttnNvfp4QuantPattern( output_block_scale=None, kv_cache_dummy_dep=kv_cache_dummy_dep, ) - output_quant, output_scale = create_fp4_output_tensors( - at1[1].shape[0], at1[1].shape[1], at1[1].device, True - ) at2 = auto_functionalized( self._QUANT_OP, input=at1[1], @@ -235,7 +237,14 @@ class MLAAttnNvfp4QuantPattern( return _pattern_with_ln def _pattern( - q, kv_c_normed, k_pe, output_attn, input_scale, kv_cache_dummy_dep + q, + kv_c_normed, + k_pe, + output_attn, + output_quant, + output_scale, + input_scale, + kv_cache_dummy_dep, ): at1 = auto_functionalized( MLA_ATTN_OP, @@ -248,11 +257,6 @@ class MLAAttnNvfp4QuantPattern( output_block_scale=None, kv_cache_dummy_dep=kv_cache_dummy_dep, ) - # Replicate what scaled_fp4_quant() does: allocate output - # tensors inline then call the .out variant. - output_quant, output_scale = create_fp4_output_tensors( - at1[1].shape[0], at1[1].shape[1], at1[1].device, True - ) at2 = auto_functionalized( self._QUANT_OP, input=at1[1], @@ -279,6 +283,8 @@ class MLAAttnNvfp4QuantPattern( kv_c_normed, k_pe, output_attn, + _output_quant, + output_scale, input_scale, kv_cache_dummy_dep, layer_name, @@ -289,9 +295,6 @@ class MLAAttnNvfp4QuantPattern( dtype=FP4_DTYPE, device=q.device, ) - output_scale = create_fp4_output_tensors( - q.shape[0], self._output_dim, q.device, True - )[1] output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE) at2 = auto_functionalized( MLA_ATTN_OP, @@ -309,7 +312,14 @@ class MLAAttnNvfp4QuantPattern( return _replacement_with_ln def _replacement( - q, kv_c_normed, k_pe, output_attn, input_scale, kv_cache_dummy_dep + q, + kv_c_normed, + k_pe, + output_attn, + _output_quant, + output_scale, + input_scale, + kv_cache_dummy_dep, ): # MLA output in quant_dtype (FP4 packed as uint8) output_attn = torch.empty( @@ -317,10 +327,6 @@ class MLAAttnNvfp4QuantPattern( dtype=FP4_DTYPE, device=q.device, ) - # attention output block scale - output_scale = create_fp4_output_tensors( - q.shape[0], self._output_dim, q.device, True - )[1] output_scale_view = torch.ops.aten.view.dtype(output_scale, FP8_DTYPE) at2 = auto_functionalized( MLA_ATTN_OP, @@ -343,6 +349,8 @@ class MLAAttnNvfp4QuantPattern( self.empty(5, self._kv_lora_rank, dtype=self._dtype), self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype), self.empty(5, self._output_dim, dtype=self._dtype), + self.empty(5, self._output_dim // 2, dtype=FP4_DTYPE), + self.empty_i32(128, round_up(self._output_dim // 16, 4)), self.empty_fp32(1, 1), self.empty(0, dtype=self._dtype), ] @@ -351,6 +359,218 @@ class MLAAttnNvfp4QuantPattern( return inputs +class MLAAttnFp8GroupQuantPattern( + VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]] +): + """ + Fusion for MLA Attention+Fp8GroupQuant (per-group dynamic FP8). + + Matches the pattern: MLA attention -> per_token_group_fp8_quant, and + replaces it with MLA attention(output_block_scale=group_scale_buffer). + Used by models with block FP8 quantization (e.g. DeepSeek V3). + """ + + def __init__( + self, + layer: MLAAttention, + dtype: torch.dtype, + quant_key: QuantKey, + has_col_major_scales: bool, + is_e8m0: bool, + is_tma_aligned: bool, + ) -> None: + self._layer_name = layer.layer_name + self._num_heads = layer.num_heads + self._v_head_dim = layer.v_head_dim + self._kv_lora_rank = layer.kv_lora_rank + self._qk_rope_head_dim = layer.qk_rope_head_dim + self._qk_head_dim = layer.qk_nope_head_dim + layer.qk_rope_head_dim + self._output_dim = layer.num_heads * layer.v_head_dim + self._dtype = dtype + self._layer = layer + self._group_size = quant_key.scale.group_shape[1] + self._has_col_major_scales = has_col_major_scales + self._is_e8m0 = is_e8m0 + self._is_tma_aligned = is_tma_aligned + + self._quant_matcher = MatcherQuantFP8( + quant_key, + has_col_major_scales=has_col_major_scales, + is_e8m0=is_e8m0, + is_tma_aligned=is_tma_aligned, + ) + + @property + def pattern( + self, + ) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]: + _ln = _encode_layer_name(self._layer_name) + + if _USE_LAYERNAME: + + def _pattern_with_ln( # type: ignore[misc] + q, + kv_c_normed, + k_pe, + output_attn, + kv_cache_dummy_dep, + scale, + layer_name, + ): + at1 = auto_functionalized( + MLA_ATTN_OP, + q=q, + kv_c_normed=kv_c_normed, + k_pe=k_pe, + output=output_attn, + layer_name=layer_name, + output_scale=None, + output_block_scale=None, + kv_cache_dummy_dep=kv_cache_dummy_dep, + ) + attn_out = at1[1] + result = torch.empty( + attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE + ) + finfo = torch.finfo(FP8_DTYPE) + _, result, scale = auto_functionalized( + self._quant_matcher.QUANT_OP, + input=attn_out, + output_q=result, + output_s=scale, + group_size=self._group_size, + eps=1e-10, + fp8_min=finfo.min, + fp8_max=finfo.max, + scale_ue8m0=self._is_e8m0, + dummy_is_scale_transposed=self._has_col_major_scales, + dummy_is_tma_aligned=self._is_tma_aligned, + ) + return result, scale + + return _pattern_with_ln + + def _pattern( + q: torch.Tensor, + kv_c_normed: torch.Tensor, + k_pe: torch.Tensor, + output_attn: torch.Tensor, + kv_cache_dummy_dep: torch.Tensor, + scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + at1 = auto_functionalized( + MLA_ATTN_OP, + q=q, + kv_c_normed=kv_c_normed, + k_pe=k_pe, + output=output_attn, + layer_name=_ln, + output_scale=None, + output_block_scale=None, + kv_cache_dummy_dep=kv_cache_dummy_dep, + ) + attn_out = at1[1] + result = torch.empty( + attn_out.shape, device=attn_out.device, dtype=FP8_DTYPE + ) + finfo = torch.finfo(FP8_DTYPE) + _, result, scale = auto_functionalized( + self._quant_matcher.QUANT_OP, + input=attn_out, + output_q=result, + output_s=scale, + group_size=self._group_size, + eps=1e-10, + fp8_min=finfo.min, + fp8_max=finfo.max, + scale_ue8m0=self._is_e8m0, + dummy_is_scale_transposed=self._has_col_major_scales, + dummy_is_tma_aligned=self._is_tma_aligned, + ) + return result, scale + + return _pattern + + @property + def replacement( + self, + ) -> Callable[..., tuple[torch.Tensor, torch.Tensor]]: + _ln = _encode_layer_name(self._layer_name) + + if _USE_LAYERNAME: + + def _replacement_with_ln( # type: ignore[misc] + q, + kv_c_normed, + k_pe, + output_attn, + kv_cache_dummy_dep, + scale, + layer_name, + ): + output_attn = torch.empty( + [q.shape[0], self._output_dim], + dtype=FP8_DTYPE, + device=q.device, + ) + at1 = auto_functionalized( + MLA_ATTN_OP, + q=q, + kv_c_normed=kv_c_normed, + k_pe=k_pe, + output=output_attn, + layer_name=layer_name, + output_scale=None, + output_block_scale=scale, + kv_cache_dummy_dep=kv_cache_dummy_dep, + quant_group_size=self._group_size, + quant_scale_ue8m0=self._is_e8m0, + quant_col_major=self._has_col_major_scales, + quant_tma_aligned=self._is_tma_aligned, + ) + return at1[1], at1[2] + + return _replacement_with_ln + + def _replacement(q, kv_c_normed, k_pe, output_attn, kv_cache_dummy_dep, scale): + output_attn = torch.empty( + [q.shape[0], self._output_dim], + dtype=FP8_DTYPE, + device=q.device, + ) + at1 = auto_functionalized( + MLA_ATTN_OP, + q=q, + kv_c_normed=kv_c_normed, + k_pe=k_pe, + output=output_attn, + layer_name=_ln, + output_scale=None, + output_block_scale=scale, + kv_cache_dummy_dep=kv_cache_dummy_dep, + quant_group_size=self._group_size, + quant_scale_ue8m0=self._is_e8m0, + quant_col_major=self._has_col_major_scales, + quant_tma_aligned=self._is_tma_aligned, + ) + return at1[1], at1[2] + + return _replacement + + def get_inputs(self) -> list[torch.Tensor]: + inputs: list = [ + self.empty(5, self._num_heads, self._qk_head_dim, dtype=self._dtype), + self.empty(5, self._kv_lora_rank, dtype=self._dtype), + self.empty(5, 1, self._qk_rope_head_dim, dtype=self._dtype), + self.empty(5, self._output_dim, dtype=self._dtype), + self.empty(0, dtype=self._dtype), + self._quant_matcher.empty_f32(1, 1), + ] + if _USE_LAYERNAME: + inputs.append(_encode_layer_name(self._layer_name)) + return inputs + + class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass): """ This pass fuses post-attention quantization onto MLA attention if supported. @@ -389,4 +609,25 @@ class MLAAttnQuantFusionPass(VllmFusionPatternMatcherPass): if _USE_LAYERNAME: break + # Per-group FP8 (block quant) — register all flag combinations. + if current_platform.is_cuda(): + for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]: + for col_major in [True, False]: + for is_e8m0 in [True, False]: + for tma_aligned in [False, True]: + for layer in layers: + if layer.impl.fused_output_quant_supported(quant_key): + self.register( + MLAAttnFp8GroupQuantPattern( + layer, + dtype, + quant_key, + col_major, + is_e8m0, + tma_aligned, + ) + ) + if _USE_LAYERNAME: + break + self.dump_patterns(config, self.pm_pass) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index a1e3921b0c5..9d6ae6bf601 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -234,7 +234,12 @@ from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8 from vllm.model_executor.layers.quantization.utils.quant_utils import ( GroupShape, + QuantKey, get_and_maybe_dequant_weights, + kFp8Dynamic64Sym, + kFp8Dynamic128Sym, + kFp8StaticTensorSym, + kNvfp4Dynamic, ) from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer, has_nvidia_artifactory @@ -276,6 +281,44 @@ from vllm.v1.kv_cache_interface import ( logger = init_logger(__name__) +_FP8_DTYPE = current_platform.fp8_dtype() + + +def _detect_output_quant_key( + output: torch.Tensor, + output_scale: torch.Tensor | None, + output_block_scale: torch.Tensor | None, + output_dim: int, +) -> QuantKey | None: + """Detect the output quantization key from fusion pass parameters. + + Returns the appropriate QuantKey, or None if no quantization is needed. + Detection is based on output dtype and which scale tensors are present. + """ + if output_scale is None and output_block_scale is None: + return None + if output_block_scale is not None: + if output.dtype == _FP8_DTYPE: + # Per-group FP8 uses block scales only, not a separate output_scale + assert output_scale is None + # Infer group size from scale shape + num_groups = output_block_scale.shape[-1] + group_size = output_dim // num_groups + if group_size == 128: + return kFp8Dynamic128Sym + elif group_size == 64: + return kFp8Dynamic64Sym + else: + raise ValueError( + f"Unsupported group FP8 group_size={group_size} " + f"(output_dim={output_dim}, num_groups={num_groups}). " + f"Only group_size 128 and 64 are supported." + ) + # output_scale None implies MXFP4, not supported + assert output_scale is not None + return kNvfp4Dynamic + return kFp8StaticTensorSym + class MLAAttention(nn.Module, AttentionLayerBase): """Multi-Head Latent Attention layer. @@ -549,9 +592,17 @@ class MLAAttention(nn.Module, AttentionLayerBase): output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, + quant_group_size: int | None = None, + quant_scale_ue8m0: bool | None = None, + quant_col_major: bool | None = None, + quant_tma_aligned: bool | None = None, ) -> torch.Tensor: - use_quant = output_scale is not None or output_block_scale is not None - if use_quant: + assert output is not None, "Output tensor must be provided." + + quant_key = _detect_output_quant_key( + output, output_scale, output_block_scale, self.num_heads * self.v_head_dim + ) + if quant_key is not None: # The fusion pass has allocated output with quantized dtype # (FP8 or uint8 for FP4). We can't write into it directly, # so we swap in a temp buffer for computation, then quantize @@ -582,7 +633,7 @@ class MLAAttention(nn.Module, AttentionLayerBase): # The zero fill is required when used with DP + EP # to ensure all ranks within a DP group compute the # same expert outputs. - if use_quant: + if quant_key is not None: return quant_output.fill_(0) return output.fill_(0) @@ -724,18 +775,41 @@ class MLAAttention(nn.Module, AttentionLayerBase): # v_up projection self._v_up_proj(attn_out, out=mqa_output_slice) - if use_quant: + if quant_key is not None: # Quantize the BF16 computation result into the quantized output actual = output[:num_actual_toks] - if output_block_scale is not None: + if quant_key == kNvfp4Dynamic: # NVFP4: two FP4 values packed into one uint8 + assert output_block_scale is not None fp4_data, fp4_scales = ops.scaled_fp4_quant(actual, output_scale) quant_output[:num_actual_toks].copy_(fp4_data) - output_block_scale.copy_(fp4_scales) - else: + output_block_scale[: fp4_scales.shape[0]].copy_(fp4_scales) + elif quant_key in (kFp8Dynamic128Sym, kFp8Dynamic64Sym): + # Per-group FP8 + assert output_block_scale is not None + assert quant_group_size is not None, ( + "Group FP8 output quant requested but " + "quant_group_size not passed through custom op" + ) + finfo = torch.finfo(_FP8_DTYPE) + torch.ops._C.per_token_group_fp8_quant( + actual, + quant_output[:num_actual_toks], + output_block_scale[:num_actual_toks], + quant_group_size, + 1e-10, # eps + finfo.min, + finfo.max, + quant_scale_ue8m0, + quant_col_major, + quant_tma_aligned, + ) + elif quant_key == kFp8StaticTensorSym: # Static FP8 quantization fp8_data, _ = self._quant_fp8_op(actual, output_scale) quant_output[:num_actual_toks].copy_(fp8_data) + else: + raise ValueError(f"Unsupported quant_key: {quant_key}") return quant_output return output_padded @@ -980,6 +1054,10 @@ def unified_mla_attention_with_output( output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, kv_cache_dummy_dep: torch.Tensor | None = None, + quant_group_size: int | None = None, + quant_scale_ue8m0: bool | None = None, + quant_col_major: bool | None = None, + quant_tma_aligned: bool | None = None, ) -> None: # kv_cache_dummy_dep is not used but accepting it creates a data dependency # that ensures torch.compile preserves ordering between KV cache update and @@ -996,6 +1074,10 @@ def unified_mla_attention_with_output( output=output, output_scale=output_scale, output_block_scale=output_block_scale, + quant_group_size=quant_group_size, + quant_scale_ue8m0=quant_scale_ue8m0, + quant_col_major=quant_col_major, + quant_tma_aligned=quant_tma_aligned, ) @@ -1008,6 +1090,10 @@ def unified_mla_attention_with_output_fake( output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, kv_cache_dummy_dep: torch.Tensor | None = None, + quant_group_size: int | None = None, + quant_scale_ue8m0: bool | None = None, + quant_col_major: bool | None = None, + quant_tma_aligned: bool | None = None, ) -> None: return @@ -2078,13 +2164,13 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): """ def fused_output_quant_supported(self, quant_key): - from vllm.model_executor.layers.quantization.utils.quant_utils import ( + return quant_key in ( kFp8StaticTensorSym, kNvfp4Dynamic, + kFp8Dynamic128Sym, + kFp8Dynamic64Sym, ) - return quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic) - def __init__( self, num_heads: int, diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 28d077fcb77..d2005181992 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -11,6 +11,8 @@ import torch from typing_extensions import deprecated from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kFp8Dynamic64Sym, + kFp8Dynamic128Sym, kFp8StaticTensorSym, kNvfp4Dynamic, ) @@ -880,7 +882,12 @@ class MLAAttentionImpl(AttentionImplBase[T], Generic[T]): Since MLA quantization is done manually in forward_impl (common code), all MLA backends support it by default. """ - return quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic) + return quant_key in ( + kFp8StaticTensorSym, + kNvfp4Dynamic, + kFp8Dynamic128Sym, + kFp8Dynamic64Sym, + ) def do_kv_cache_update( self, @@ -918,7 +925,12 @@ class SparseMLAAttentionImpl(AttentionImplBase[T], Generic[T]): Since MLA quantization is done manually in forward_impl (common code), all MLA backends support it by default. """ - return quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic) + return quant_key in ( + kFp8StaticTensorSym, + kNvfp4Dynamic, + kFp8Dynamic128Sym, + kFp8Dynamic64Sym, + ) @abstractmethod def __init__( From 6d097697001ed8cf06190076b70ce36c8bab3437 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Tue, 21 Apr 2026 22:57:12 -0500 Subject: [PATCH 025/153] [ROCm] Support non-causal attention in ROCM_ATTN (#40176) Signed-off-by: Micah Williamson --- .../backends/rocm_aiter_unified_attn.py | 8 +++-- vllm/v1/attention/backends/rocm_attn.py | 14 +++++++-- .../ops/chunked_prefill_paged_decode.py | 2 ++ vllm/v1/attention/ops/prefix_prefill.py | 30 ++++++++++++++----- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index eb0fe046e34..55ed5c5b3c4 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -13,10 +13,10 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.utils.torch_utils import is_quantized_kv_cache from vllm.v1.attention.backend import AttentionLayer, AttentionType, MultipleOf -from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata from vllm.v1.attention.backends.rocm_attn import ( RocmAttentionBackend, RocmAttentionImpl, + RocmAttentionMetadata, RocmAttentionMetadataBuilder, ) @@ -53,6 +53,10 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend): def supports_sink(cls) -> bool: return True + @classmethod + def supports_non_causal(cls) -> bool: + return False + forward_includes_kv_cache_update: bool = False @staticmethod @@ -140,7 +144,7 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, - attn_metadata: FlashAttentionMetadata, + attn_metadata: RocmAttentionMetadata, output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, diff --git a/vllm/v1/attention/backends/rocm_attn.py b/vllm/v1/attention/backends/rocm_attn.py index 3a906233272..a238ff4ad53 100644 --- a/vllm/v1/attention/backends/rocm_attn.py +++ b/vllm/v1/attention/backends/rocm_attn.py @@ -27,7 +27,6 @@ from vllm.v1.attention.backend import ( CommonAttentionMetadata, MultipleOf, ) -from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata from vllm.v1.attention.ops.chunked_prefill_paged_decode import ( chunked_prefill_paged_decode, ) @@ -69,6 +68,9 @@ class RocmAttentionMetadata: scheduler_metadata: torch.Tensor | None = None prefix_scheduler_metadata: torch.Tensor | None = None + # DFlash drafting sets this to False via CommonAttentionMetadata. + causal: bool = True + class RocmAttentionMetadataBuilder(AttentionMetadataBuilder[RocmAttentionMetadata]): _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.ALWAYS @@ -154,6 +156,7 @@ class RocmAttentionMetadataBuilder(AttentionMetadataBuilder[RocmAttentionMetadat prefix_kv_lens=prefix_kv_lens, suffix_kv_lens=suffix_kv_lens, prefix_scheduler_metadata=prefix_scheduler_metadata, + causal=common_attn_metadata.causal, ) return attn_metadata @@ -200,6 +203,10 @@ class RocmAttentionBackend(AttentionBackend): # kernel, which is less efficient than the proper triton backends. return False + @classmethod + def supports_non_causal(cls) -> bool: + return True + forward_includes_kv_cache_update: bool = False @staticmethod @@ -301,7 +308,7 @@ class RocmAttentionImpl(AttentionImpl): key: torch.Tensor, value: torch.Tensor, output: torch.Tensor, - attn_metadata: FlashAttentionMetadata, + attn_metadata: RocmAttentionMetadata, layer: torch.nn.Module, ) -> torch.Tensor: """Forward pass for encoder attention without KV cache. @@ -350,7 +357,7 @@ class RocmAttentionImpl(AttentionImpl): key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, - attn_metadata: FlashAttentionMetadata, + attn_metadata: RocmAttentionMetadata, output: torch.Tensor, output_scale: torch.Tensor | None = None, output_block_scale: torch.Tensor | None = None, @@ -438,6 +445,7 @@ class RocmAttentionImpl(AttentionImpl): sm_scale=self.scale, output_scale=output_scale, sinks=self.sinks, + causal=attn_metadata.causal, ) return output diff --git a/vllm/v1/attention/ops/chunked_prefill_paged_decode.py b/vllm/v1/attention/ops/chunked_prefill_paged_decode.py index 000fd4d43b9..ea1f075ef65 100644 --- a/vllm/v1/attention/ops/chunked_prefill_paged_decode.py +++ b/vllm/v1/attention/ops/chunked_prefill_paged_decode.py @@ -269,6 +269,7 @@ def chunked_prefill_paged_decode( # Optional tensor for sinks sinks=None, is_block_table_ptr: bool = False, + causal: bool = True, ): if sm_scale is None: sm_scale = 1.0 / (query.shape[2] ** 0.5) @@ -300,6 +301,7 @@ def chunked_prefill_paged_decode( skip_decode=True, fp8_out_scale=output_scale, sinks=sinks, + causal=causal, ) block_size = value_cache.shape[3] diff --git a/vllm/v1/attention/ops/prefix_prefill.py b/vllm/v1/attention/ops/prefix_prefill.py index afa5f517838..8488c72aeaf 100644 --- a/vllm/v1/attention/ops/prefix_prefill.py +++ b/vllm/v1/attention/ops/prefix_prefill.py @@ -89,6 +89,7 @@ def _fwd_kernel( SKIP_DECODE: tl.constexpr, USE_SINKS: tl.constexpr, USE_FP8: tl.constexpr, + CAUSAL: tl.constexpr = True, MAX_Q_LEN: tl.constexpr = 0, MAX_CTX_LEN: tl.constexpr = 0, FP8_MIN: tl.constexpr = float8_info.min, @@ -283,10 +284,17 @@ def _fwd_kernel( # block_mask is 0 when we're already past the current query length block_mask = tl.where(block_start_loc < cur_batch_query_len, 1, 0) - # compute query against itself (with causal mask) + # compute query against itself (causal among queries by default; + # CAUSAL=False for bidirectional attention over query tokens, e.g. DFlash.) + if CAUSAL: + key_range_upper = block_mask * (start_m + 1) * BLOCK_M + else: + q_len_pad = (cur_batch_query_len + BLOCK_N - 1) // BLOCK_N * BLOCK_N + key_range_upper = block_mask * q_len_pad + for start_n in tl.range( 0, - block_mask * (start_m + 1) * BLOCK_M, + key_range_upper, BLOCK_N, loop_unroll_factor=num_unroll_request, ): @@ -302,14 +310,17 @@ def _fwd_kernel( qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) qk = tl.dot(q, k, acc=qk, input_precision=IN_PRECISION) qk *= sm_scale - # apply causal mask - qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk, float("-inf")) + + valid_kv = (start_n + offs_n[None, :]) < cur_batch_query_len + if CAUSAL: + attn_mask = valid_kv & (offs_m[:, None] >= (start_n + offs_n[None, :])) + else: + attn_mask = valid_kv if SLIDING_WINDOW > 0: - qk = tl.where( - offs_m[:, None] - (start_n + offs_n[None, :]) < SLIDING_WINDOW, - qk, - float("-inf"), + attn_mask = attn_mask & ( + offs_m[:, None] - (start_n + offs_n[None, :]) < SLIDING_WINDOW ) + qk = tl.where(attn_mask, qk, float("-inf")) # compute running maximum m_ij = tl.maximum(m_i, tl.max(qk, axis=1)) @@ -656,6 +667,7 @@ def context_attention_fwd( fp8_out_scale=None, sinks=None, is_block_table_ptr: bool = False, + causal: bool = True, ): q_dtype_is_f32 = q.dtype is torch.float32 @@ -722,6 +734,7 @@ def context_attention_fwd( processed_b_loc = b_loc.to(torch.int32) if alibi_slopes is not None: + assert causal, "Non-causal prefix attention is not supported with alibi" assert sinks is None, "Sinks arg is not supported with alibi" assert fp8_out_scale is None, "FP8 output not supported with alibi" # need to reduce num. blocks when using fp32 @@ -859,6 +872,7 @@ def context_attention_fwd( num_warps=4, num_stages=1, USE_SINKS=sinks is not None, + CAUSAL=causal, **extra_kargs, ) return From 4eafc729285e459a5fc96efd6f7b313b155cad48 Mon Sep 17 00:00:00 2001 From: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Date: Wed, 22 Apr 2026 00:24:18 -0400 Subject: [PATCH 026/153] [Audio] Bundle `get_generation_prompt()` params into `SpeechToTextParams` (#36268) Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung --- docs/contributing/model/transcription.md | 39 +++++++------- vllm/config/__init__.py | 3 +- vllm/config/speech_to_text.py | 40 ++++++++++++++ .../openai/speech_to_text/protocol.py | 54 ++++++++++++++++++- .../openai/speech_to_text/speech_to_text.py | 18 +++---- vllm/model_executor/models/cohere_asr.py | 22 ++++---- vllm/model_executor/models/fireredasr2.py | 16 +++--- vllm/model_executor/models/funasr.py | 16 +++--- vllm/model_executor/models/gemma3n_mm.py | 18 +++---- vllm/model_executor/models/glmasr.py | 17 +++--- vllm/model_executor/models/granite_speech.py | 17 +++--- vllm/model_executor/models/interfaces.py | 10 +--- vllm/model_executor/models/kimi_audio.py | 17 +++--- vllm/model_executor/models/qwen3_asr.py | 19 +++---- .../models/qwen3_omni_moe_thinker.py | 21 ++++---- vllm/model_executor/models/voxtral.py | 16 +++--- .../model_executor/models/voxtral_realtime.py | 15 +++--- vllm/model_executor/models/whisper.py | 17 +++--- 18 files changed, 214 insertions(+), 161 deletions(-) diff --git a/docs/contributing/model/transcription.md b/docs/contributing/model/transcription.md index db868686e2a..3e2ee38d2bd 100644 --- a/docs/contributing/model/transcription.md +++ b/docs/contributing/model/transcription.md @@ -66,7 +66,7 @@ This is for controlling general behavior of the API when serving your model: See [Audio preprocessing and chunking](#audio-preprocessing-and-chunking) for what each field controls. -Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server passes you the resampled waveform and task parameters; you return a valid [PromptType][vllm.inputs.llm.PromptType]. There are two common patterns: +Implement the prompt construction via [get_generation_prompt][vllm.model_executor.models.interfaces.SupportsTranscription.get_generation_prompt]. The server builds a [SpeechToTextParams][vllm.config.speech_to_text.SpeechToTextParams] object that bundles the resampled waveform, task parameters, and request-specific options. Your model receives this single object and returns a valid [PromptType][vllm.inputs.llm.PromptType]. There are two common patterns: #### Multimodal LLM with audio embeddings (e.g., Voxtral, Gemma3n) @@ -75,21 +75,20 @@ Return a dict containing `multi_modal_data` with the audio, and either a `prompt ??? code "get_generation_prompt()" ```python + from vllm.config.speech_to_text import SpeechToTextParams + class YourASRModel(nn.Module, SupportsTranscription): ... @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - stt_config: SpeechToTextConfig, - model_config: ModelConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: - # Example with a free-form instruction prompt + audio = stt_params.audio + stt_config = stt_params.stt_config + task_type = stt_params.task_type + task_word = "Transcribe" if task_type == "transcribe" else "Translate" prompt = ( "user\n" @@ -112,20 +111,22 @@ Return a dict with separate `encoder_prompt` and `decoder_prompt` entries: ??? code "get_generation_prompt()" ```python + from vllm.config.speech_to_text import SpeechToTextParams + class YourASRModel(nn.Module, SupportsTranscription): ... @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - stt_config: SpeechToTextConfig, - model_config: ModelConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: + audio = stt_params.audio + stt_config = stt_params.stt_config + language = stt_params.language + task_type = stt_params.task_type + request_prompt = stt_params.request_prompt + if language is None: raise ValueError("Language must be specified") @@ -213,15 +214,13 @@ Relevant server logic: chunks = [y] if not do_split_audio else self._split_audio(y, int(sr)) prompts = [] for chunk in chunks: - prompt = self.model_cls.get_generation_prompt( + stt_params = request.build_stt_params( audio=chunk, stt_config=self.asr_config, model_config=self.model_config, - language=language, task_type=self.task_type, - request_prompt=request.prompt, - to_language=to_language, ) + prompt = self.model_cls.get_generation_prompt(stt_params) prompts.append(prompt) return prompts, duration ``` diff --git a/vllm/config/__init__.py b/vllm/config/__init__.py index 758605d25c6..b189c45c8d7 100644 --- a/vllm/config/__init__.py +++ b/vllm/config/__init__.py @@ -37,7 +37,7 @@ from vllm.config.profiler import ProfilerConfig from vllm.config.reasoning import ReasoningConfig from vllm.config.scheduler import SchedulerConfig from vllm.config.speculative import SpeculativeConfig -from vllm.config.speech_to_text import SpeechToTextConfig +from vllm.config.speech_to_text import SpeechToTextConfig, SpeechToTextParams from vllm.config.structured_outputs import StructuredOutputsConfig from vllm.config.utils import ( ConfigType, @@ -113,6 +113,7 @@ __all__ = [ "SpeculativeConfig", # From vllm.config.speech_to_text "SpeechToTextConfig", + "SpeechToTextParams", # From vllm.config.structured_outputs "StructuredOutputsConfig", # From vllm.config.profiler diff --git a/vllm/config/speech_to_text.py b/vllm/config/speech_to_text.py index e0d72eb203a..37350e86126 100644 --- a/vllm/config/speech_to_text.py +++ b/vllm/config/speech_to_text.py @@ -1,9 +1,49 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations +from dataclasses import dataclass +from typing import TYPE_CHECKING from vllm.config.utils import config +if TYPE_CHECKING: + import numpy as np + + from vllm.config.model import ModelConfig + + +@dataclass +class SpeechToTextParams: + """All parameters consumed by ``get_generation_prompt()``. + + ``TranscriptionRequest.build_stt_params()`` constructs this object, + mapping API-level fields into typed attributes. Models only receive + this object, so new parameters can be added here without changing the + ``get_generation_prompt`` signature. + """ + + audio: np.ndarray + """Resampled audio waveform for a single chunk.""" + + stt_config: SpeechToTextConfig + """Server-level speech-to-text configuration.""" + + model_config: ModelConfig + """Model configuration.""" + + language: str | None = None + """ISO 639-1 language code (validated / auto-detected).""" + + task_type: str = "transcribe" + """``"transcribe"`` or ``"translate"``.""" + + request_prompt: str = "" + """Optional text prompt to guide the model.""" + + to_language: str | None = None + """Target language for translation (model-dependent).""" + @config class SpeechToTextConfig: diff --git a/vllm/entrypoints/openai/speech_to_text/protocol.py b/vllm/entrypoints/openai/speech_to_text/protocol.py index a8d978e33eb..623ae3fc2c2 100644 --- a/vllm/entrypoints/openai/speech_to_text/protocol.py +++ b/vllm/entrypoints/openai/speech_to_text/protocol.py @@ -1,9 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json import time from http import HTTPStatus -from typing import Literal, TypeAlias +from typing import TYPE_CHECKING, Literal, TypeAlias import torch from fastapi import HTTPException, UploadFile @@ -12,6 +13,7 @@ from pydantic import ( model_validator, ) +from vllm.config.speech_to_text import SpeechToTextParams from vllm.entrypoints.openai.engine.protocol import ( DeltaMessage, OpenAIBaseModel, @@ -26,6 +28,11 @@ from vllm.sampling_params import ( ) from vllm.utils import random_uuid +if TYPE_CHECKING: + import numpy as np + + from vllm.config import ModelConfig, SpeechToTextConfig + logger = init_logger(__name__) _LONG_INFO = torch.iinfo(torch.long) @@ -183,6 +190,23 @@ class TranscriptionRequest(OpenAIBaseModel): "min_p": 0.0, } + def build_stt_params( + self, + audio: "np.ndarray", + stt_config: "SpeechToTextConfig", + model_config: "ModelConfig", + task_type: str, + ) -> SpeechToTextParams: + return SpeechToTextParams( + audio=audio, + stt_config=stt_config, + model_config=model_config, + language=self.language, + task_type=task_type, + request_prompt=self.prompt, + to_language=self.to_language, + ) + def to_beam_search_params( self, default_max_tokens: int, @@ -277,6 +301,17 @@ class TranscriptionRequest(OpenAIBaseModel): parameter=invalid_param, ) + # Parse vllm_xargs from JSON string (form data sends it as a string) + xargs = data.get("vllm_xargs") + if isinstance(xargs, str): + try: + data["vllm_xargs"] = json.loads(xargs) + except json.JSONDecodeError as e: + raise VLLMValidationError( + f"Failed to parse vllm_xargs. Must be valid JSON: {e}", + parameter="vllm_xargs", + ) from e + return data @@ -472,6 +507,23 @@ class TranslationRequest(OpenAIBaseModel): "temperature": 0, } + def build_stt_params( + self, + audio: "np.ndarray", + stt_config: "SpeechToTextConfig", + model_config: "ModelConfig", + task_type: str, + ) -> SpeechToTextParams: + return SpeechToTextParams( + audio=audio, + stt_config=stt_config, + model_config=model_config, + language=self.language, + task_type=task_type, + request_prompt=self.prompt, + to_language=self.to_language, + ) + def to_beam_search_params( self, default_max_tokens: int, diff --git a/vllm/entrypoints/openai/speech_to_text/speech_to_text.py b/vllm/entrypoints/openai/speech_to_text/speech_to_text.py index 4ebc612a415..c4c10c35f3c 100644 --- a/vllm/entrypoints/openai/speech_to_text/speech_to_text.py +++ b/vllm/entrypoints/openai/speech_to_text/speech_to_text.py @@ -184,9 +184,8 @@ class OpenAISpeechToText(OpenAIServing): request_id: str, ) -> tuple[list[EngineInput], float]: # Validate request - language = self.model_cls.validate_language(request.language) - # Skip to_language validation to avoid extra logging for Whisper. - to_language = ( + request.language = self.model_cls.validate_language(request.language) + request.to_language = ( self.model_cls.validate_language(request.to_language) if request.to_language else None @@ -229,28 +228,23 @@ class OpenAISpeechToText(OpenAIServing): min_energy_window_size=self.asr_config.min_energy_split_window_size, ) - if language is None and getattr( + if request.language is None and getattr( self.model_cls, "supports_explicit_language_detection", False ): # Auto-detect language from the first chunk. - language = await self._detect_language( + request.language = await self._detect_language( chunks[0], f"{request_id}-lang_detect" ) - request.language = language parsed_prompts: list[DictPrompt] = [] for chunk in chunks: - # The model has control over the construction, as long as it - # returns a valid PromptType. - prompt = self.model_cls.get_generation_prompt( + stt_params = request.build_stt_params( audio=chunk, stt_config=self.asr_config, model_config=self.model_config, - language=language, task_type=self.task_type, - request_prompt=request.prompt, - to_language=to_language, ) + prompt = self.model_cls.get_generation_prompt(stt_params) parsed_prompt: DictPrompt if request.response_format == "verbose_json": diff --git a/vllm/model_executor/models/cohere_asr.py b/vllm/model_executor/models/cohere_asr.py index 42206c11cb9..81ba1483bff 100644 --- a/vllm/model_executor/models/cohere_asr.py +++ b/vllm/model_executor/models/cohere_asr.py @@ -3,9 +3,7 @@ import math from collections.abc import Iterable, Mapping, Sequence -from typing import Literal -import numpy as np import torch import torch.nn.functional as F from torch import nn @@ -13,6 +11,8 @@ from transformers import PretrainedConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig +from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt from vllm.logger import init_logger @@ -1900,7 +1900,7 @@ class CohereASRDummyInputsBuilder(BaseDummyInputsBuilder[CohereASRProcessingInfo self, seq_len: int, mm_counts: Mapping[str, int], - mm_options=None, + mm_options: Mapping[str, BaseDummyOptions], mm_processor_kwargs=None, ) -> MultiModalDataDict: feature_extractor = self.info.get_feature_extractor() @@ -2021,16 +2021,12 @@ class CohereAsrForConditionalGeneration( return super().validate_language(language) @classmethod - def get_generation_prompt( - cls, - audio: np.ndarray, - model_config: ModelConfig, # not needed here - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, - ) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: + audio = stt_params.audio + stt_config = stt_params.stt_config + language = stt_params.language + request_prompt = stt_params.request_prompt + if language is None: raise ValueError( "Language must be specified when creating the CohereASR prompt" diff --git a/vllm/model_executor/models/fireredasr2.py b/vllm/model_executor/models/fireredasr2.py index 41b4318504f..eea0c7d8897 100644 --- a/vllm/model_executor/models/fireredasr2.py +++ b/vllm/model_executor/models/fireredasr2.py @@ -2,9 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal, cast +from typing import Annotated, cast -import numpy as np import torch from torch import nn from transformers import ( @@ -14,6 +13,7 @@ from transformers import ( from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType from vllm.logger import init_logger from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY @@ -356,14 +356,12 @@ class FireRedASR2ForConditionalGeneration( @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - model_config: ModelConfig, # not needed here - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: + audio = stt_params.audio + stt_config = stt_params.stt_config + language = stt_params.language + if language is None: raise ValueError( "Language must be specified when creating the fireredasr2 prompt" diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index 98313db7980..ab9d532f73b 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -3,9 +3,8 @@ import math from collections.abc import Iterable, Mapping, Sequence -from typing import Annotated, Literal, cast +from typing import Annotated, cast -import numpy as np import torch import torch.nn.functional as F from torch import nn @@ -16,6 +15,7 @@ from transformers import ( from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict, PromptType from vllm.logger import init_logger @@ -876,14 +876,12 @@ class FunASRForConditionalGeneration( @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - model_config: ModelConfig, # not needed here - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: + audio = stt_params.audio + stt_config = stt_params.stt_config + language = stt_params.language + if language is None: raise ValueError( "Language must be specified when creating the funasr prompt" diff --git a/vllm/model_executor/models/gemma3n_mm.py b/vllm/model_executor/models/gemma3n_mm.py index 342d6c476df..4e9838805e5 100644 --- a/vllm/model_executor/models/gemma3n_mm.py +++ b/vllm/model_executor/models/gemma3n_mm.py @@ -3,7 +3,6 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Any, Literal -import numpy as np import torch from torch import nn from transformers import AutoModel, BatchFeature @@ -19,6 +18,7 @@ from transformers.models.siglip import SiglipImageProcessorFast from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm @@ -769,21 +769,17 @@ class Gemma3nForConditionalGeneration( raise ValueError(f"Unsupported modality: {modality}") @classmethod - def get_generation_prompt( - cls, - audio: np.ndarray, - stt_config: SpeechToTextConfig, - model_config: ModelConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, - ) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: """ Gemma3n supports "free-form" transcription. We fix its prompt here to standardize transcriptions/translations requests. """ + audio = stt_params.audio + stt_config = stt_params.stt_config + language = stt_params.language + task_type = stt_params.task_type + to_language = stt_params.to_language # Transcribe this audio [into <>] | for transcription # Translate this audio [from <> into <>] | for translation prompt = "user\n" diff --git a/vllm/model_executor/models/glmasr.py b/vllm/model_executor/models/glmasr.py index 96be79a3093..cd168b6b461 100644 --- a/vllm/model_executor/models/glmasr.py +++ b/vllm/model_executor/models/glmasr.py @@ -4,7 +4,6 @@ from collections.abc import Iterable, Mapping, Sequence from typing import Annotated, Any, Literal, TypeAlias -import numpy as np import torch import torch.nn as nn from transformers import BatchFeature @@ -13,6 +12,7 @@ from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt from vllm.model_executor.layers.activation import get_act_fn @@ -1131,17 +1131,12 @@ class GlmAsrForConditionalGeneration( ) @classmethod - def get_generation_prompt( - cls, - audio: np.ndarray, - model_config: ModelConfig, - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, - ) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: """Get the generation prompt to be used for transcription requests.""" + audio = stt_params.audio + model_config = stt_params.model_config + task_type = stt_params.task_type + to_language = stt_params.to_language tokenizer = cached_tokenizer_from_config(model_config) audio_token = cls._get_audio_token(model_config) diff --git a/vllm/model_executor/models/granite_speech.py b/vllm/model_executor/models/granite_speech.py index dca54425c70..036b92ed880 100644 --- a/vllm/model_executor/models/granite_speech.py +++ b/vllm/model_executor/models/granite_speech.py @@ -26,9 +26,8 @@ import math from collections.abc import Iterable, Mapping -from typing import Annotated, Literal +from typing import Annotated -import numpy as np import torch import torch.nn.functional as F from torch import nn @@ -36,6 +35,7 @@ from transformers import BatchFeature, PretrainedConfig from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.quantization import QuantizationConfig @@ -852,15 +852,14 @@ class GraniteSpeechForConditionalGeneration( @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - model_config: ModelConfig, - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: """Get the generation prompt to be used for transcription requests.""" + audio = stt_params.audio + model_config = stt_params.model_config + task_type = stt_params.task_type + to_language = stt_params.to_language + # Audio placeholders don't use an index, so value doesn't matter audio_tok = cls.get_placeholder_str("audio", 0) diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 3ee8ac23d72..7caf4c22075 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -29,7 +29,7 @@ from torch import Tensor from transformers.models.whisper.tokenization_whisper import LANGUAGES from typing_extensions import Self, TypeIs -from vllm.config import ModelConfig, SpeechToTextConfig +from vllm.config import ModelConfig, SpeechToTextConfig, SpeechToTextParams from vllm.inputs import PromptType, TokensPrompt from vllm.logger import init_logger from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc @@ -1119,13 +1119,7 @@ class SupportsTranscription(Protocol): @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - stt_config: SpeechToTextConfig, - model_config: ModelConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: """Get the prompt for the ASR model. The model has control over the construction, as long as it diff --git a/vllm/model_executor/models/kimi_audio.py b/vllm/model_executor/models/kimi_audio.py index fc5065065e9..475661e0eba 100644 --- a/vllm/model_executor/models/kimi_audio.py +++ b/vllm/model_executor/models/kimi_audio.py @@ -14,6 +14,7 @@ from transformers import WhisperConfig as HFWhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import PromptType, TokensPrompt from vllm.model_executor.model_loader import DefaultModelLoader from vllm.model_executor.model_loader.weight_utils import default_weight_loader @@ -626,16 +627,12 @@ class KimiAudioForConditionalGeneration( ) @classmethod - def get_generation_prompt( - cls, - audio: np.ndarray, - model_config: ModelConfig, - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, - ) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: + audio = stt_params.audio + model_config = stt_params.model_config + task_type = stt_params.task_type + request_prompt = stt_params.request_prompt + tokenizer = cached_get_tokenizer( model_config.tokenizer, tokenizer_cls=KimiAudioTokenizer, diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index 37903462dd4..950beba7754 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -23,9 +23,8 @@ """Inference-only Qwen3-ASR model.""" from collections.abc import Iterable, Mapping, Sequence -from typing import Any, Literal +from typing import Any -import numpy as np import torch import torch.nn as nn from transformers.feature_extraction_utils import BatchFeature @@ -33,6 +32,7 @@ from transformers.models.whisper import WhisperFeatureExtractor from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt from vllm.logger import init_logger from vllm.model_executor.models.interfaces import ( @@ -549,17 +549,12 @@ class Qwen3ASRForConditionalGeneration( ) @classmethod - def get_generation_prompt( - cls, - audio: np.ndarray, - model_config: ModelConfig, - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, - ) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: """Get the generation prompt to be used for transcription requests.""" + audio = stt_params.audio + model_config = stt_params.model_config + task_type = stt_params.task_type + to_language = stt_params.to_language tokenizer = cached_tokenizer_from_config(model_config) audio_placeholder = cls.get_placeholder_str("audio", 0) diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index b4842e06388..fa5df4a1f92 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -24,7 +24,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from functools import partial -from typing import Any, Literal, cast +from typing import Any, cast import numpy as np import torch @@ -46,6 +46,7 @@ from transformers.models.whisper import WhisperFeatureExtractor from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig +from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.inputs import PromptType from vllm.logger import init_logger @@ -2201,19 +2202,17 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ) @classmethod - def get_generation_prompt( - cls, - audio: np.ndarray, - stt_config: SpeechToTextConfig, - model_config: ModelConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, - ) -> PromptType: + def get_generation_prompt(cls, stt_params: SpeechToTextParams) -> PromptType: """ Construct a transcription/translation prompt for Qwen3-Omni. """ + audio = stt_params.audio + stt_config = stt_params.stt_config + model_config = stt_params.model_config + language = stt_params.language + task_type = stt_params.task_type + to_language = stt_params.to_language + request_prompt = stt_params.request_prompt # Transcribe this audio [into ] | for transcription # Translate this audio [from into ] | for translation instruction = "Transcribe" if task_type == "transcribe" else "Translate" diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index d44960ca811..42e811edaec 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -4,7 +4,7 @@ import math from collections.abc import Iterable, Mapping, Sequence from functools import partial -from typing import Literal, cast +from typing import cast import numpy as np import regex as re @@ -19,6 +19,7 @@ from transformers import BatchFeature, WhisperConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt from vllm.logger import init_logger from vllm.model_executor.layers.quantization import QuantizationConfig @@ -446,14 +447,13 @@ class VoxtralForConditionalGeneration( # for speech-to-text transcription def get_generation_prompt( cls, - audio: np.ndarray, - model_config: ModelConfig, - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: + audio = stt_params.audio + model_config = stt_params.model_config + stt_config = stt_params.stt_config + language = stt_params.language + tokenizer = cached_tokenizer_from_config(model_config) audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless req = TranscriptionRequest( diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index b70714a0d83..2628e1443e2 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -4,7 +4,6 @@ import asyncio import math from collections.abc import AsyncGenerator, Iterable, Iterator, Mapping -from typing import Literal import numpy as np import torch @@ -18,6 +17,7 @@ from mistral_common.tokens.tokenizers.audio import AudioConfig from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig +from vllm.config.speech_to_text import SpeechToTextParams from vllm.engine.protocol import StreamingInput from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S from vllm.inputs import PromptType, TokensPrompt @@ -465,14 +465,13 @@ class VoxtralRealtimeGeneration(VoxtralForConditionalGeneration, SupportsRealtim # for speech-to-text transcription def get_generation_prompt( cls, - audio: np.ndarray, - model_config: ModelConfig, - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: + audio = stt_params.audio + model_config = stt_params.model_config + stt_config = stt_params.stt_config + language = stt_params.language + tokenizer = cached_tokenizer_from_config(model_config) audio = Audio(audio, int(stt_config.sample_rate), format="wav") # lossless diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index f0f6f619b02..628186e7598 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -5,7 +5,7 @@ import enum import math from collections.abc import Iterable, Mapping, Sequence from contextlib import nullcontext -from typing import Annotated, Literal +from typing import Annotated import numpy as np import torch @@ -20,6 +20,7 @@ from transformers.models.whisper.modeling_whisper import sinusoids from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.multimodal import BaseDummyOptions +from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import ( ExplicitEncoderDecoderPrompt, @@ -830,14 +831,14 @@ class WhisperForConditionalGeneration( @classmethod def get_generation_prompt( cls, - audio: np.ndarray, - model_config: ModelConfig, # not needed here - stt_config: SpeechToTextConfig, - language: str | None, - task_type: Literal["transcribe", "translate"], - request_prompt: str, - to_language: str | None, + stt_params: SpeechToTextParams, ) -> PromptType: + audio = stt_params.audio + stt_config = stt_params.stt_config + language = stt_params.language + task_type = stt_params.task_type + request_prompt = stt_params.request_prompt + if language is None: raise ValueError( "Language must be specified when creating the Whisper prompt" From 0210024ae796446a121f96d2d31053668ac0fd85 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 22 Apr 2026 13:17:51 +0800 Subject: [PATCH 027/153] [Bugfix] Pass effective chat template kwargs to reasoning parsers (#40460) Signed-off-by: Bugen Zhao --- .../openai/chat_completion/batch_serving.py | 11 ++++--- .../openai/chat_completion/serving.py | 17 +++++++--- .../openai/parser/responses_parser.py | 33 ++++++++++++++++--- vllm/entrypoints/openai/responses/context.py | 5 +-- vllm/entrypoints/openai/responses/serving.py | 18 ++++++++-- vllm/entrypoints/serve/render/serving.py | 4 ++- 6 files changed, 71 insertions(+), 17 deletions(-) diff --git a/vllm/entrypoints/openai/chat_completion/batch_serving.py b/vllm/entrypoints/openai/chat_completion/batch_serving.py index f97c93bb03c..b8b26d393c7 100644 --- a/vllm/entrypoints/openai/chat_completion/batch_serving.py +++ b/vllm/entrypoints/openai/chat_completion/batch_serving.py @@ -114,12 +114,15 @@ class OpenAIServingChatBatch(OpenAIServingChat): """ tokenizer = self.renderer.tokenizer assert tokenizer is not None + single_requests = [ + request.to_chat_completion_request(messages) + for messages in request.messages + ] reasoning_parser: ReasoningParser | None = None if self.reasoning_parser_cls: - chat_template_kwargs = self._prepare_extra_chat_template_kwargs( - request.chat_template_kwargs, - self.default_chat_template_kwargs, + chat_template_kwargs = self._effective_chat_template_kwargs( + single_requests[0] ) reasoning_parser = self.reasoning_parser_cls( tokenizer, @@ -155,7 +158,7 @@ class OpenAIServingChatBatch(OpenAIServingChat): self.default_sampling_params, self.override_max_tokens, ) - single_request = request.to_chat_completion_request(request.messages[i]) + single_request = single_requests[i] sampling_params = single_request.to_sampling_params( max_tokens, self.default_sampling_params ) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 4a56c63ecd4..fd8a5a66029 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -189,6 +189,18 @@ class OpenAIServingChat(OpenAIServing): ) ) + def _effective_chat_template_kwargs( + self, request: ChatCompletionRequest + ) -> dict[str, Any]: + return ( + request.build_chat_params( + self.chat_template, + self.chat_template_content_format, + ) + .with_defaults(self.default_chat_template_kwargs) + .chat_template_kwargs + ) + async def render_chat_request( self, request: ChatCompletionRequest, @@ -231,10 +243,7 @@ class OpenAIServingChat(OpenAIServing): # Streaming response tokenizer = self.renderer.tokenizer assert tokenizer is not None - chat_template_kwargs = self._prepare_extra_chat_template_kwargs( - request.chat_template_kwargs, - self.default_chat_template_kwargs, - ) + chat_template_kwargs = self._effective_chat_template_kwargs(request) reasoning_parser: ReasoningParser | None = None if self.reasoning_parser_cls: reasoning_parser = self.reasoning_parser_cls( diff --git a/vllm/entrypoints/openai/parser/responses_parser.py b/vllm/entrypoints/openai/parser/responses_parser.py index a31f20501e0..1868a31ca28 100644 --- a/vllm/entrypoints/openai/parser/responses_parser.py +++ b/vllm/entrypoints/openai/parser/responses_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import logging -from collections.abc import Callable +from typing import Any from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem from openai.types.responses.response_function_tool_call_output_item import ( @@ -15,6 +15,7 @@ from openai.types.responses.response_reasoning_item import ( ResponseReasoningItem, ) +from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.entrypoints.constants import MCP_PREFIX from vllm.entrypoints.openai.responses.protocol import ( ResponseInputOutputItem, @@ -36,10 +37,12 @@ class ResponsesParser: self, *, tokenizer: TokenizerLike, - reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser], + reasoning_parser_cls: type[ReasoningParser], response_messages: list[ResponseInputOutputItem], request: ResponsesRequest, tool_parser_cls: type[ToolParser] | None, + chat_template: str | None, + chat_template_content_format: ChatTemplateContentFormatOption, ): self.response_messages: list[ResponseInputOutputItem] = ( # TODO: initial messages may not be properly typed @@ -49,7 +52,14 @@ class ResponsesParser: self.tokenizer = tokenizer self.request = request - self.reasoning_parser_instance = reasoning_parser_cls(tokenizer) + self.reasoning_parser_instance = reasoning_parser_cls( + tokenizer, + chat_template_kwargs=_effective_chat_template_kwargs( + request, + chat_template=chat_template, + chat_template_content_format=chat_template_content_format, + ), + ) self.tool_parser_instance = None if tool_parser_cls is not None: self.tool_parser_instance = tool_parser_cls(tokenizer, request.tools) @@ -159,10 +169,12 @@ class ResponsesParser: def get_responses_parser_for_simple_context( *, tokenizer: TokenizerLike, - reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser], + reasoning_parser_cls: type[ReasoningParser], response_messages: list[ResponseInputOutputItem], request: ResponsesRequest, tool_parser_cls, + chat_template: str | None, + chat_template_content_format: ChatTemplateContentFormatOption, ) -> ResponsesParser: """Factory function to create a ResponsesParser with optional reasoning parser. @@ -176,4 +188,17 @@ def get_responses_parser_for_simple_context( response_messages=response_messages, request=request, tool_parser_cls=tool_parser_cls, + chat_template=chat_template, + chat_template_content_format=chat_template_content_format, ) + + +def _effective_chat_template_kwargs( + request: ResponsesRequest, + chat_template: str | None, + chat_template_content_format: ChatTemplateContentFormatOption, +) -> dict[str, Any]: + return request.build_chat_params( + default_template=chat_template, + default_template_content_format=chat_template_content_format, + ).chat_template_kwargs diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 48360173cf4..f0920ab09f4 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -6,7 +6,6 @@ import copy import json import logging from abc import ABC, abstractmethod -from collections.abc import Callable from contextlib import AsyncExitStack from dataclasses import replace from typing import TYPE_CHECKING, Any, Final, Union @@ -273,7 +272,7 @@ class ParsableContext(ConversationContext): *, response_messages: list[ResponseInputOutputItem], tokenizer: TokenizerLike, - reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None, + reasoning_parser_cls: type[ReasoningParser] | None, request: ResponsesRequest, available_tools: list[str] | None, tool_parser_cls: type[ToolParser] | None, @@ -296,6 +295,8 @@ class ParsableContext(ConversationContext): response_messages=response_messages, request=request, tool_parser_cls=tool_parser_cls, + chat_template=chat_template, + chat_template_content_format=chat_template_content_format, ) self.tool_parser_cls = tool_parser_cls self.request = request diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 6a0c4c1e9b6..6af25c9bcce 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -267,6 +267,14 @@ class OpenAIServingResponses(OpenAIServing): self.tool_server = tool_server + def _effective_chat_template_kwargs( + self, request: ResponsesRequest + ) -> dict[str, Any]: + return request.build_chat_params( + self.chat_template, + self.chat_template_content_format, + ).chat_template_kwargs + def _validate_generator_input( self, engine_input: EngineInput, @@ -464,7 +472,10 @@ class OpenAIServingResponses(OpenAIServing): context = SimpleContext() if self.parser and self.parser.reasoning_parser_cls is not None: - reasoning_parser = self.parser.reasoning_parser_cls(tokenizer) + reasoning_parser = self.parser.reasoning_parser_cls( + tokenizer, + chat_template_kwargs=self._effective_chat_template_kwargs(request), + ) if ( isinstance( struct_out := sampling_params.structured_outputs, @@ -835,7 +846,10 @@ class OpenAIServingResponses(OpenAIServing): and self.parser.reasoning_parser_cls is not None and isinstance(context, (SimpleContext, ParsableContext)) ): - reasoning_parser = self.parser.reasoning_parser_cls(tokenizer) + reasoning_parser = self.parser.reasoning_parser_cls( + tokenizer, + chat_template_kwargs=self._effective_chat_template_kwargs(request), + ) accumulated = getattr(context, "_accumulated_token_ids", []) or [] num_reasoning_tokens = reasoning_parser.count_reasoning_tokens(accumulated) diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/serve/render/serving.py index 3cbf3cc90cc..25c5a6d199e 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/serve/render/serving.py @@ -566,7 +566,9 @@ class OpenAIServingRender: if reasoning_parser is not None: tokenizer = renderer.get_tokenizer() request = reasoning_parser( - tokenizer, model_config=self.model_config + tokenizer, + model_config=self.model_config, + chat_template_kwargs=chat_params.chat_template_kwargs, ).adjust_request(request=request) # tool parsing is done only if a tool_parser has been set and if From aad88f84862b29917241a4b67774186956459350 Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Wed, 22 Apr 2026 08:44:00 +0300 Subject: [PATCH 028/153] [kv_offload+HMA][8/N]: Support multi-group worker transfer (#38453) Signed-off-by: Or Ozeri Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/v1/kv_offload/test_cpu_gpu.py | 246 +++++++++++++++++- .../kv_connector/v1/offloading/scheduler.py | 4 +- vllm/v1/kv_offload/mediums.py | 10 +- vllm/v1/kv_offload/worker/cpu_gpu.py | 154 +++++++---- 4 files changed, 341 insertions(+), 73 deletions(-) diff --git a/tests/v1/kv_offload/test_cpu_gpu.py b/tests/v1/kv_offload/test_cpu_gpu.py index de482aec4a4..db851edbccb 100644 --- a/tests/v1/kv_offload/test_cpu_gpu.py +++ b/tests/v1/kv_offload/test_cpu_gpu.py @@ -27,6 +27,7 @@ SEEDS = [0] DEVICE_TYPE = current_platform.device_type DEVICES = [f"{DEVICE_TYPE}:0"] NUM_MAPPINGS = [3] +NUM_MAPPINGS_PER_GROUP = [2] @pytest.mark.parametrize("gpu_to_cpu", [True, False]) @@ -58,9 +59,7 @@ def test_transfer( # build CanonicalKVCacheTensor list: one per tensor kv_cache_tensors: list[CanonicalKVCacheTensor] = [] for i in range(num_tensors): - gpu_tensor = torch.randint( - -128, - 127, + gpu_tensor = torch.zeros( (num_gpu_blocks, gpu_page_size_bytes), dtype=torch.int8, device=device, @@ -119,26 +118,36 @@ def test_transfer( for j in range(block_size_factor) ] - # maybe skip some GPU blocks to test reading from the middle of a CPU block - if not gpu_to_cpu: - blocks_to_skip = block_size_factor - 1 + # maybe skip some GPU blocks to test reading/writing from the middle of a CPU block + blocks_to_skip = block_size_factor - 1 + if blocks_to_skip > 0: gpu_blocks = gpu_blocks[blocks_to_skip:] cpu_blocks_expanded = cpu_blocks_expanded[blocks_to_skip:] # set transfer direction if gpu_to_cpu: handler = handlers.gpu_to_cpu_handler - src_spec = GPULoadStoreSpec(gpu_blocks, group_sizes=(len(gpu_blocks),)) + src_spec = GPULoadStoreSpec( + gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,) + ) dst_spec = CPULoadStoreSpec(cpu_blocks) dst_to_src = dict(zip(cpu_blocks_expanded, gpu_blocks)) - num_dst_sub_blocks = num_cpu_blocks * block_size_factor + num_dst_sub_blocks = num_gpu_blocks else: handler = handlers.cpu_to_gpu_handler src_spec = CPULoadStoreSpec(cpu_blocks) - dst_spec = GPULoadStoreSpec(gpu_blocks, group_sizes=(len(gpu_blocks),)) + dst_spec = GPULoadStoreSpec( + gpu_blocks, group_sizes=(len(gpu_blocks),), block_indices=(blocks_to_skip,) + ) dst_to_src = dict(zip(gpu_blocks, cpu_blocks_expanded)) num_dst_sub_blocks = num_gpu_blocks + # randomize src and dst tensors before transfer + for tensor in handler.src_tensors: + tensor.random_() + for tensor in handler.dst_tensors: + tensor.random_() + # clone src and dst tensors before transfer orig_src_tensors = [x.clone() for x in handler.src_tensors] orig_dst_tensors = [x.clone() for x in handler.dst_tensors] @@ -146,7 +155,7 @@ def test_transfer( # call transfer function start_time = time.time() assert handler.transfer_async(1, (src_spec, dst_spec)) - assert set({x.job_id for x in handler._transfers}) == {1} + assert {x.job_id for x in handler._transfers} == {1} # wait for transfer to complete end_time = time.time() + 10 @@ -155,11 +164,14 @@ def test_transfer( if finished: assert finished[0].job_id == 1 assert finished[0].success - assert finished[0].transfer_type == ( - ("GPU", "CPU") if gpu_to_cpu else ("CPU", "GPU") + assert ( + finished[0].transfer_type == ("GPU", "CPU") + if gpu_to_cpu + else ("CPU", "GPU") ) assert finished[0].transfer_size == ( - len(gpu_blocks) * handler.group_block_size_in_bytes[0] + len(gpu_blocks) + * sum([x.page_size_bytes for x in handler.kv_cache_groups_data_refs[0]]) ) assert finished[0].transfer_time > 0 assert finished[0].transfer_time < (time.time() - start_time) @@ -196,3 +208,211 @@ def test_transfer( handlers.gpu_to_cpu_handler.shutdown() if mmap_region: mmap_region.cleanup() + + +@pytest.mark.parametrize("gpu_to_cpu", [True, False]) +@pytest.mark.parametrize("num_mappings_per_group", NUM_MAPPINGS_PER_GROUP) +@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES) +@pytest.mark.parametrize("block_size_factor", BLOCK_SIZE_FACTORS) +@pytest.mark.parametrize("num_gpu_blocks", NUM_GPU_BLOCKS) +@pytest.mark.parametrize("num_cpu_blocks", NUM_CPU_BLOCKS) +@pytest.mark.parametrize("seed", SEEDS) +@pytest.mark.parametrize("device", DEVICES) +@torch.inference_mode() +def test_transfer_multi_group( + default_vllm_config, + gpu_to_cpu: bool, + num_mappings_per_group: int, + gpu_page_size_bytes: int, + block_size_factor: int, + num_gpu_blocks: int, + num_cpu_blocks: int, + seed: int, + device: str, +) -> None: + """Test transfers with three KV cache groups: + - Group 0: aligned transfer with num_mappings_per_group blocks + - Group 1: zero blocks (empty group) + - Group 2: unaligned CPU->GPU transfer (logical_offset=block_size_factor-1, + causing the implementation to skip source sub-blocks) with + num_mappings_per_group blocks + """ + set_random_seed(seed) + + # 3 groups, each with 2 tensors + num_groups = 3 + tensors_per_group = 2 + num_tensors = num_groups * tensors_per_group + kv_cache_tensors: list[CanonicalKVCacheTensor] = [] + for _ in range(num_tensors): + gpu_tensor = torch.zeros( + (num_gpu_blocks, gpu_page_size_bytes), + dtype=torch.int8, + device=device, + ) + kv_cache_tensors.append( + CanonicalKVCacheTensor( + tensor=gpu_tensor, + page_size_bytes=gpu_page_size_bytes, + ) + ) + + kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]] = [ + [ + CanonicalKVCacheRef( + tensor_idx=g * tensors_per_group + i, + page_size_bytes=gpu_page_size_bytes, + ) + for i in range(tensors_per_group) + ] + for g in range(num_groups) + ] + + canonical_kv_caches = CanonicalKVCaches( + tensors=kv_cache_tensors, group_data_refs=kv_cache_groups_data_refs + ) + + handlers = CpuGpuOffloadingHandlers( + kv_caches=canonical_kv_caches, + block_size_factor=block_size_factor, + num_cpu_blocks=num_cpu_blocks, + ) + + # group 0: aligned, group 1: empty, group 2: unaligned on CPU->GPU + group_sizes_in_cpu_blocks = [num_mappings_per_group, 0, num_mappings_per_group] + + total_cpu_blocks = sum(group_sizes_in_cpu_blocks) + total_gpu_blocks_needed = total_cpu_blocks * block_size_factor + gpu_blocks_all = random.sample(range(num_gpu_blocks), total_gpu_blocks_needed) + cpu_blocks_all = random.sample(range(num_cpu_blocks), total_cpu_blocks) + + # split gpu/cpu blocks per group + gpu_blocks_per_group: list[list[int]] = [] + cpu_blocks_per_group: list[list[int]] = [] + gpu_offset = 0 + cpu_offset = 0 + for size in group_sizes_in_cpu_blocks: + gpu_count = size * block_size_factor + gpu_blocks_per_group.append(gpu_blocks_all[gpu_offset : gpu_offset + gpu_count]) + cpu_blocks_per_group.append(cpu_blocks_all[cpu_offset : cpu_offset + size]) + gpu_offset += gpu_count + cpu_offset += size + + # expand cpu blocks to gpu-page granularity + cpu_blocks_expanded_per_group = [ + [ + cpu_block * block_size_factor + j + for cpu_block in cpu_blocks + for j in range(block_size_factor) + ] + for cpu_blocks in cpu_blocks_per_group + ] + + # skip sub-blocks from group 2 to test unaligned transfers. + sub_blocks_to_skip = block_size_factor - 1 # e.g. 2 when block_size_factor=3 + if sub_blocks_to_skip > 0: + gpu_blocks_per_group[2] = gpu_blocks_per_group[2][ + sub_blocks_to_skip:-sub_blocks_to_skip + ] + cpu_blocks_expanded_per_group[2] = cpu_blocks_expanded_per_group[2][ + sub_blocks_to_skip:-sub_blocks_to_skip + ] + + # build flat gpu_blocks list and group_sizes in GPU blocks + gpu_blocks: list[int] = [] + group_sizes: list[int] = [] + for gpu_blks in gpu_blocks_per_group: + gpu_blocks.extend(gpu_blks) + group_sizes.append(len(gpu_blks)) + + # build flat cpu_blocks list + cpu_blocks = [] + for cpu_blks in cpu_blocks_per_group: + cpu_blocks.extend(cpu_blks) + + # block_indices: only relevant for unaligned transfers + block_indices: list[int] = [0, 0, sub_blocks_to_skip] + + if gpu_to_cpu: + handler = handlers.gpu_to_cpu_handler + src_spec = GPULoadStoreSpec( + gpu_blocks, group_sizes=group_sizes, block_indices=block_indices + ) + dst_spec = CPULoadStoreSpec(cpu_blocks) + # per-group mapping: cpu sub-block -> gpu sub-block + dst_to_src_per_group = [ + dict(zip(expanded, gpu_blks)) + for expanded, gpu_blks in zip( + cpu_blocks_expanded_per_group, gpu_blocks_per_group + ) + ] + num_dst_sub_blocks = num_cpu_blocks * block_size_factor + else: + handler = handlers.cpu_to_gpu_handler + src_spec = CPULoadStoreSpec(cpu_blocks) + dst_spec = GPULoadStoreSpec( + gpu_blocks, group_sizes=group_sizes, block_indices=block_indices + ) + # per-group mapping: gpu sub-block -> cpu sub-block + dst_to_src_per_group = [ + dict(zip(gpu_blks, expanded)) + for gpu_blks, expanded in zip( + gpu_blocks_per_group, cpu_blocks_expanded_per_group + ) + ] + num_dst_sub_blocks = num_gpu_blocks + + # randomize src and dst tensors before transfer + for tensor in handler.src_tensors: + tensor.random_() + for tensor in handler.dst_tensors: + tensor.random_() + + orig_src_tensors = [x.clone() for x in handler.src_tensors] + orig_dst_tensors = [x.clone() for x in handler.dst_tensors] + + assert handler.transfer_async(1, (src_spec, dst_spec)) + assert {x.job_id for x in handler._transfers} == {1} + + end_time = time.time() + 10 + while time.time() < end_time: + finished = handler.get_finished() + if finished: + assert finished[0].job_id == 1 + assert finished[0].success + expected_bytes = sum( + group_size * sum([x.page_size_bytes for x in data_refs]) + for group_size, data_refs in zip( + group_sizes, handler.kv_cache_groups_data_refs + ) + ) + assert finished[0].transfer_size == expected_bytes + break + time.sleep(0.1) + + # verify src tensors did not change + for orig_tensor, tensor in zip(orig_src_tensors, handler.src_tensors): + assert torch.equal(orig_tensor, tensor) + + # verify dst tensors at gpu-page granularity + for group_idx, dst_to_src in enumerate(dst_to_src_per_group): + group_tensor_offset = group_idx * tensors_per_group + for tensor_idx in range(tensors_per_group): + src_tensor = handler.src_tensors[group_tensor_offset + tensor_idx] + dst_tensor = handler.dst_tensors[group_tensor_offset + tensor_idx] + orig_dst_tensor = orig_dst_tensors[group_tensor_offset + tensor_idx] + src_view = src_tensor.view(-1, gpu_page_size_bytes) + dst_view = dst_tensor.view(-1, gpu_page_size_bytes) + orig_dst_view = orig_dst_tensor.view(-1, gpu_page_size_bytes) + for dst_sub_block in range(num_dst_sub_blocks): + src_sub_block = dst_to_src.get(dst_sub_block) + if src_sub_block is not None: + expected = src_view[src_sub_block] + else: + expected = orig_dst_view[dst_sub_block] + torch.testing.assert_close( + dst_view[dst_sub_block].cpu(), expected.cpu() + ) + + handlers.cpu_to_gpu_handler.shutdown() + handlers.gpu_to_cpu_handler.shutdown() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index cd5a4f113dc..bff512815a6 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -381,7 +381,9 @@ class OffloadingConnectorScheduler: for i in range(self.config.block_size_factor): src_block_ids.append(block_ids[gpu_block_idx + i]) src_spec = GPULoadStoreSpec( - src_block_ids, group_sizes=(len(src_block_ids),) + src_block_ids, + group_sizes=(len(src_block_ids),), + block_indices=(0,), ) reqs_to_store[req_id] = (src_spec, dst_spec) diff --git a/vllm/v1/kv_offload/mediums.py b/vllm/v1/kv_offload/mediums.py index 85ef2a95a6b..02e36a80a8e 100644 --- a/vllm/v1/kv_offload/mediums.py +++ b/vllm/v1/kv_offload/mediums.py @@ -34,26 +34,24 @@ class GPULoadStoreSpec(BlockIDsLoadStoreSpec): will correspond to logically contiguous blocks, e.g. blocks 5-10 of a some request. block_indices[i] will represent the block index of the first block in group #i. Thus, len(block_indices) == len(group_sizes) = number of KV cache groups. - This information is required in order to support loading from offloaded blocks + This information is required in order to support off/loading from offloaded blocks which are larger than GPU blocks. In such cases, the first GPU block per each group may be unaligned to the offloaded block size, and so knowing block_indices[i] allows the worker to correctly skip part of the first matching offloaded block. - Offloading from GPU is always aligned to offloaded block size, and so - block_indices will only be set by the offloading connector when loading into GPU. """ def __init__( self, block_ids: list[int], group_sizes: Sequence[int], - block_indices: Sequence[int] | None = None, + block_indices: Sequence[int], ): super().__init__(block_ids) assert sum(group_sizes) == len(block_ids) - assert block_indices is None or len(block_indices) == len(group_sizes) + assert len(block_indices) == len(group_sizes) self.group_sizes: Sequence[int] = group_sizes - self.block_indices: Sequence[int] | None = block_indices + self.block_indices: Sequence[int] = block_indices @staticmethod def medium() -> str: diff --git a/vllm/v1/kv_offload/worker/cpu_gpu.py b/vllm/v1/kv_offload/worker/cpu_gpu.py index dd12a533ede..aab57ef2be4 100644 --- a/vllm/v1/kv_offload/worker/cpu_gpu.py +++ b/vllm/v1/kv_offload/worker/cpu_gpu.py @@ -9,9 +9,10 @@ import torch from vllm import _custom_ops as ops from vllm.logger import init_logger +from vllm.utils.math_utils import cdiv from vllm.utils.platform_utils import is_pin_memory_available from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion -from vllm.v1.kv_offload.mediums import BlockIDsLoadStoreSpec +from vllm.v1.kv_offload.mediums import BlockIDsLoadStoreSpec, GPULoadStoreSpec from vllm.v1.kv_offload.spec import CanonicalKVCacheRef, CanonicalKVCaches from vllm.v1.kv_offload.worker.worker import ( OffloadingHandler, @@ -135,9 +136,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): assert len(gpu_tensors) == len(cpu_tensors) assert len(gpu_tensors) > 0 - # assert a single KV group until transfer_async supports multiple groups - assert len(kv_cache_groups_data_refs) == 1 - # assert input tensors are as expected for gpu_tensor, cpu_tensor in zip(gpu_tensors, cpu_tensors): assert gpu_tensor.dtype == torch.int8 @@ -157,29 +155,13 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): cpu_tensors if gpu_to_cpu else gpu_tensors ) self.gpu_to_cpu: bool = gpu_to_cpu + self.kv_cache_groups_data_refs = kv_cache_groups_data_refs # GPU blocks may be smaller # cpu_page_size = gpu_page_size * block_size_factor. self.src_block_size_factor = 1 if self.gpu_to_cpu else block_size_factor self.dst_block_size_factor = block_size_factor if self.gpu_to_cpu else 1 - # per-tensor block size in byte - self.tensor_block_size_in_bytes = [ - gpu_tensor.shape[1] for gpu_tensor in gpu_tensors - ] - - # per-group block size in bytes - self.group_block_size_in_bytes = [] - for kv_cache_group_data_refs in kv_cache_groups_data_refs: - group_block_size_in_bytes = 0 - for kv_cache_data_ref in kv_cache_group_data_refs: - # TODO(orozery): use kv_cache_data_ref.page_size_bytes - # once swap_blocks support it - group_block_size_in_bytes += self.tensor_block_size_in_bytes[ - kv_cache_data_ref.tensor_idx - ] - self.group_block_size_in_bytes.append(group_block_size_in_bytes) - self.transfer_type = ("GPU", "CPU") if self.gpu_to_cpu else ("CPU", "GPU") # job_id -> event self._transfer_events: dict[int, torch.Event] = {} @@ -190,11 +172,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): # list of CUDA events available for re-use self._event_pool: list[torch.Event] = [] - # Pre-compute block sizes for batch copies. - self._block_size_in_bytes_arr = np.array( - self.tensor_block_size_in_bytes, dtype=np.int64 - ) - def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool: src_spec, dst_spec = transfer_spec assert isinstance(src_spec, BlockIDsLoadStoreSpec) @@ -205,37 +182,108 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): assert src_blocks.ndim == 1 assert dst_blocks.ndim == 1 - src_sub_block_count = src_blocks.size * self.src_block_size_factor - dst_sub_block_count = dst_blocks.size * self.dst_block_size_factor - src_sub_blocks_to_skip = -dst_blocks.size % self.src_block_size_factor + num_src_blocks = len(src_blocks) + num_dst_blocks = len(dst_blocks) - assert dst_sub_block_count == src_sub_block_count - src_sub_blocks_to_skip + # There are 2 types of transfers: + # 1. GPU -> CPU + # 2. CPU -> GPU + # + # transfers are also to CPU blocks, EXCEPT MAYBE for the first and last block. + # i.e. the first and last CPU blocks in src_blocks can match against + # a smaller (byte-wise) set of GPU blocks in dst_blocks. + # In such cases, we may need to skip some gpu-sized sub-blocks, + # and start reading/writing from the middle of the first CPU block. + # If we have multiple KV cache groups (when using HMA with hybrid models), + # we may have a partial first/last CPU block per each group. + # The group_sizes parameter encodes the size of each group of blocks + # in the GPU dst_blocks. + # If group_sizes is None, we assume all blocks belong to a single group. + # The logical_offset parameter maps each group of blocks to its logical + # offset inside the request, counting in GPU blocks. + # This allows us to find the correct starting position + # in the matching first CPU block. - num_pairs = dst_sub_block_count - num_tensors = len(self.src_tensors) - total = num_pairs * num_tensors + # extract group_sizes from the GPU spec + gpu_spec = src_spec if self.gpu_to_cpu else dst_spec + assert isinstance(gpu_spec, GPULoadStoreSpec) + group_sizes = gpu_spec.group_sizes + assert len(group_sizes) == len(self.kv_cache_groups_data_refs) - all_src = np.empty(total, dtype=np.int64) - all_dst = np.empty(total, dtype=np.int64) - all_sizes = np.empty(total, dtype=np.int64) + # extract block indices from the GPU spec + block_indices = gpu_spec.block_indices + assert len(block_indices) == len(self.kv_cache_groups_data_refs) - for t_idx, bsz in enumerate(self._block_size_in_bytes_arr): - start = t_idx * num_pairs - end = start + num_pairs - compute_sub_block_ptrs( - block_ids=src_blocks, - block_size_factor=self.src_block_size_factor, - output=all_src[start:end], - tensor=self.src_tensors[t_idx], - skip_count=src_sub_blocks_to_skip, + num_copy_ops = 0 + for group_size, group_data_refs in zip( + group_sizes, self.kv_cache_groups_data_refs + ): + num_copy_ops += group_size * len(group_data_refs) + + all_src = np.empty(num_copy_ops, dtype=np.int64) + all_dst = np.empty(num_copy_ops, dtype=np.int64) + all_sizes = np.empty(num_copy_ops, dtype=np.int64) + + src_offset = 0 + dst_offset = 0 + op_idx = 0 + # count total number of bytes copied + num_transfer_bytes = 0 + for group_size, block_idx, group_data_refs in zip( + group_sizes, block_indices, self.kv_cache_groups_data_refs + ): + if group_size == 0: + continue + + src_logical_blocks_to_skip = block_idx % self.src_block_size_factor + dst_logical_blocks_to_skip = block_idx % self.dst_block_size_factor + src_logical_blocks_count = group_size + src_logical_blocks_to_skip + dst_logical_blocks_count = group_size + dst_logical_blocks_to_skip + + dst_blocks_count = cdiv( + dst_logical_blocks_count, self.dst_block_size_factor ) - compute_sub_block_ptrs( - block_ids=dst_blocks, - block_size_factor=self.dst_block_size_factor, - output=all_dst[start:end], - tensor=self.dst_tensors[t_idx], + dst_end_offset = dst_offset + dst_blocks_count + assert dst_end_offset <= num_dst_blocks + + src_blocks_count = cdiv( + src_logical_blocks_count, self.src_block_size_factor ) - all_sizes[start:end] = bsz + src_end_offset = src_offset + src_blocks_count + assert src_end_offset <= num_src_blocks + + group_src = src_blocks[src_offset:src_end_offset] + group_dst = dst_blocks[dst_offset:dst_end_offset] + + for data_ref in group_data_refs: + t_idx = data_ref.tensor_idx + end_idx = op_idx + group_size + + compute_sub_block_ptrs( + group_src, + self.src_block_size_factor, + all_src[op_idx:end_idx], + self.src_tensors[t_idx], + skip_count=src_logical_blocks_to_skip, + ) + compute_sub_block_ptrs( + group_dst, + self.dst_block_size_factor, + all_dst[op_idx:end_idx], + self.dst_tensors[t_idx], + skip_count=dst_logical_blocks_to_skip, + ) + + all_sizes[op_idx:end_idx] = data_ref.page_size_bytes + num_transfer_bytes += group_size * data_ref.page_size_bytes + op_idx = end_idx + + src_offset = src_end_offset + dst_offset = dst_end_offset + + assert src_offset == num_src_blocks + assert dst_offset == num_dst_blocks + assert op_idx == num_copy_ops batch_src = torch.from_numpy(all_src) batch_dst = torch.from_numpy(all_dst) @@ -263,7 +311,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): stream.wait_event(last_event) with torch.cuda.stream(stream): start_event.record(stream) - if total > 0: + if num_copy_ops > 0: ops.swap_blocks_batch(batch_src, batch_dst, batch_sizes) end_event.record(stream) @@ -274,7 +322,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler): stream=stream, start_event=start_event, end_event=end_event, - num_bytes=dst_sub_block_count * self.group_block_size_in_bytes[0], + num_bytes=num_transfer_bytes, ) ) From 4254aeb56f280609e57e6b0134b3d6268d2fa87f Mon Sep 17 00:00:00 2001 From: Carl Y <4531192+carlyou@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:29:58 -0700 Subject: [PATCH 029/153] [fix] flaky test_mla_attn_quant_fusion.py (#40530) Signed-off-by: Carl You <4531192+carlyou@users.noreply.github.com> --- tests/compile/passes/test_mla_attn_quant_fusion.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tests/compile/passes/test_mla_attn_quant_fusion.py b/tests/compile/passes/test_mla_attn_quant_fusion.py index 8a575909612..0a38ffca483 100644 --- a/tests/compile/passes/test_mla_attn_quant_fusion.py +++ b/tests/compile/passes/test_mla_attn_quant_fusion.py @@ -83,10 +83,6 @@ class MLAAttentionQuantPatternModel(torch.nn.Module): self.vllm_config = vllm_config self.dtype = vllm_config.model_config.dtype - # Create kv_b_proj (ColumnParallelLinear) on device. - # Reuse weights from prior model instance when available, because - # ColumnParallelLinear may get NaN from recycled CUDA memory after - # torch.compile runs in the same process. kv_b_proj = ColumnParallelLinear( input_size=kv_lora_rank, output_size=num_heads * (qk_nope_head_dim + v_head_dim), @@ -96,8 +92,7 @@ class MLAAttentionQuantPatternModel(torch.nn.Module): kv_b_proj_weight = kwargs.get("kv_b_proj_weight") if kv_b_proj_weight is not None: kv_b_proj.weight.data.copy_(kv_b_proj_weight) - elif kv_b_proj.weight.data.isnan().any(): - # Sanitize NaN from recycled CUDA memory + else: kv_b_proj.weight.data.normal_() # Create MLAAttention From 123674879e8ebb714b9a88c853806826b95b8f81 Mon Sep 17 00:00:00 2001 From: philip-essential <169196560+philip-essential@users.noreply.github.com> Date: Tue, 21 Apr 2026 23:34:50 -0700 Subject: [PATCH 030/153] [Model] Add block-local attention and YaRN for local layers to Gemma3 (#39823) Signed-off-by: Philip Monk <169196560+philip-essential@users.noreply.github.com> --- docs/models/supported_models.md | 1 + tests/models/registry.py | 4 + .../layers/attention/attention.py | 6 + vllm/model_executor/models/registry.py | 1 + vllm/model_executor/models/rnj1.py | 470 ++++++++++++++++++ vllm/v1/attention/backends/triton_attn.py | 3 + .../attention/ops/triton_unified_attention.py | 63 ++- 7 files changed, 540 insertions(+), 8 deletions(-) create mode 100644 vllm/model_executor/models/rnj1.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index f1ba46b707d..70956f5092a 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -472,6 +472,7 @@ th { | `Qwen3MoeForCausalLM` | Qwen3MoE | `Qwen/Qwen3-30B-A3B`, etc. | ✅︎ | ✅︎ | | `Qwen3NextForCausalLM` | Qwen3NextMoE | `Qwen/Qwen3-Next-80B-A3B-Instruct`, etc. | ✅︎ | ✅︎ | | `RWForCausalLM` | Falcon RW | `tiiuae/falcon-40b`, etc. | | ✅︎ | +| `Rnj1ForCausalLM` | Rnj1 | `EssentialAI/rnj-1-instruct`, etc. | | | | `SarvamMoEForCausalLM` | Sarvam 2 | `sarvamai/sarvam2-30b-a3b`, etc. | ✅︎ | ✅︎ | | `SarvamMLAForCausalLM` | Sarvam 2 | `sarvamai/sarvam2-105b-a9b`, etc. | | ✅︎ | | `SeedOssForCausalLM` | SeedOss | `ByteDance-Seed/Seed-OSS-36B-Instruct`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index ea1f0190562..4c418ae4ee7 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -518,6 +518,10 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { extras={"tiny-random": "tiny-random/qwen3-next-moe"}, min_transformers_version="4.56.3", ), + "Rnj1ForCausalLM": _HfExamplesInfo( + "EssentialAI/rnj-1-instruct", + is_available_online=False, + ), "RWForCausalLM": _HfExamplesInfo("tiiuae/falcon-40b"), "SarvamMoEForCausalLM": _HfExamplesInfo( "sarvamai/sarvam-30b", diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index 9d2e29d02de..d229e32be75 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -336,6 +336,12 @@ class Attention(nn.Module, AttentionLayerBase): ) cache_config.enable_prefix_caching = False + if extra_impl_args.get("chunk_lookback", -1) > -1: + assert self.attn_backend.get_name() == "TRITON_ATTN", ( + f"Chunked attention with lookback requires the Triton backend, " + f"but got {self.attn_backend.get_name()}." + ) + impl_cls = self.attn_backend.get_impl_cls() self.impl = impl_cls( # type: ignore[assignment] # impl_cls always returns an AttentionImpl subclass num_heads, diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 4ba774a9fe8..baac4a6c664 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -110,6 +110,7 @@ _TEXT_GENERATION_MODELS = { "GemmaForCausalLM": ("gemma", "GemmaForCausalLM"), "Gemma2ForCausalLM": ("gemma2", "Gemma2ForCausalLM"), "Gemma3ForCausalLM": ("gemma3", "Gemma3ForCausalLM"), + "Rnj1ForCausalLM": ("rnj1", "Rnj1ForCausalLM"), "Gemma3nForCausalLM": ("gemma3n", "Gemma3nForCausalLM"), "Gemma4ForCausalLM": ("gemma4", "Gemma4ForCausalLM"), "Qwen3NextForCausalLM": ("qwen3_next", "Qwen3NextForCausalLM"), diff --git a/vllm/model_executor/models/rnj1.py b/vllm/model_executor/models/rnj1.py new file mode 100644 index 00000000000..f83577b7a39 --- /dev/null +++ b/vllm/model_executor/models/rnj1.py @@ -0,0 +1,470 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# RNJ-1 model: Gemma3-based architecture with chunked (block-local) attention. +# Chunked attention restricts local layers to attend within aligned blocks, +# with lookback to one previous block. +from collections.abc import Iterable +from itertools import islice + +import torch +from torch import nn +from transformers import Gemma3TextConfig + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig +from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import GeluAndMul +from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.layernorm import GemmaRMSNorm +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.sequence import IntermediateTensors +from vllm.v1.attention.backend import AttentionType + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import ( + AutoWeightsLoader, + extract_layer_index, + is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) + +logger = init_logger(__name__) + + +class Rnj1MLP(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_activation: str, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.down_proj", + ) + if hidden_activation != "gelu_pytorch_tanh": + raise ValueError( + "RNJ-1 uses `gelu_pytorch_tanh` as the hidden activation " + "function. Please set `hidden_act` and `hidden_activation` to " + "`gelu_pytorch_tanh`." + ) + self.act_fn = GeluAndMul(approximate="tanh") + + def forward(self, x: torch.Tensor) -> torch.Tensor: + gate_up, _ = self.gate_up_proj(x) + x = self.act_fn(gate_up) + x, _ = self.down_proj(x) + return x + + +class Rnj1Attention(nn.Module): + def __init__( + self, + config: Gemma3TextConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + head_dim: int, + max_position_embeddings: int, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + attn_logits_soft_cap: float | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.config = config + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + assert self.total_num_kv_heads % tp_size == 0 + else: + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + self.head_dim = head_dim + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = config.query_pre_attn_scalar**-0.5 + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + bias=config.attention_bias, + quant_config=quant_config, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=config.attention_bias, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + + self.q_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + self.k_norm = GemmaRMSNorm(self.head_dim, eps=config.rms_norm_eps) + + layer_idx = extract_layer_index(prefix) + layer_type = config.layer_types[layer_idx] + self.is_chunked = layer_type == "chunked_attention" + self.chunk_lookback = 1 if self.is_chunked else -1 + sliding_window = config.sliding_window if self.is_chunked else None + + # Initialize the rotary embedding. + # Expects v5-style rope_parameters keyed by layer type. + if layer_type in config.rope_parameters: + rope_parameters = config.rope_parameters[layer_type] + else: + rope_parameters = config.rope_parameters + + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + rope_parameters=rope_parameters, + is_neox_style=True, + ) + + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + attn_type=AttentionType.DECODER, + logits_soft_cap=attn_logits_soft_cap, + per_layer_sliding_window=sliding_window, + chunk_lookback=self.chunk_lookback, + prefix=f"{prefix}.attn", + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + **kwargs, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + q = q.unflatten(-1, (self.num_heads, self.head_dim)) + q = self.q_norm(q) + q = q.flatten(-2, -1) + k = k.unflatten(-1, (self.num_kv_heads, self.head_dim)) + k = self.k_norm(k) + k = k.flatten(-2, -1) + + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v) + output, _ = self.o_proj(attn_output) + return output + + +class Rnj1DecoderLayer(nn.Module): + def __init__( + self, + config: Gemma3TextConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + self.self_attn = Rnj1Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + head_dim=config.head_dim, + max_position_embeddings=config.max_position_embeddings, + cache_config=cache_config, + quant_config=quant_config, + attn_logits_soft_cap=None, + prefix=f"{prefix}.self_attn", + ) + self.hidden_size = config.hidden_size + self.mlp = Rnj1MLP( + hidden_size=self.hidden_size, + intermediate_size=config.intermediate_size, + hidden_activation=config.hidden_activation, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.input_layernorm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.post_attention_layernorm = GemmaRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.pre_feedforward_layernorm = GemmaRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + self.post_feedforward_layernorm = GemmaRMSNorm( + config.hidden_size, eps=config.rms_norm_eps + ) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + **kwargs, + ) -> tuple[torch.Tensor, torch.Tensor]: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + **kwargs, + ) + hidden_states = self.post_attention_layernorm(hidden_states) + + hidden_states, residual = self.pre_feedforward_layernorm( + hidden_states, residual + ) + hidden_states = self.mlp(hidden_states) + hidden_states = self.post_feedforward_layernorm(hidden_states) + return hidden_states, residual + + +@support_torch_compile +class Rnj1Model(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + self.config = config + self.quant_config = quant_config + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=f"{prefix}.embed_tokens", + ) + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: Rnj1DecoderLayer( + config, cache_config, quant_config, prefix=prefix + ), + prefix=f"{prefix}.layers", + ) + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + normalizer = self.config.hidden_size**0.5 + self.register_buffer("normalizer", torch.tensor(normalizer), persistent=False) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) * self.normalizer + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | IntermediateTensors: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_input_ids(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + for layer in islice(self.layers, self.start_layer, self.end_layer): + hidden_states, residual = layer( + positions, + hidden_states, + residual, + **kwargs, + ) + if not get_pp_group().is_last_rank: + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) + hidden_states, _ = self.norm(hidden_states, residual) + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + ("qkv_proj", "q_proj", "q"), + ("qkv_proj", "k_proj", "k"), + ("qkv_proj", "v_proj", "v"), + ("gate_up_proj", "gate_proj", 0), + ("gate_up_proj", "up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if ( + self.quant_config + and self.quant_config.get_name() == "gguf" + and name.endswith("norm.weight") + ): + loaded_weight -= 1 + + if self.quant_config is not None and ( + scale_name := self.quant_config.get_cache_scale(name) + ): + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + loaded_params.add(scale_name) + continue + + if name.endswith((".k_scale", ".v_scale", ".q_scale", ".prob_scale")): + remapped_name = maybe_remap_kv_scale_name(name, params_dict) + if remapped_name is not None and remapped_name in params_dict: + param = params_dict[remapped_name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + loaded_params.add(remapped_name) + continue + + for param_name, shard_name, shard_id in stacked_params_mapping: + if shard_name not in name: + continue + name = name.replace(shard_name, param_name) + if name.endswith(".bias") and name not in params_dict: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + break + else: + if name.endswith(".bias") and name not in params_dict: + continue + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params + + +class Rnj1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP): + packed_modules_mapping = { + "qkv_proj": [ + "q_proj", + "k_proj", + "v_proj", + ], + "gate_up_proj": [ + "gate_proj", + "up_proj", + ], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + + super().__init__() + self.config = config + self.quant_config = quant_config + self.model = Rnj1Model( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + if config.tie_word_embeddings: + self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens) + + self.logits_processor = LogitsProcessor( + config.vocab_size, soft_cap=config.final_logit_softcapping + ) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor | None, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor | IntermediateTensors: + hidden_states = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(weights) diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 76cae14aedb..4739d48e870 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -458,6 +458,7 @@ class TritonAttentionImpl(AttentionImpl): kv_sharing_target_layer_name: int | None = None, sinks: torch.Tensor | None = None, use_alibi_sqrt: bool = False, + chunk_lookback: int = -1, ) -> None: self.num_heads = num_heads self.head_size = head_size @@ -492,6 +493,7 @@ class TritonAttentionImpl(AttentionImpl): f"num_heads: {num_heads}." ) self.use_alibi_sqrt = use_alibi_sqrt + self.chunk_lookback = chunk_lookback self.supports_quant_query_input = current_platform.is_cuda() self._kv_quant_mode = get_kv_quant_mode(kv_cache_dtype) @@ -631,6 +633,7 @@ class TritonAttentionImpl(AttentionImpl): kv_quant_mode=self._kv_quant_mode, k_scale_cache=k_scale_cache, v_scale_cache=v_scale_cache, + chunk_lookback=self.chunk_lookback, ) return output diff --git a/vllm/v1/attention/ops/triton_unified_attention.py b/vllm/v1/attention/ops/triton_unified_attention.py index 150f022f848..285c7b263be 100644 --- a/vllm/v1/attention/ops/triton_unified_attention.py +++ b/vllm/v1/attention/ops/triton_unified_attention.py @@ -177,6 +177,8 @@ def kernel_unified_attention_2d( stride_vs_blk=0, stride_vs_slot=0, stride_vs_head=0, + CHUNK_LOOKBACK: tl.constexpr = -1, + CHUNK_SIZE: tl.constexpr = -1, ): q_block_global_idx = tl.program_id(0) kv_head_idx = tl.program_id(1) @@ -293,7 +295,11 @@ def kernel_unified_attention_2d( # where q_abs = context_len + q # The union of allowed key positions for this Q-block is: # [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi] - first_allowed_key = context_len + qpos_lo - SLIDING_WINDOW + 1 + q_abs = context_len + qpos_lo + if CHUNK_LOOKBACK > -1: + first_allowed_key = ((q_abs // CHUNK_SIZE) - CHUNK_LOOKBACK) * CHUNK_SIZE + else: + first_allowed_key = q_abs - SLIDING_WINDOW + 1 last_allowed_key = context_len + qpos_hi # Convert to tile indices and clamp tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE) @@ -370,9 +376,19 @@ def kernel_unified_attention_2d( query_abs_pos = context_len + query_pos[:, None] seq_mask = seq_offset[None, :] <= query_abs_pos - # Apply sliding window to base mask BEFORE mm_prefix OR. - # Order must match FlexAttention: (causal AND sliding_window) OR mm_prefix - if SLIDING_WINDOW > 0: + # Apply sliding window / chunked attention to base mask + # BEFORE mm_prefix OR. + # Order must match FlexAttention: + # (causal AND sliding_window) OR mm_prefix + if CHUNK_LOOKBACK > -1: + seq_mask = seq_mask & ( + ( + (context_len + query_pos[:, None]) // CHUNK_SIZE + - (seq_offset[None, :] // CHUNK_SIZE) + ) + <= CHUNK_LOOKBACK + ) + elif SLIDING_WINDOW > 0: seq_mask = seq_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW) # PrefixLM: extend mask with bidirectional ranges for multimodal tokens. @@ -559,6 +575,8 @@ def kernel_unified_attention_3d( stride_vs_blk=0, stride_vs_slot=0, stride_vs_head=0, + CHUNK_LOOKBACK: tl.constexpr = -1, + CHUNK_SIZE: tl.constexpr = -1, ): q_block_global_idx = tl.program_id(0) kv_head_idx = tl.program_id(1) @@ -681,7 +699,11 @@ def kernel_unified_attention_3d( # where q_abs = context_len + q # The union of allowed key positions for this Q-block is: # [context_len + qpos_lo - SLIDING_WINDOW + 1, context_len + qpos_hi] - first_allowed_key = context_len + qpos_lo - SLIDING_WINDOW + 1 + q_abs = context_len + qpos_lo + if CHUNK_LOOKBACK > -1: + first_allowed_key = ((q_abs // CHUNK_SIZE) - CHUNK_LOOKBACK) * CHUNK_SIZE + else: + first_allowed_key = q_abs - SLIDING_WINDOW + 1 last_allowed_key = context_len + qpos_hi # Convert to tile indices and clamp tile_start = tl.maximum(0, first_allowed_key // TILE_SIZE) @@ -761,9 +783,19 @@ def kernel_unified_attention_3d( query_abs_pos = context_len + query_pos[:, None] seq_mask = seq_offset[None, :] <= query_abs_pos - # Apply sliding window to base mask BEFORE mm_prefix OR. - # Order must match FlexAttention: (causal AND sliding_window) OR mm_prefix - if SLIDING_WINDOW > 0: + # Apply sliding window / chunked attention to base mask + # BEFORE mm_prefix OR. + # Order must match FlexAttention: + # (causal AND sliding_window) OR mm_prefix + if CHUNK_LOOKBACK > -1: + seq_mask = seq_mask & ( + ( + (context_len + query_pos[:, None]) // CHUNK_SIZE + - (seq_offset[None, :] // CHUNK_SIZE) + ) + <= CHUNK_LOOKBACK + ) + elif SLIDING_WINDOW > 0: seq_mask = seq_mask & ((query_abs_pos - seq_offset) < SLIDING_WINDOW) # PrefixLM: extend mask with bidirectional ranges for multimodal tokens. @@ -1046,6 +1078,8 @@ def unified_attention( kv_quant_mode: KVQuantMode = KVQuantMode.NONE, k_scale_cache=None, # [num_blocks, block_size, num_kv_heads] float32 v_scale_cache=None, # [num_blocks, block_size, num_kv_heads] float32 + # Chunked attention: restrict attention to aligned blocks with lookback. + chunk_lookback=-1, ): assert causal, "Only causal attention is supported" assert q_descale is None, "Q scales not supported" @@ -1093,6 +1127,15 @@ def unified_attention( # Tile sizes for prefill and decode. Gemma3 models use optimized values. # Note: tile size must be at least 32 for fp8 (element_size == 1). sliding_window_val = 1 + window_size[0] if window_size[0] >= 0 else 0 + + # Compute chunked block size from sliding window if needed. + chunk_size = -1 + if sliding_window_val > 0 and chunk_lookback > -1: + chunk_size = sliding_window_val // (chunk_lookback + 1) + assert chunk_size > 0, "sliding_window must be > chunk_lookback+1" + elif sliding_window_val <= 0: + chunk_lookback = -1 + TILE_SIZE_PREFILL = _get_tile_size( head_size, sliding_window_val, @@ -1184,6 +1227,8 @@ def unified_attention( stride_vs_blk=v_scale_cache.stride(0) if v_scale_cache is not None else 0, stride_vs_slot=v_scale_cache.stride(1) if v_scale_cache is not None else 0, stride_vs_head=v_scale_cache.stride(2) if v_scale_cache is not None else 0, + CHUNK_LOOKBACK=chunk_lookback, + CHUNK_SIZE=chunk_size, ) else: kernel_unified_attention_3d[ @@ -1245,6 +1290,8 @@ def unified_attention( stride_vs_blk=v_scale_cache.stride(0) if v_scale_cache is not None else 0, stride_vs_slot=v_scale_cache.stride(1) if v_scale_cache is not None else 0, stride_vs_head=v_scale_cache.stride(2) if v_scale_cache is not None else 0, + CHUNK_LOOKBACK=chunk_lookback, + CHUNK_SIZE=chunk_size, ) reduce_segments[(q.shape[0], num_query_heads)]( output_ptr=out, From a2bd09c960e584b5df1481de21cdc9a2b08c3e8b Mon Sep 17 00:00:00 2001 From: Chauncey Date: Wed, 22 Apr 2026 15:27:44 +0800 Subject: [PATCH 031/153] [Bugfix] [Reasoning] Add reasoning_start_str/reasoning_end_str properties to reasoning parsers (#40566) Signed-off-by: chaunceyjiang --- vllm/reasoning/deepseek_v3_reasoning_parser.py | 8 ++++++++ vllm/reasoning/identity_reasoning_parser.py | 8 ++++++++ vllm/reasoning/kimi_k2_reasoning_parser.py | 8 ++++++++ vllm/reasoning/olmo3_reasoning_parser.py | 8 ++++++++ vllm/reasoning/step3_reasoning_parser.py | 9 +++++++++ 5 files changed, 41 insertions(+) diff --git a/vllm/reasoning/deepseek_v3_reasoning_parser.py b/vllm/reasoning/deepseek_v3_reasoning_parser.py index d2f7f50a328..bb79afd8ded 100644 --- a/vllm/reasoning/deepseek_v3_reasoning_parser.py +++ b/vllm/reasoning/deepseek_v3_reasoning_parser.py @@ -40,6 +40,14 @@ class DeepSeekV3ReasoningParser(ReasoningParser): else: self._parser = IdentityReasoningParser(tokenizer, *args, **kwargs) + @property + def reasoning_start_str(self) -> str | None: + return self._parser.reasoning_start_str + + @property + def reasoning_end_str(self) -> str | None: + return self._parser.reasoning_end_str + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: return self._parser.is_reasoning_end(input_ids) diff --git a/vllm/reasoning/identity_reasoning_parser.py b/vllm/reasoning/identity_reasoning_parser.py index b02a9d3184a..c6f117e2f98 100644 --- a/vllm/reasoning/identity_reasoning_parser.py +++ b/vllm/reasoning/identity_reasoning_parser.py @@ -33,6 +33,14 @@ class IdentityReasoningParser(ReasoningParser): "constructor during construction." ) + @property + def reasoning_start_str(self) -> str | None: + return None + + @property + def reasoning_end_str(self) -> str | None: + return None + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: # Always return True, since we never treat reasoning specially return True diff --git a/vllm/reasoning/kimi_k2_reasoning_parser.py b/vllm/reasoning/kimi_k2_reasoning_parser.py index 8ee05ffd23a..7a92703426f 100644 --- a/vllm/reasoning/kimi_k2_reasoning_parser.py +++ b/vllm/reasoning/kimi_k2_reasoning_parser.py @@ -65,6 +65,14 @@ class KimiK2ReasoningParser(ReasoningParser): "tokens in the tokenizer!" ) + @property + def reasoning_start_str(self) -> str | None: + return self._start_token + + @property + def reasoning_end_str(self) -> str | None: + return self._end_token + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: """ Check if the reasoning content ends in the input_ids. diff --git a/vllm/reasoning/olmo3_reasoning_parser.py b/vllm/reasoning/olmo3_reasoning_parser.py index 9697b500447..b685aa23185 100644 --- a/vllm/reasoning/olmo3_reasoning_parser.py +++ b/vllm/reasoning/olmo3_reasoning_parser.py @@ -237,6 +237,14 @@ class Olmo3ReasoningParser(ReasoningParser): think_start=self.think_start, think_end=self.think_end ) + @property + def reasoning_start_str(self) -> str: + return self.think_start + + @property + def reasoning_end_str(self) -> str: + return self.think_end + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: text = self.model_tokenizer.decode(input_ids) return self.think_end in text diff --git a/vllm/reasoning/step3_reasoning_parser.py b/vllm/reasoning/step3_reasoning_parser.py index 5837f0673b7..a50fcf02db4 100644 --- a/vllm/reasoning/step3_reasoning_parser.py +++ b/vllm/reasoning/step3_reasoning_parser.py @@ -29,6 +29,7 @@ class Step3ReasoningParser(ReasoningParser): def __init__(self, tokenizer: PreTrainedTokenizerBase, *args, **kwargs): super().__init__(tokenizer, *args, **kwargs) + self.think_start_token = "" self.think_end_token = "" self.reasoning_regex = re.compile(rf"(.*?){self.think_end_token}", re.DOTALL) @@ -47,6 +48,14 @@ class Step3ReasoningParser(ReasoningParser): ) self.think_end_token_id: int = think_end_token_id + @property + def reasoning_start_str(self) -> str: + return self.think_start_token + + @property + def reasoning_end_str(self) -> str: + return self.think_end_token + def extract_reasoning_streaming( self, previous_text: str, From 6aa057c9d7de0b2535cd9d18369b60f9a506a55f Mon Sep 17 00:00:00 2001 From: storyicon Date: Wed, 22 Apr 2026 15:50:04 +0800 Subject: [PATCH 032/153] [Multimodal] Support custom video metadata for pre-extracted frame sequences (#40133) Signed-off-by: storyicon Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- docs/features/multimodal_inputs.md | 64 ++++++++++++++++++++++++++++++ vllm/multimodal/media/video.py | 42 ++++++++++++++++++-- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index d9b49f7cb7f..df7aef3f14f 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -780,6 +780,70 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ Works with common video formats like MP4 when using OpenCV backends. +#### Pre-extracted Frame Sequences with `media_io_kwargs` + +When you extract video frames on the client side and send them as `video/jpeg` (base64-concatenated JPEG frames), you can preserve the original video metadata by using `media_io_kwargs` in your request. This enables more accurate video understanding by preserving temporal information that would otherwise be lost during client-side frame extraction. + +**Supported Parameters:** + +| Parameter | Type | Description | +| --------- | ---- | ----------- | +| `fps` | float | Frame rate of the original video | +| `frames_indices` | list[int] | Indices of the actually sampled frames | +| `total_num_frames` | int | Total frame count of the original video | +| `duration` | float | Duration of the original video in seconds | +| `do_sample_frames` | bool | Whether to perform frame sampling | + +??? code + + ```python + from openai import OpenAI + + client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY") + + # Client-side frame extraction + frames = extract_frames(video_path, num_frames=32) + frames_b64 = ",".join([encode_image(f) for f in frames]) + video_url = f"data:video/jpeg;base64,{frames_b64}" + + # Pass video metadata via media_io_kwargs + response = client.chat.completions.create( + model="your-multimodal-model", + messages=[{ + "role": "user", + "content": [ + {"type": "video_url", "video_url": {"url": video_url}}, + {"type": "text", "text": "Describe what happens in this video."} + ] + }], + extra_body={ + "media_io_kwargs": { + "video": { + "fps": 30.0, + "frames_indices": [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, + 100, 110, 120, 130, 140, 150, 160, 170, + 180, 190, 200, 210, 220, 230, 240, 250, + 260, 270, 280, 290, 300, 310], + "total_num_frames": 900, + "duration": 30.0, + } + } + }, + ) + + print(response.choices[0].message.content) + ``` + +**Why use `media_io_kwargs`?** + +When extracting frames client-side, the server loses important context about the original video: + +- **Temporal information**: Which frames were sampled and their positions in the original timeline +- **Video duration**: How long the original video was +- **Frame rate**: The original playback speed + +By passing this metadata, the model can better understand the temporal distribution of the sampled frames and whether important moments might have been skipped. + #### Custom RGBA Background Color To use a custom background color for RGBA images, pass the `rgba_background_color` parameter via `--media-io-kwargs`: diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index 691d9444b81..404f5a0e7cf 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -92,14 +92,48 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): ) total = int(frames.shape[0]) fps = float(self.kwargs.get("fps", 1)) - duration = total / fps if fps > 0 else 0.0 + + # validate and extract frames_indices + frames_indices = self.kwargs.get("frames_indices") + if frames_indices is not None: + if not ( + isinstance(frames_indices, list) + and all(isinstance(i, int) for i in frames_indices) + ): + raise ValueError("frames_indices must be a list of integers") + if len(frames_indices) != total: + raise ValueError( + f"frames_indices length ({len(frames_indices)}) must " + f"match number of frames sent ({total})" + ) + else: + frames_indices = list(range(total)) + + # validate and extract total_num_frames + total_num_frames = self.kwargs.get("total_num_frames", total) + if not isinstance(total_num_frames, int) or total_num_frames < 1: + raise ValueError("total_num_frames must be a positive integer") + if total_num_frames < total: + raise ValueError( + f"total_num_frames ({total_num_frames}) must be >= " + f"number of frames sent ({total})" + ) + + # validate and extract duration + duration = self.kwargs.get("duration") + if duration is not None: + if not isinstance(duration, (int, float)) or duration < 0: + raise ValueError("duration must be a non-negative number") + else: + duration = total_num_frames / fps if fps > 0 else 0.0 + metadata = { - "total_num_frames": total, + "total_num_frames": total_num_frames, "fps": fps, "duration": duration, "video_backend": "jpeg_sequence", - "frames_indices": list(range(total)), - "do_sample_frames": False, + "frames_indices": frames_indices, + "do_sample_frames": self.kwargs.get("do_sample_frames", False), } return frames, metadata From ed6d30377d0fb6f05cc63e54fac0dca2f0d2d8f2 Mon Sep 17 00:00:00 2001 From: Johnny Yang <24908445+jcyang43@users.noreply.github.com> Date: Wed, 22 Apr 2026 01:33:45 -0700 Subject: [PATCH 033/153] upgrade tpu-inference to v0.18.0 (#40395) --- requirements/tpu.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/tpu.txt b/requirements/tpu.txt index 7695b4ba2f4..cee9fa6576e 100644 --- a/requirements/tpu.txt +++ b/requirements/tpu.txt @@ -11,4 +11,4 @@ ray[default] ray[data] setuptools==78.1.0 nixl==0.3.0 -tpu-inference==0.12.0 +tpu-inference==0.18.0 From 9047288b68f387331932598a8ba398c8d03a7d8c Mon Sep 17 00:00:00 2001 From: AllenDou Date: Wed, 22 Apr 2026 17:25:06 +0800 Subject: [PATCH 034/153] support hotwords for FunASR model (#39674) Signed-off-by: zixiao Co-authored-by: zixiao --- .../openai_transcription_client.py | 25 ++++++++++++++++--- vllm/config/speech_to_text.py | 6 +++++ .../openai/speech_to_text/protocol.py | 14 +++++++++++ vllm/model_executor/models/funasr.py | 9 ++++++- 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/examples/online_serving/openai_transcription_client.py b/examples/online_serving/openai_transcription_client.py index 478a0a7ea9e..396edba1155 100644 --- a/examples/online_serving/openai_transcription_client.py +++ b/examples/online_serving/openai_transcription_client.py @@ -27,7 +27,12 @@ from vllm.assets.audio import AudioAsset def sync_openai( - audio_path: str, client: OpenAI, model: str, *, repetition_penalty: float = 1.3 + audio_path: str, + client: OpenAI, + model: str, + *, + repetition_penalty: float = 1.3, + hotwords: str = None, ): """ Perform synchronous transcription using OpenAI-compatible API. @@ -43,12 +48,15 @@ def sync_openai( extra_body=dict( seed=4419, repetition_penalty=repetition_penalty, + hotwords=hotwords, ), ) print("transcription result [sync]:", transcription.text) -async def stream_openai_response(audio_path: str, client: AsyncOpenAI, model: str): +async def stream_openai_response( + audio_path: str, client: AsyncOpenAI, model: str, hotwords: str = None +): """ Perform asynchronous transcription using OpenAI-compatible API. """ @@ -64,6 +72,7 @@ async def stream_openai_response(audio_path: str, client: AsyncOpenAI, model: st extra_body=dict( seed=420, top_p=0.6, + hotwords=hotwords, ), stream=True, ) @@ -136,6 +145,7 @@ def main(args): client=client, model=model, repetition_penalty=args.repetition_penalty, + hotwords=args.hotwords, ) # Run the asynchronous function @@ -146,7 +156,10 @@ def main(args): ) asyncio.run( stream_openai_response( - args.audio_path if args.audio_path else winning_call, client, model + args.audio_path if args.audio_path else winning_call, + client, + model, + hotwords=args.hotwords, ) ) else: @@ -174,5 +187,11 @@ if __name__ == "__main__": default=1.3, help="repetition penalty", ) + parser.add_argument( + "--hotwords", + type=str, + default=None, + help="hotwords", + ) args = parser.parse_args() main(args) diff --git a/vllm/config/speech_to_text.py b/vllm/config/speech_to_text.py index 37350e86126..6d31713c866 100644 --- a/vllm/config/speech_to_text.py +++ b/vllm/config/speech_to_text.py @@ -35,6 +35,12 @@ class SpeechToTextParams: language: str | None = None """ISO 639-1 language code (validated / auto-detected).""" + hotwords: str | None = None + """ + hotwords refers to a list of important words or phrases that the model + should pay extra attention to during transcription. + """ + task_type: str = "transcribe" """``"transcribe"`` or ``"translate"``.""" diff --git a/vllm/entrypoints/openai/speech_to_text/protocol.py b/vllm/entrypoints/openai/speech_to_text/protocol.py index 623ae3fc2c2..af1aaf08655 100644 --- a/vllm/entrypoints/openai/speech_to_text/protocol.py +++ b/vllm/entrypoints/openai/speech_to_text/protocol.py @@ -78,6 +78,12 @@ class TranscriptionRequest(OpenAIBaseModel): will improve accuracy and latency. """ + hotwords: str | None = None + """ + hotwords refers to a list of important words or phrases that the model + should pay extra attention to during transcription. + """ + prompt: str = Field(default="") """An optional text to guide the model's style or continue a previous audio segment. @@ -205,6 +211,7 @@ class TranscriptionRequest(OpenAIBaseModel): task_type=task_type, request_prompt=self.prompt, to_language=self.to_language, + hotwords=self.hotwords, ) def to_beam_search_params( @@ -481,6 +488,12 @@ class TranslationRequest(OpenAIBaseModel): will improve accuracy. """ + hotwords: str | None = None + """ + hotwords refers to a list of important words or phrases that the model + should pay extra attention to during transcription. + """ + to_language: str | None = None """The language of the input audio we translate to. @@ -522,6 +535,7 @@ class TranslationRequest(OpenAIBaseModel): task_type=task_type, request_prompt=self.prompt, to_language=self.to_language, + hotwords=self.hotwords, ) def to_beam_search_params( diff --git a/vllm/model_executor/models/funasr.py b/vllm/model_executor/models/funasr.py index ab9d532f73b..4b5a1c02593 100644 --- a/vllm/model_executor/models/funasr.py +++ b/vllm/model_executor/models/funasr.py @@ -881,13 +881,20 @@ class FunASRForConditionalGeneration( audio = stt_params.audio stt_config = stt_params.stt_config language = stt_params.language + hotwords = stt_params.hotwords if language is None: raise ValueError( "Language must be specified when creating the funasr prompt" ) - funasr_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n语音转写：<|AUDIO|><|im_end|>\n<|im_start|>assistant\n" # noqa: E501 + if hotwords is not None: + funasr_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n请结合上下文信息，更加准确地完成语音转写任务。如果没有相关信息，我们会留空。\n\n\n**上下文信息：**\n\n\n热词列表：[{}]\n语音转写：<|AUDIO|><|im_end|>\n<|im_start|>assistant\n".format( # noqa: E501 + hotwords + ) + else: + funasr_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n语音转写：<|AUDIO|><|im_end|>\n<|im_start|>assistant\n" # noqa: E501 + prompt = { "prompt": funasr_prompt, "multi_modal_data": { From 04eac6ba24d22e0e280dff53695544facfaf5ca0 Mon Sep 17 00:00:00 2001 From: lyd1992 <105697319+lyd1992@users.noreply.github.com> Date: Wed, 22 Apr 2026 17:38:18 +0800 Subject: [PATCH 035/153] [Bugfix][CPU][RISC-V] Clamp exp() input to prevent NaN (#40428) Signed-off-by: liuyudong --- csrc/cpu/cpu_types_riscv_impl.hpp | 38 +++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/csrc/cpu/cpu_types_riscv_impl.hpp b/csrc/cpu/cpu_types_riscv_impl.hpp index 7c25ccc5059..3952a811c2c 100644 --- a/csrc/cpu/cpu_types_riscv_impl.hpp +++ b/csrc/cpu/cpu_types_riscv_impl.hpp @@ -15,16 +15,12 @@ #include namespace vec_op { -#ifdef RISCV_BF16_SUPPORT - #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) -#else - #define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ - AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ - AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) -#endif +// BFloat16 is always supported on RISC-V: natively when RISCV_BF16_SUPPORT +// is defined, otherwise via the FP32-simulation fallback path. +#define VLLM_DISPATCH_CASE_FLOATING_TYPES(...) \ + AT_DISPATCH_CASE(at::ScalarType::Float, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::Half, __VA_ARGS__) \ + AT_DISPATCH_CASE(at::ScalarType::BFloat16, __VA_ARGS__) #define VLLM_DISPATCH_FLOATING_TYPES(TYPE, NAME, ...) \ AT_DISPATCH_SWITCH(TYPE, NAME, VLLM_DISPATCH_CASE_FLOATING_TYPES(__VA_ARGS__)) @@ -486,9 +482,18 @@ struct FP32Vec8 : public Vec { } FP32Vec8 exp() const { + // Clamp input to prevent NaN: exp(-inf) must return 0, not NaN. + // Without clamping, -inf * 0.0 = NaN in the final poly * scale step. + // Matches the clamping strategy used by x86 AVX-512 and ARM NEON. + constexpr float exp_lo = -87.3365447505f; // ln(FLT_MIN) + constexpr float exp_hi = 88.7228391117f; // ln(FLT_MAX) + fixed_fp32x8_t x = RVVI(__riscv_vfmin_vf_f32, LMUL_256)( + RVVI(__riscv_vfmax_vf_f32, LMUL_256)(reg, exp_lo, VEC_ELEM_NUM), exp_hi, + VEC_ELEM_NUM); + const float inv_ln2 = 1.44269504088896341f; fixed_fp32x8_t x_scaled = - RVVI(__riscv_vfmul_vf_f32, LMUL_256)(reg, inv_ln2, VEC_ELEM_NUM); + RVVI(__riscv_vfmul_vf_f32, LMUL_256)(x, inv_ln2, VEC_ELEM_NUM); fixed_i32x8_t n_int = RVVI(__riscv_vfcvt_x_f_v_i32, LMUL_256)(x_scaled, VEC_ELEM_NUM); fixed_fp32x8_t n_float = @@ -706,9 +711,18 @@ struct FP32Vec16 : public Vec { } FP32Vec16 exp() const { + // Clamp input to prevent NaN: exp(-inf) must return 0, not NaN. + // Without clamping, -inf * 0.0 = NaN in the final poly * scale step. + // Matches the clamping strategy used by x86 AVX-512 and ARM NEON. + constexpr float exp_lo = -87.3365447505f; // ln(FLT_MIN) + constexpr float exp_hi = 88.7228391117f; // ln(FLT_MAX) + fixed_fp32x16_t x = RVVI(__riscv_vfmin_vf_f32, LMUL_512)( + RVVI(__riscv_vfmax_vf_f32, LMUL_512)(reg, exp_lo, VEC_ELEM_NUM), exp_hi, + VEC_ELEM_NUM); + const float inv_ln2 = 1.44269504088896341f; fixed_fp32x16_t x_scaled = - RVVI(__riscv_vfmul_vf_f32, LMUL_512)(reg, inv_ln2, VEC_ELEM_NUM); + RVVI(__riscv_vfmul_vf_f32, LMUL_512)(x, inv_ln2, VEC_ELEM_NUM); fixed_i32x16_t n_int = RVVI(__riscv_vfcvt_x_f_v_i32, LMUL_512)(x_scaled, VEC_ELEM_NUM); fixed_fp32x16_t n_float = From a250f1bd5fd8ac7a7d97bc8b5b2082417b4564d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=84=8D=F0=9D=95=A0=F0=9D=95=9D=F0=9D=95=9D=F0=9D=95=A0?= =?UTF-8?q?=F0=9D=95=A8=20=F0=9D=95=84=F0=9D=95=92=F0=9D=95=9F?= Date: Wed, 22 Apr 2026 14:33:50 +0300 Subject: [PATCH 036/153] [Bugfix] LoRA for DeepSeek V3.2 (#35077) Signed-off-by: Hollow Man Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Jee Jee Li --- tests/kernels/moe/test_moe_layer.py | 19 +- tests/lora/test_layers.py | 272 +++++++++++++++++- tests/lora/test_lora_manager.py | 150 ++++++++++ tests/lora/test_lora_utils.py | 21 ++ vllm/lora/layers/base_linear.py | 9 + vllm/lora/layers/column_parallel_linear.py | 52 +++- vllm/lora/layers/replicated_linear.py | 8 +- vllm/lora/model_manager.py | 59 +++- vllm/lora/utils.py | 28 +- vllm/lora/worker_manager.py | 9 +- .../layers/fused_moe/oracle/unquantized.py | 13 + .../layers/quantization/utils/quant_utils.py | 6 + vllm/v1/worker/lora_model_runner_mixin.py | 5 +- 13 files changed, 614 insertions(+), 37 deletions(-) diff --git a/tests/kernels/moe/test_moe_layer.py b/tests/kernels/moe/test_moe_layer.py index 07c04a16802..14cfd00c2bd 100644 --- a/tests/kernels/moe/test_moe_layer.py +++ b/tests/kernels/moe/test_moe_layer.py @@ -1620,18 +1620,13 @@ def _parallel_worker( else: print("F", end="") finally: - # Note: for some reason DeepEP buffers don't seem to be - # entirely reusable on B200. In order to work around this - # we clear the all2all manager's cache after each testpoint. - cap = current_platform.get_device_capability() - if ( - cap is not None - and cap.major == 10 - and ( - test_config.backend == "deepep_low_latency" - or test_config.backend == "deepep_high_throughput" - ) - ): + # DeepEP managers are not reliably reusable across many subtests in + # a single worker process. Tear them down after each DeepEP case so + # later subtests do not inherit stale communication state. + if test_config.backend in { + "deepep_low_latency", + "deepep_high_throughput", + }: torch.accelerator.synchronize() all2all_manager = get_ep_group().device_communicator.all2all_manager if all2all_manager is not None: diff --git a/tests/lora/test_layers.py b/tests/lora/test_layers.py index c2b4f551564..a0028687a32 100644 --- a/tests/lora/test_layers.py +++ b/tests/lora/test_layers.py @@ -44,6 +44,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, get_masked_input_and_mask, ) +from vllm.model_executor.models.deepseek_v2 import DeepSeekV2FusedQkvAProjLinear from vllm.platforms import current_platform from vllm.utils.torch_utils import set_random_seed @@ -1422,7 +1423,107 @@ def test_variable_slice_lora_class_selection(default_vllm_config, dist_init): f"for 2 packed modules, got {type(selected_layer_merged).__name__}" ) - # Case 5: Plain ColumnParallelLinear (not merged) - common in many models + fully_sharded_tp_lora_config = LoRAConfig( + max_loras=8, + max_lora_rank=16, + lora_dtype=torch.float16, + fully_sharded_loras=True, + ) + fully_sharded_tp_layer = MergedColumnParallelLinear( + 4096, [2048, 2048], bias=False, params_dtype=torch.float16 + ) + fully_sharded_tp_layer.tp_size = 2 + + assert not MergedColumnParallelLinearWithLoRA.can_replace_layer( + source_layer=fully_sharded_tp_layer, + lora_config=fully_sharded_tp_lora_config, + packed_modules_list=packed_modules_two, + ), "Generic merged wrapper should reject fully sharded TP layers" + + assert MergedColumnParallelLinearWithShardedLoRA.can_replace_layer( + source_layer=fully_sharded_tp_layer, + lora_config=fully_sharded_tp_lora_config, + packed_modules_list=packed_modules_two, + ), "Sharded merged wrapper should remain eligible for fully sharded TP layers" + + selected_fully_sharded_tp_layer = from_layer( + fully_sharded_tp_layer, + max_loras=8, + lora_config=fully_sharded_tp_lora_config, + packed_modules_list=packed_modules_two, + ) + assert isinstance( + selected_fully_sharded_tp_layer, + MergedColumnParallelLinearWithShardedLoRA, + ), ( + "from_layer should select MergedColumnParallelLinearWithShardedLoRA " + "for fully sharded TP merged layers, got " + f"{type(selected_fully_sharded_tp_layer).__name__}" + ) + + # Case 5: DeepSeek's fused_qkv_a_proj should reuse the generic merged + # wrapper while preserving its custom base forward path. + deepseek_fused_layer = DeepSeekV2FusedQkvAProjLinear( + 4096, [2048, 2048], prefix="model.layers.0.self_attn.fused_qkv_a_proj" + ) + selected_deepseek_layer = from_layer( + deepseek_fused_layer, + max_loras=8, + lora_config=lora_config, + packed_modules_list=packed_modules_two, + ) + assert isinstance(selected_deepseek_layer, MergedColumnParallelLinearWithLoRA), ( + "from_layer should select MergedColumnParallelLinearWithLoRA " + f"for DeepSeek fused_qkv_a_proj, got {type(selected_deepseek_layer).__name__}" + ) + + fully_sharded_lora_config = LoRAConfig( + max_loras=8, + max_lora_rank=16, + lora_dtype=torch.float16, + fully_sharded_loras=True, + ) + selected_fully_sharded_deepseek_layer = from_layer( + deepseek_fused_layer, + max_loras=8, + lora_config=fully_sharded_lora_config, + packed_modules_list=packed_modules_two, + ) + assert isinstance( + selected_fully_sharded_deepseek_layer, + MergedColumnParallelLinearWithLoRA, + ), ( + "from_layer should keep using MergedColumnParallelLinearWithLoRA " + "for fused_qkv_a_proj when the base layer is effectively unsharded, got " + f"{type(selected_fully_sharded_deepseek_layer).__name__}" + ) + + # Case 6: Generic subclass of MergedColumnParallelLinear with 2 packed + # modules should still use the generic merged wrapper. + class CustomMergedColumnParallelLinear(MergedColumnParallelLinear): + pass + + custom_merged_layer = CustomMergedColumnParallelLinear( + 4096, [2048, 2048], bias=False, params_dtype=torch.float16 + ) + assert MergedColumnParallelLinearWithLoRA.can_replace_layer( + source_layer=custom_merged_layer, + lora_config=lora_config, + packed_modules_list=packed_modules_two, + ), "MergedColumnParallelLinearWithLoRA should handle subclasses" + + selected_custom_layer = from_layer( + custom_merged_layer, + max_loras=8, + lora_config=lora_config, + packed_modules_list=packed_modules_two, + ) + assert isinstance(selected_custom_layer, MergedColumnParallelLinearWithLoRA), ( + f"from_layer should select MergedColumnParallelLinearWithLoRA " + f"for subclassed merged layers, got {type(selected_custom_layer).__name__}" + ) + + # Case 7: Plain ColumnParallelLinear (not merged) - common in many models # -> ColumnParallelLinearWithLoRA should be selected plain_column_parallel = ColumnParallelLinear( 4096, 4096, bias=False, params_dtype=torch.float16 @@ -1455,7 +1556,7 @@ def test_variable_slice_lora_class_selection(default_vllm_config, dist_init): f"for plain ColumnParallelLinear, got {type(selected_plain).__name__}" ) - # Case 6: MergedColumnParallelLinear with exactly 2 output sizes + # Case 8: MergedColumnParallelLinear with exactly 2 output sizes # and empty packed_modules_list # -> ColumnParallelLinearWithLoRA should NOT match (packed_modules_list != 1) # -> MergedColumnParallelLinearVariableSliceWithLoRA should NOT match (< 3 slices) @@ -1473,3 +1574,170 @@ def test_variable_slice_lora_class_selection(default_vllm_config, dist_init): "MergedColumnParallelLinearVariableSliceWithLoRA " "should NOT handle 2 slices even with empty packed_modules_list" ) + + +@pytest.mark.parametrize( + "wrapper_cls", + [ColumnParallelLinearWithLoRA, ColumnParallelLinearWithShardedLoRA], +) +def test_get_and_maybe_dequant_weights_accepts_lora_wrappers(dist_init, wrapper_cls): + from vllm.model_executor.layers.quantization.utils.quant_utils import ( + get_and_maybe_dequant_weights, + ) + + linear = ColumnParallelLinear(4096, 4096, bias=False, params_dtype=torch.float16) + lora_linear = wrapper_cls(linear) + + # Should work with LoRA wrappers and return [out, in] weights. + dequant_weight = get_and_maybe_dequant_weights(lora_linear, out_dtype=torch.float16) + assert dequant_weight.shape == linear.weight.shape + + +@torch.inference_mode() +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("stage", STAGES) +@pytest.mark.parametrize("fully_sharded", [False, True]) +def test_deepseek_fused_qkv_a_proj_lora_preserves_base_forward( + default_vllm_config, dist_init, device, stage, fully_sharded +): + if current_platform.is_cuda_alike(): + torch.accelerator.set_device_index(device) + + torch.set_default_device(device) + dtype = torch.float16 if current_platform.is_cuda_alike() else torch.float32 + max_loras = 8 + lora_config = LoRAConfig( + max_loras=max_loras, + max_lora_rank=8, + lora_dtype=dtype, + fully_sharded_loras=fully_sharded, + ) + punica_wrapper = get_punica_wrapper(8192, 256, device, lora_config=lora_config) + assert check_punica_wrapper(punica_wrapper) + + class OffsetDeepSeekFusedQkvAProjLinear(DeepSeekV2FusedQkvAProjLinear): + def forward(self, input_): + output, output_bias = super().forward(input_) + return output + 1, output_bias + + layer = OffsetDeepSeekFusedQkvAProjLinear( + 32, [16, 16], prefix="model.layers.0.self_attn.fused_qkv_a_proj" + ) + layer.weight.data = torch.rand_like(layer.weight.data, dtype=dtype) + + lora_layer = MergedColumnParallelLinearWithLoRA(layer) + lora_layer.create_lora_weights(max_loras, lora_config) + lora_layer.set_mapping(punica_wrapper) + + id_to_index = get_random_id_to_index(1, max_loras, log=False) + active_slot = next(i for i, lora_id in enumerate(id_to_index) if lora_id == 1) + lora_a = [ + torch.rand(8, 32, dtype=dtype, device=device), + torch.rand(8, 32, dtype=dtype, device=device), + ] + lora_b = [ + torch.rand(16, 8, dtype=dtype, device=device), + torch.rand(16, 8, dtype=dtype, device=device), + ] + lora_layer.set_lora(active_slot, lora_a=lora_a, lora_b=lora_b) + + inputs, index_mapping, prompt_mapping = create_random_inputs( + active_lora_ids=[1], + num_inputs=4, + input_size=(1, 32), + input_range=(0, 1), + input_type=dtype, + device=device, + ) + lora_mapping = LoRAMapping(index_mapping, prompt_mapping, is_prefill=stage) + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras, 512) + + lora_result = lora_layer(torch.cat(inputs))[0] + + expected_results = [] + for input_ in inputs: + result = layer(input_)[0] + result[:, :16] += input_ @ lora_a[0].T @ lora_b[0].T + result[:, 16:] += input_ @ lora_a[1].T @ lora_b[1].T + expected_results.append(result) + + rtol, atol = TOLERANCES[lora_result.dtype] + torch.testing.assert_close( + lora_result, torch.cat(expected_results), rtol=rtol, atol=atol + ) + + merged_layer = OffsetDeepSeekFusedQkvAProjLinear( + 32, [16, 16], prefix="model.layers.0.self_attn.fused_qkv_a_proj" + ) + merged_layer.weight.data = layer.weight.data.clone() + merged_layer.weight.data[:16].add_(lora_b[0] @ lora_a[0]) + merged_layer.weight.data[16:].add_(lora_b[1] @ lora_a[1]) + merged_result = merged_layer(torch.cat(inputs))[0] + + torch.testing.assert_close(lora_result, merged_result, rtol=rtol, atol=atol) + + +@torch.inference_mode() +@pytest.mark.parametrize("device", DEVICES) +@pytest.mark.parametrize("stage", STAGES) +def test_replicated_lora_preserves_base_forward_for_subclasses( + default_vllm_config, dist_init, device, stage +): + if current_platform.is_cuda_alike(): + torch.accelerator.set_device_index(device) + + torch.set_default_device(device) + dtype = torch.float16 if current_platform.is_cuda_alike() else torch.float32 + max_loras = 8 + lora_config = LoRAConfig(max_loras=max_loras, max_lora_rank=8, lora_dtype=dtype) + punica_wrapper = get_punica_wrapper(8192, 256, device, lora_config=lora_config) + assert check_punica_wrapper(punica_wrapper) + + class OffsetReplicatedLinear(ReplicatedLinear): + def forward(self, input_): + output, output_bias = super().forward(input_) + return output + 1, output_bias + + layer = OffsetReplicatedLinear(32, 16, bias=False, params_dtype=dtype) + layer.weight.data = torch.rand_like(layer.weight.data, dtype=dtype) + + lora_layer = ReplicatedLinearWithLoRA(layer) + lora_layer.create_lora_weights(max_loras, lora_config) + lora_layer.set_mapping(punica_wrapper) + + id_to_index = get_random_id_to_index(1, max_loras, log=False) + active_slot = next(i for i, lora_id in enumerate(id_to_index) if lora_id == 1) + lora_a = torch.rand(8, 32, dtype=dtype, device=device) + lora_b = torch.rand(16, 8, dtype=dtype, device=device) + lora_layer.set_lora(active_slot, lora_a=lora_a, lora_b=lora_b) + + inputs, index_mapping, prompt_mapping = create_random_inputs( + active_lora_ids=[1], + num_inputs=4, + input_size=(1, 32), + input_range=(0, 1), + input_type=dtype, + device=device, + ) + lora_mapping = LoRAMapping(index_mapping, prompt_mapping, is_prefill=stage) + punica_wrapper.update_metadata(lora_mapping, id_to_index, max_loras, 512) + + lora_result = lora_layer(torch.cat(inputs))[0] + + expected_results = [] + for input_ in inputs: + result = layer(input_)[0] + result += input_ @ lora_a.T @ lora_b.T + expected_results.append(result) + + rtol, atol = TOLERANCES[lora_result.dtype] + torch.testing.assert_close( + lora_result, torch.cat(expected_results), rtol=rtol, atol=atol + ) + + merged_layer = OffsetReplicatedLinear(32, 16, bias=False, params_dtype=dtype) + merged_layer.weight.data = layer.weight.data.clone() + merged_layer.weight.data.add_(lora_b @ lora_a) + merged_result = merged_layer(torch.cat(inputs))[0] + + torch.testing.assert_close(lora_result, merged_result, rtol=rtol, atol=atol) diff --git a/tests/lora/test_lora_manager.py b/tests/lora/test_lora_manager.py index e80d96f00e7..1c07dc4ae67 100644 --- a/tests/lora/test_lora_manager.py +++ b/tests/lora/test_lora_manager.py @@ -13,6 +13,7 @@ from vllm.config.lora import LoRAConfig from vllm.lora.layers import ( ColumnParallelLinearWithLoRA, MergedColumnParallelLinearWithLoRA, + ReplicatedLinearWithLoRA, RowParallelLinearWithLoRA, ) from vllm.lora.lora_model import LoRAModel @@ -26,6 +27,7 @@ from vllm.lora.model_manager import ( from vllm.lora.peft_helper import PEFTHelper from vllm.lora.request import LoRARequest from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager, WorkerLoRAManager +from vllm.model_executor.layers.fused_moe import GateLinear from vllm.platforms import current_platform from .utils import create_peft_lora @@ -132,6 +134,135 @@ def test_replace_submodules(default_vllm_config, dist_init, dummy_model): assert isinstance(model.get_submodule("layer1.dense2"), RowParallelLinearWithLoRA) +def test_wrap_replicated_linear_subclasses(default_vllm_config, dist_init, dummy_model): + from vllm.model_executor.layers.linear import ReplicatedLinear + + class CustomReplicatedLinear(ReplicatedLinear): + pass + + model = dummy_model + model.add_module("custom_gate", CustomReplicatedLinear(10, 10, bias=False)) + + manager = LoRAModelManager( + model, + 1, + 1, + 1, + LoRAConfig( + max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE + ), + torch.device(DEVICES[0]), + ) + + assert isinstance( + manager.model.get_submodule("custom_gate"), ReplicatedLinearWithLoRA + ) + + +def test_wrap_gate_linear(default_vllm_config, dist_init, dummy_model): + model = dummy_model + model.add_module("router_gate", GateLinear(10, 4, bias=False)) + + manager = LoRAModelManager( + model, + 1, + 1, + 1, + LoRAConfig( + max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE + ), + torch.device(DEVICES[0]), + ) + + assert isinstance( + manager.model.get_submodule("router_gate"), ReplicatedLinearWithLoRA + ) + + +def test_skip_unsupported_matched_modules(default_vllm_config, dist_init, dummy_model): + class UnsupportedContainer(nn.Module): + def __init__(self): + super().__init__() + # This name matches a supported target suffix ("dense1"), + # but nn.Linear is not currently a LoRA-wrappable layer type. + self.dense1 = nn.Linear(10, 10, bias=False) + + model = dummy_model + model.add_module("unsupported", UnsupportedContainer()) + + manager = LoRAModelManager( + model, + 1, + 1, + 1, + LoRAConfig( + max_lora_rank=8, max_cpu_loras=8, max_loras=8, lora_dtype=DEFAULT_DTYPE + ), + torch.device(DEVICES[0]), + ) + + # Should not crash and should keep unsupported matched modules unchanged. + assert isinstance(manager.model.get_submodule("unsupported.dense1"), nn.Linear) + assert "unsupported.dense1" not in manager.modules + + +def test_target_modules_fail_closed_on_unsupported_matched_modules( + default_vllm_config, dist_init, dummy_model +): + class UnsupportedContainer(nn.Module): + def __init__(self): + super().__init__() + self.dense1 = nn.Linear(10, 10, bias=False) + + model = dummy_model + model.add_module("unsupported", UnsupportedContainer()) + + with pytest.raises(ValueError, match="unsupported.dense1"): + LoRAModelManager( + model, + 1, + 1, + 1, + LoRAConfig( + max_lora_rank=8, + max_cpu_loras=8, + max_loras=8, + lora_dtype=DEFAULT_DTYPE, + target_modules=["dense1"], + ), + torch.device(DEVICES[0]), + ) + + +def test_get_dummy_lora_warmup_rank_for_fully_sharded_moe(): + manager = LoRAModelManager.__new__(LoRAModelManager) + manager.lora_config = LoRAConfig( + max_lora_rank=64, + max_cpu_loras=1, + max_loras=1, + lora_dtype=DEFAULT_DTYPE, + fully_sharded_loras=True, + ) + + class DummyModule: + def __init__(self, tp_size: int, fully_sharded: bool): + self.tp_size = tp_size + self.fully_sharded = fully_sharded + + manager.modules = { + "model.layers.0.self_attn.q_proj": DummyModule( + tp_size=32, + fully_sharded=True, + ), + "model.layers.0.mlp.experts": DummyModule( + tp_size=32, + fully_sharded=True, + ), + } + + assert manager.get_dummy_lora_warmup_rank(8) == 32 + + @pytest.mark.parametrize("device", DEVICES) def test_lora_model_manager(default_vllm_config, dist_init, dummy_model, device): model = dummy_model @@ -795,6 +926,25 @@ def test_target_modules_none_uses_all( ) +@pytest.mark.parametrize("device", DEVICES) +def test_target_modules_match_packed_runtime_modules( + default_vllm_config, dist_init, dummy_model_gate_up, device +): + """Packed runtime modules should be selected by their adapter-visible names.""" + _test_target_modules( + dummy_model_gate_up, + ["gate_proj"], + device, + expected_lora=[("gate_up_proj", MergedColumnParallelLinearWithLoRA)], + expected_no_lora=[ + ("dense1", ColumnParallelLinearWithLoRA), + ("dense2", RowParallelLinearWithLoRA), + ("layer1.dense1", ColumnParallelLinearWithLoRA), + ("layer1.dense2", RowParallelLinearWithLoRA), + ], + ) + + @pytest.mark.parametrize("device", DEVICES) def test_load_adapter_warns_on_unsupported_modules( default_vllm_config, dist_init, dummy_model_gate_up, device, tmp_path diff --git a/tests/lora/test_lora_utils.py b/tests/lora/test_lora_utils.py index da66aa60b0d..603ec929749 100644 --- a/tests/lora/test_lora_utils.py +++ b/tests/lora/test_lora_utils.py @@ -58,3 +58,24 @@ class TestIsInTargetModules: def test_exact_name_no_match(self): assert not is_in_target_modules("dense3", ["dense1", "dense2"]) + + def test_packed_parent_matches_child_target_modules(self): + assert is_in_target_modules( + "model.layers.0.mlp.gate_up_proj", + ["gate_proj", "up_proj"], + {"gate_up_proj": ["gate_proj", "up_proj"]}, + ) + + def test_packed_child_matches_parent_target_modules(self): + assert is_in_target_modules( + "model.layers.0.mlp.gate_proj", + ["gate_up_proj"], + {"gate_up_proj": ["gate_proj", "up_proj"]}, + ) + + def test_fused_parent_matches_child_target_modules(self): + assert is_in_target_modules( + "model.layers.0.self_attn.fused_qkv_a_proj", + ["q_a_proj", "kv_a_proj_with_mqa"], + {"fused_qkv_a_proj": ["q_a_proj", "kv_a_proj_with_mqa"]}, + ) diff --git a/vllm/lora/layers/base_linear.py b/vllm/lora/layers/base_linear.py index 4ea6b1ec8f0..68783ae50d4 100644 --- a/vllm/lora/layers/base_linear.py +++ b/vllm/lora/layers/base_linear.py @@ -203,7 +203,16 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA): self, x: torch.Tensor, bias: torch.Tensor | None = None ) -> torch.Tensor: output = self.base_layer.quant_method.apply(self.base_layer, x, bias) + return self._apply_lora_to_output(x, output) + def _apply_base_forward(self, x: torch.Tensor) -> torch.Tensor: + base_output = self.base_layer(x) + output = base_output[0] if isinstance(base_output, tuple) else base_output + return self._apply_lora_to_output(x, output) + + def _apply_lora_to_output( + self, x: torch.Tensor, output: torch.Tensor + ) -> torch.Tensor: original_shape = output.shape if output.ndim == 3 else None # In transformers backend, x and output have extra batch dimension like diff --git a/vllm/lora/layers/column_parallel_linear.py b/vllm/lora/layers/column_parallel_linear.py index f49a3fcbb94..aed6b5ba891 100644 --- a/vllm/lora/layers/column_parallel_linear.py +++ b/vllm/lora/layers/column_parallel_linear.py @@ -40,11 +40,19 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"): # Since communication is needed, the buffer is directly initialized as a # tensor rather than a tuple of tensor. - buffers = torch.zeros( - (layer.n_slices, x.shape[0], layer.lora_a_stacked[0].shape[2]), + local_lora_rank = layer.lora_a_stacked[0].shape[2] + buffer_shape = (layer.n_slices, x.shape[0], local_lora_rank) + # Under torch.compile, the local-rank-1 fully-sharded path can otherwise + # get lowered to a reinterpret view with a non-canonical layout. The + # Triton shrink op mutates this buffer in place and expects the standard + # contiguous [slice, token, rank] stride contract. + buffers = torch.empty_strided( + buffer_shape, + (x.shape[0] * local_lora_rank, local_lora_rank, 1), dtype=torch.float32, device=x.device, ) + buffers.zero_() shrunk_buffers: torch.Tensor | None = layer.punica_wrapper.add_shrink( buffers, x, layer.lora_a_stacked, 1.0 @@ -86,7 +94,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA): # The base_layer type is ColumnParallelLinear or # MergedColumnParallelLinear, their weight sharding logic is # inconsistent when TP is greater than 1. - self.is_merged_col_linear = type(base_layer) is MergedColumnParallelLinear + self.is_merged_col_linear = isinstance(base_layer, MergedColumnParallelLinear) self.output_size = self.base_layer.output_size_per_partition # There is only one LoRA layer self.n_slices = 1 @@ -158,7 +166,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA): ) -> bool: if type(source_layer) is maybe_get_oot_by_class(ColumnParallelLinear): return True - if type(source_layer) is maybe_get_oot_by_class(MergedColumnParallelLinear): + if isinstance(source_layer, maybe_get_oot_by_class(MergedColumnParallelLinear)): if len(packed_modules_list) != 1: return False # Exclude layers with 3+ output sizes - those are handled by @@ -275,19 +283,41 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): index, 0, : lora_b_i.shape[0], : lora_b_i.shape[1] ].copy_(lora_b_i, non_blocking=True) + def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: + merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear) + # Effectively unsharded subclasses can safely reuse their custom + # forward() implementation before applying the LoRA delta. + if ( + self.tp_size == 1 + and type(self.base_layer) is not merged_cls + and type(self.base_layer).forward is not merged_cls.forward + ): + return self._apply_base_forward(x) + return _mcp_apply(x, bias, self) + @classmethod - @_not_fully_sharded_can_replace def can_replace_layer( cls, source_layer: nn.Module, lora_config: LoRAConfig, packed_modules_list: list, model_config: PretrainedConfig | None = None, + decorate: bool = True, ) -> bool: - return ( - type(source_layer) is MergedColumnParallelLinear - and len(packed_modules_list) == 2 - ) + merged_cls = maybe_get_oot_by_class(MergedColumnParallelLinear) + if not isinstance(source_layer, merged_cls) or len(packed_modules_list) != 2: + return False + + tp_size = getattr(source_layer, "tp_size", 1) + if type(source_layer) is merged_cls: + if not decorate: + return True + return not lora_config.fully_sharded_loras or tp_size == 1 + + # Only support effectively unsharded subclasses here. Sharded + # subclasses may have custom communication semantics that the generic + # merged-column LoRA path does not know how to preserve. + return tp_size == 1 class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA): @@ -607,7 +637,9 @@ class MergedColumnParallelLinearVariableSliceWithLoRA( ) -> bool: # Support MergedColumnParallelLinear with 3 or more slices # (2 slices are handled by MergedColumnParallelLinearWithLoRA) - if type(source_layer) is not maybe_get_oot_by_class(MergedColumnParallelLinear): + if not isinstance( + source_layer, maybe_get_oot_by_class(MergedColumnParallelLinear) + ): return False # If packed_modules_list has 3+ items, use this class diff --git a/vllm/lora/layers/replicated_linear.py b/vllm/lora/layers/replicated_linear.py index f1f499b841b..53ae26be4c3 100644 --- a/vllm/lora/layers/replicated_linear.py +++ b/vllm/lora/layers/replicated_linear.py @@ -46,6 +46,12 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA): return output, output_bias + def apply(self, x: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: + # ReplicatedLinear subclasses such as GateLinear override forward() to + # dispatch custom kernels and/or adjust the output dtype. Apply LoRA on + # top of the actual base-layer output instead of bypassing that path. + return self._apply_base_forward(x) + # ReplicatedLinear should always be replaced, regardless of the fully # sharded LoRAs setting, because it is, by definition, copied per GPU. @classmethod @@ -56,7 +62,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA): packed_modules_list: list, model_config: PretrainedConfig | None = None, ) -> bool: - return type(source_layer) is maybe_get_oot_by_class(ReplicatedLinear) + return isinstance(source_layer, maybe_get_oot_by_class(ReplicatedLinear)) def slice_lora_a( self, lora_a: torch.Tensor | list[torch.Tensor | None] diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index 9d377256043..3b58031dcba 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -437,12 +437,21 @@ class LoRAModelManager: ), ) - # In some models, especially multimodal ones, layers with the same - # name may have different types, such as nn.Linear and - # ReplicatedLinear. The nn.Linear layers cannot be replaced with - # LoRA layers, leading to assertion error. The following check - # aims to prevent this error - if self.supports_mm and not isinstance(new_module, BaseLayerWithLoRA): + # Some matched modules can be unsupported by LoRA wrappers + # (e.g. subclasses with specialized forward behavior). + if not isinstance(new_module, BaseLayerWithLoRA): + error_msg = ( + "LoRA target module " + f"{module_name} ({type(module).__name__}) matched the " + "deployment configuration but could not be wrapped by any " + "LoRA layer implementation." + ) + if self.lora_config.target_modules is not None: + raise ValueError( + f"{error_msg} target_modules=" + f"{sorted(self.lora_config.target_modules)}" + ) + logger.warning_once("%s It will be ignored.", error_msg) continue self.register_module(module_name, new_module) @@ -578,6 +587,38 @@ class LoRAModelManager: model.loras[module_name] = lora return model + def get_dummy_lora_warmup_rank(self, default_rank: int) -> int: + """Return a dummy LoRA rank compatible with wrapped modules. + + Dummy LoRAs keep warmup memory low by using a small rank. Fully + sharded MoE wrappers additionally require the dummy rank to be divisible + by tensor parallel size because they shard W13 along the rank axis. + """ + if not self.lora_config.fully_sharded_loras: + return default_rank + + required_multiple = 1 + for module in self.modules.values(): + if not getattr(module, "fully_sharded", False): + continue + required_multiple = math.lcm(required_multiple, module.tp_size) + + if required_multiple == 1 or default_rank % required_multiple == 0: + return default_rank + + adjusted_rank = ( + (default_rank + required_multiple - 1) // required_multiple + ) * required_multiple + if adjusted_rank > self.lora_config.max_lora_rank: + raise ValueError( + "Unable to choose a dummy LoRA warmup rank compatible with " + "fully sharded MoE modules: " + f"default_rank={default_rank}, " + f"required_multiple={required_multiple}, " + f"max_lora_rank={self.lora_config.max_lora_rank}" + ) + return adjusted_rank + def _match_target_modules(self, module_name: str) -> bool: """Check if a module should have LoRA applied. @@ -594,7 +635,11 @@ class LoRAModelManager: """ if not is_supported_lora_module(module_name, self.supported_lora_modules): return False - return is_in_target_modules(module_name, self.lora_config.target_modules) + return is_in_target_modules( + module_name, + self.lora_config.target_modules, + self.packed_modules_mapping, + ) def _get_punica_wrapper(self, module_name: str) -> PunicaWrapperBase | None: """ diff --git a/vllm/lora/utils.py b/vllm/lora/utils.py index 2349ace7084..2991447a6ad 100644 --- a/vllm/lora/utils.py +++ b/vllm/lora/utils.py @@ -73,7 +73,9 @@ def get_lora_id(): return _GLOBAL_LORA_ID -_all_lora_classes: set[type[BaseLayerWithLoRA]] = { +# Order matters here: more specific wrappers must be checked before generic +# merged/column-parallel wrappers in from_layer(). +_all_lora_classes: tuple[type[BaseLayerWithLoRA], ...] = ( VocabParallelEmbeddingWithLoRA, ColumnParallelLinearWithLoRA, MergedColumnParallelLinearWithLoRA, @@ -90,7 +92,7 @@ _all_lora_classes: set[type[BaseLayerWithLoRA]] = { RowParallelLinearWithShardedLoRA, FusedMoEWithLoRA, FusedMoE3DWithLoRA, -} +) def is_moe_model(model: nn.Module) -> bool: @@ -258,6 +260,7 @@ def is_supported_lora_module( def is_in_target_modules( module_name: str, target_modules: list[str] | None, + packed_modules_mapping: dict[str, list[str]] | None = None, ) -> bool: """Check if a module passes the deployment-time target_modules filter. @@ -268,14 +271,33 @@ def is_in_target_modules( module_name: Full dot-separated module name. target_modules: Optional deployment-time restriction list from LoRAConfig.target_modules. + packed_modules_mapping: Optional model-defined mapping from packed + runtime module names to their adapter-visible submodule names + (e.g. ``{"gate_up_proj": ["gate_proj", "up_proj"]}``). Returns: True if the module passes the filter, False otherwise. """ if target_modules is None: return True + target_module_set = set(target_modules) module_suffix = module_name.split(".")[-1] - return module_suffix in set(target_modules) + if module_suffix in target_module_set or module_name in target_module_set: + return True + + if not packed_modules_mapping: + return False + + # Runtime packed parent matched by deployment-time child targets. + packed_children = packed_modules_mapping.get(module_suffix) + if packed_children and any(child in target_module_set for child in packed_children): + return True + + # Adapter-visible packed child matched by deployment-time parent target. + return any( + module_suffix in children and packed_parent in target_module_set + for packed_parent, children in packed_modules_mapping.items() + ) def get_adapter_absolute_path(lora_path: str) -> str: diff --git a/vllm/lora/worker_manager.py b/vllm/lora/worker_manager.py index bea6d015e0a..6d8ef2db51a 100644 --- a/vllm/lora/worker_manager.py +++ b/vllm/lora/worker_manager.py @@ -160,7 +160,11 @@ class WorkerLoRAManager: lora_request.lora_path, ", ".join(sorted(expected_lora_modules_lst)), ) - elif not is_in_target_modules(module_name, target_modules): + elif not is_in_target_modules( + module_name, + target_modules, + packed_modules_mapping, + ): logger.warning_once( "LoRA module '%s' in adapter '%s' is not in the " "deployment-time target_modules restriction [%s]." @@ -197,6 +201,9 @@ class WorkerLoRAManager: self._cached_dummy_lora = dummy_lora return self._adapter_manager.add_adapter(dummy_lora) + def get_dummy_lora_warmup_rank(self, default_rank: int) -> int: + return self._adapter_manager.get_dummy_lora_warmup_rank(default_rank) + def pin_adapter(self, adapter_id: int) -> bool: return self._adapter_manager.pin_adapter(adapter_id) diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index 8fcb8fa1da1..af7cb7baf96 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -214,6 +214,19 @@ def select_unquantized_moe_backend( return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) + # LoRA needs Triton's unfused activation/reduction hooks. Selecting the + # backend here ensures weights stay in a LoRA-compatible layout instead of + # being permuted for a backend like FlashInfer or AITER during load. + if moe_config.is_lora_enabled: + backend = UnquantizedMoeBackend.TRITON + if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + backend = UnquantizedMoeBackend.BATCHED_TRITON + return _return_or_raise( + backend, + moe_config, + activation_format, + ) + runner_backend = moe_config.moe_backend if runner_backend != "auto": requested_backend = map_unquantized_backend(runner_backend) diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index de76deb191d..f57eb39f42b 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -356,6 +356,12 @@ def get_and_maybe_dequant_weights( from vllm.model_executor.layers.linear import UnquantizedLinearMethod from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod + # LoRA linear wrappers store quantization metadata on `base_layer`. + # Unwrap here so callers can pass either a raw linear layer or its LoRA + # wrapper without special-casing. + while hasattr(layer, "base_layer") and hasattr(layer.base_layer, "quant_method"): + layer = layer.base_layer + weight = get_attribute_fallback(layer, ["weight", "qweight", "weight_packed"]) # Unquantized layer: just return base weights diff --git a/vllm/v1/worker/lora_model_runner_mixin.py b/vllm/v1/worker/lora_model_runner_mixin.py index 53873d156f8..3a14abfc358 100644 --- a/vllm/v1/worker/lora_model_runner_mixin.py +++ b/vllm/v1/worker/lora_model_runner_mixin.py @@ -101,9 +101,12 @@ class LoRAModelRunnerMixin: assert self.lora_manager is not None, "LoRA is not enabled" num_loras = lora_config.max_loras - lora_warmup_rank = ( + lora_warmup_rank: int = ( lora_config.max_lora_rank if lora_config.max_lora_rank < 8 else 8 ) + lora_warmup_rank = self.lora_manager.get_dummy_lora_warmup_rank( + lora_warmup_rank + ) # Make dummy lora requests lora_requests: set[LoRARequest] = { LoRARequest( From ecbe42e9911bd0c0ae471ed573ea8b2d050a0e8e Mon Sep 17 00:00:00 2001 From: xiao <102247755+Wangxiaoxiaoa@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:36:17 +0800 Subject: [PATCH 037/153] [Doc] Clarify supported keys for --speculative-config (#40455) Signed-off-by: Wangxiaoxiaoa Co-authored-by: Wangxiaoxiaoa --- docs/features/speculative_decoding/README.md | 93 +++++++++++++++++++ .../speculative_decoding/draft_model.md | 12 ++- docs/features/speculative_decoding/mtp.md | 2 +- .../parallel_draft_model.md | 6 +- 4 files changed, 105 insertions(+), 8 deletions(-) diff --git a/docs/features/speculative_decoding/README.md b/docs/features/speculative_decoding/README.md index 9793de3f4c3..25cda8059b2 100644 --- a/docs/features/speculative_decoding/README.md +++ b/docs/features/speculative_decoding/README.md @@ -35,6 +35,99 @@ For reproducible measurements in your environment, use [`examples/offline_inference/spec_decode.py`](../../../examples/offline_inference/spec_decode.py) or the [benchmark CLI guide](../../benchmarking/cli.md). +## `--speculative-config` schema + +Use `--speculative-config` to pass speculative decoding settings as a JSON +object on the CLI: + +```bash +vllm serve \ + --speculative-config '{ + "method": "draft_model", + "model": "", + "num_speculative_tokens": 5 + }' +``` + +The same keys are accepted from Python via `LLM(..., speculative_config={...})`. +The tables below highlight common user-facing keys accepted in this JSON +object; they are not an exhaustive schema reference. +For more details, see the generated [engine arguments reference](../../configuration/engine_args.md) +and the API docs for [vllm.config.SpeculativeConfig][]. + +### Common keys + +These keys are commonly used across speculative decoding setups, though some +only apply to model-based methods such as `draft_model`, `mtp`, `eagle3`, and +`dflash`. + +| Key | Type | Default | Allowed values / meaning | +| --- | --- | --- | --- | +| `method` | `string` | `None` | Speculation method. Common values include `draft_model`, `ngram`, `suffix`, `mtp`, `eagle3`, and `dflash`. If omitted, vLLM infers the method from the provided configuration when possible. | +| `model` | `string` | `None` | Draft model, EAGLE head, or auxiliary model identifier. For `ngram`, `ngram_gpu`, `suffix`, and `mtp`, this can often be omitted. | +| `num_speculative_tokens` | `integer > 0` | `None` | Number of speculative tokens to propose per step. Required for methods that do not infer it from model metadata. | +| `draft_tensor_parallel_size` | `integer >= 1` | `None` | Tensor parallel size for the draft model. | +| `max_model_len` | `integer >= 1` | `None` | Maximum context length for the draft model. | +| `parallel_drafting` | `boolean` | `false` | Enable parallel draft token generation. Only compatible with EAGLE and draft-model methods. | +| `rejection_sample_method` | `string` | `strict` | `strict`, `probabilistic`, or `synthetic`. | +| `synthetic_acceptance_rate` | `float` | `None` | Average acceptance rate to target when `rejection_sample_method` is `synthetic`. Valid range is `[0, 1]`. | + +### Method-specific keys + +#### N-gram + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `prompt_lookup_max` | `integer >= 1` | `5` if both lookup bounds are omitted; otherwise mirrors `prompt_lookup_min` when omitted | Maximum n-gram window size. | +| `prompt_lookup_min` | `integer >= 1` | `5` if both lookup bounds are omitted; otherwise mirrors `prompt_lookup_max` when omitted | Minimum n-gram window size. | + +Example: + +```bash +vllm serve \ + --speculative-config '{ + "method": "ngram", + "num_speculative_tokens": 4, + "prompt_lookup_min": 2, + "prompt_lookup_max": 5 + }' +``` + +#### Suffix decoding + +| Key | Type | Default | Meaning | +| --- | --- | --- | --- | +| `suffix_decoding_max_tree_depth` | `integer` | `24` | Maximum combined prefix-match and speculation tree depth. | +| `suffix_decoding_max_cached_requests` | `integer` | `10000` | Maximum number of requests cached in the global suffix tree. Set `0` to disable the global cache. | +| `suffix_decoding_max_spec_factor` | `float` | `1.0` | Caps speculative length as a multiple of prefix-match length. | +| `suffix_decoding_min_token_prob` | `float` | `0.1` | Minimum estimated token probability required to speculate a token. | + +Example: + +```bash +vllm serve \ + --speculative-config '{ + "method": "suffix", + "num_speculative_tokens": 8, + "suffix_decoding_max_tree_depth": 24, + "suffix_decoding_max_cached_requests": 10000, + "suffix_decoding_max_spec_factor": 1.0, + "suffix_decoding_min_token_prob": 0.1 + }' +``` + +### Notes + +- `--speculative-config` expects a JSON object on the CLI. In YAML config + files, use a nested mapping instead of an escaped JSON string. +- `tensor_parallel_size` is not a valid key in `speculative_config`. Use + `draft_tensor_parallel_size` instead. +- Keys such as `temperature` and `top_p` are sampling parameters, not + `--speculative-config` fields. +- Internal fields such as `target_model_config`, `draft_model_config`, + `target_parallel_config`, `draft_parallel_config`, and `draft_load_config` + are populated by vLLM and are not intended to be set by users. + ## Lossless guarantees of Speculative Decoding In vLLM, speculative decoding aims to enhance inference efficiency while maintaining accuracy. This section addresses the lossless guarantees of diff --git a/docs/features/speculative_decoding/draft_model.md b/docs/features/speculative_decoding/draft_model.md index ee0eaf176e7..b4662e6438f 100644 --- a/docs/features/speculative_decoding/draft_model.md +++ b/docs/features/speculative_decoding/draft_model.md @@ -33,9 +33,9 @@ vllm serve Qwen/Qwen3-4B-Thinking-2507 \ --port 8000 \ --seed 42 \ -tp 1 \ - --max_model_len 2048 \ - --gpu_memory_utilization 0.8 \ - --speculative_config '{"model": "Qwen/Qwen3-0.6B", "num_speculative_tokens": 5, "method": "draft_model"}' + --max-model-len 2048 \ + --gpu-memory-utilization 0.8 \ + --speculative-config '{"model": "Qwen/Qwen3-0.6B", "num_speculative_tokens": 5, "method": "draft_model"}' ``` The code used to request as completions as a client remains unchanged: @@ -77,4 +77,8 @@ The code used to request as completions as a client remains unchanged: ``` !!! warning - Note: Please use `--speculative_config` to set all configurations related to speculative decoding. The previous method of specifying the model through `--speculative_model` and adding related parameters (e.g., `--num_speculative_tokens`) separately has been deprecated. + Note: Please use `--speculative-config` to set all configurations related + to speculative decoding. The previous method of specifying the model + through `--speculative-model` and adding related parameters such as + `--num-speculative-tokens` separately has been deprecated. For supported + keys and examples, see the [`--speculative-config` schema](README.md#--speculative-config-schema). diff --git a/docs/features/speculative_decoding/mtp.md b/docs/features/speculative_decoding/mtp.md index bcd7153deb5..7e1d1ec7038 100644 --- a/docs/features/speculative_decoding/mtp.md +++ b/docs/features/speculative_decoding/mtp.md @@ -38,7 +38,7 @@ for output in outputs: ```bash vllm serve XiaomiMiMo/MiMo-7B-Base \ --tensor-parallel-size 1 \ - --speculative_config '{"method":"mtp","num_speculative_tokens":1}' + --speculative-config '{"method":"mtp","num_speculative_tokens":1}' ``` ## Notes diff --git a/docs/features/speculative_decoding/parallel_draft_model.md b/docs/features/speculative_decoding/parallel_draft_model.md index 2a3f11a302d..c31b8e2d2f4 100644 --- a/docs/features/speculative_decoding/parallel_draft_model.md +++ b/docs/features/speculative_decoding/parallel_draft_model.md @@ -36,9 +36,9 @@ vllm serve Qwen/Qwen3-4B \ --port 8000 \ --seed 42 \ -tp 1 \ - --max_model_len 2048 \ - --gpu_memory_utilization 0.8 \ - --speculative_config '{"model": "amd/PARD-Qwen3-0.6B", "num_speculative_tokens": 12, "method": "draft_model", "parallel_drafting": true}' + --max-model-len 2048 \ + --gpu-memory-utilization 0.8 \ + --speculative-config '{"model": "amd/PARD-Qwen3-0.6B", "num_speculative_tokens": 12, "method": "draft_model", "parallel_drafting": true}' ``` ## Pre-trained PARD weights From a4905133f375ec721be441e7ec4a3f923daa28f3 Mon Sep 17 00:00:00 2001 From: Hank_ <37239608+ILikeIneine@users.noreply.github.com> Date: Wed, 22 Apr 2026 19:39:40 +0800 Subject: [PATCH 038/153] [xpu][rocm] Update `current_platform.supports_fp8()` for TritonExperts (#40132) Signed-off-by: Hank --- .../layers/fused_moe/fused_moe.py | 19 +------------------ vllm/platforms/rocm.py | 2 +- vllm/platforms/xpu.py | 4 ++++ 3 files changed, 6 insertions(+), 19 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 9218d0aff74..3b12f294939 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -1952,24 +1952,7 @@ class TritonExperts(mk.FusedMoEExpertsModular): weight_key: QuantKey | None, activation_key: QuantKey | None, ) -> bool: - p = current_platform - if p.is_rocm(): - from vllm.platforms.rocm import on_gfx9, on_gfx12x - - is_rocm_on_gfx9 = on_gfx9() - is_rocm_on_gfx12x = on_gfx12x() - else: - is_rocm_on_gfx9 = False - is_rocm_on_gfx12x = False - - device_supports_fp8 = ( - is_rocm_on_gfx9 - or is_rocm_on_gfx12x - or (p.is_cuda() and p.has_device_capability((8, 9))) - or p.is_xpu() - ) - - if not device_supports_fp8: + if not current_platform.supports_fp8(): return (weight_key, activation_key) == (None, None) SUPPORTED_W_A = [ diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 89714f00f64..0801c852423 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -800,7 +800,7 @@ class RocmPlatform(Platform): @classmethod def supports_fp8(cls) -> bool: - return any(gfx in _GCN_ARCH for gfx in ["gfx94", "gfx95", "gfx12"]) + return on_gfx9() or on_gfx12x() @classmethod def is_fp8_fnuz(cls) -> bool: diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index aa673419730..d52ba23243f 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -323,6 +323,10 @@ class XPUPlatform(Platform): ) return "vllm.distributed.device_communicators.xpu_communicator.XpuCommunicator" # noqa + @classmethod + def supports_fp8(cls) -> bool: + return True + @classmethod def get_default_ir_op_priority( cls, vllm_config: "VllmConfig" From 33ef1941e217a2126d745caec6c6130d6aec3b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Wed, 22 Apr 2026 15:21:02 +0200 Subject: [PATCH 039/153] [Bugfix][CI] Fix `v1/kv_connector/unit/test_nixl_connector_hma.py::test_fewer_blocks_with_hma` (#40597) Signed-off-by: NickLucche --- tests/v1/kv_connector/unit/test_nixl_connector_hma.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 5b609017359..3f5a9b9cc03 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -2,9 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Unit tests for NixlConnectorScheduler with HMA and Mamba N-1 prefill.""" +import gc from unittest.mock import patch import pytest +import torch from vllm import LLM, SamplingParams from vllm.config import KVTransferConfig @@ -196,12 +198,13 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size): llm_kwargs = { "model": model_name, "enforce_eager": True, - "gpu_memory_utilization": 0.47, + "gpu_memory_utilization": 0.3, "kv_transfer_config": kv_transfer_config, "max_model_len": 2048, + "max_num_seqs": 1, # NOTE: Make sure HMA is enabled "disable_hybrid_kv_cache_manager": False, - "max_num_batched_tokens": 1024, + "max_num_batched_tokens": 2048, "enable_prefix_caching": False, "block_size": block_size, } @@ -248,6 +251,8 @@ def test_fewer_blocks_with_hma(monkeypatch, model_name, sw_size): assert len(group_block_ids) == expected_num_remote_blocks def run_test_and_cleanup(): + gc.collect() + torch.accelerator.empty_cache() llm = LLM(**llm_kwargs) try: run_hma_test(llm) From 809d83c2dc2f74d52fe852b09d0e38c91c7d8334 Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:43:17 -0400 Subject: [PATCH 040/153] [MoE Refactor] Combine MoERunnerBase + DefaultMoERunner (#40560) Signed-off-by: Bill Nell --- vllm/model_executor/layers/fused_moe/layer.py | 9 +- .../fused_moe/runner/default_moe_runner.py | 128 ---- .../layers/fused_moe/runner/moe_runner.py | 699 +++++++++++++++++- .../fused_moe/runner/moe_runner_base.py | 639 ---------------- .../fused_moe/runner/moe_runner_factory.py | 47 -- .../fused_moe/runner/moe_runner_interface.py | 44 ++ 6 files changed, 730 insertions(+), 836 deletions(-) delete mode 100644 vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py delete mode 100644 vllm/model_executor/layers/fused_moe/runner/moe_runner_base.py delete mode 100644 vllm/model_executor/layers/fused_moe/runner/moe_runner_factory.py create mode 100644 vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index bf10bc9d5c4..7adac0374cf 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -38,8 +38,11 @@ from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( from vllm.model_executor.layers.fused_moe.router.router_factory import ( create_fused_moe_router, ) -from vllm.model_executor.layers.fused_moe.runner.moe_runner_factory import ( - create_moe_runner, +from vllm.model_executor.layers.fused_moe.runner.moe_runner import ( + MoERunner, +) +from vllm.model_executor.layers.fused_moe.runner.moe_runner_interface import ( + MoERunnerInterface, ) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, @@ -586,7 +589,7 @@ class FusedMoE(PluggableLayer): # Storing the runner in the FusedMoE is an intermediate state, eventually # the runner will own the FusedMoE layer and provide the execution interface # for MoE ops. - self.runner = create_moe_runner( + self.runner: MoERunnerInterface = MoERunner( layer_name=self.layer_name, moe_config=self.moe_config, router=self.router, diff --git a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py deleted file mode 100644 index df4c0c86924..00000000000 --- a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.distributed import ( - get_ep_group, - get_pcp_group, -) -from vllm.model_executor.layers.fused_moe.runner.moe_runner_base import MoERunnerBase - - -class DefaultMoERunner(MoERunnerBase): - """ - Standard MoE runner implementation for executing Mixture of Experts layers. - - This is the primary concrete implementation of MoE execution logic, providing - comprehensive support for standard MoE operations. It handles: - - Expert routing and token dispatching using various routing strategies - - Shared experts computation with optional parallel execution using CUDA streams - - Tensor model parallel and expert parallel operations - - Multiple quantization methods and optimized kernel selection - - Both monolithic and decomposed expert execution paths - - Integration with various parallel execution modes (TP, EP, DP) - - The runner orchestrates the complete MoE forward pass including routing tokens - to experts, executing expert computations in parallel, and combining results. - It supports advanced features like overlapped execution of shared experts, - optimized kernels for different parallel configurations, and seamless - integration with vLLM's distributed execution framework. - - This implementation is suitable for most standard MoE use cases. For specialized - scenarios like large batch chunking, alternative runners like ChunkingMoERunner - may be more appropriate. - - Eventually, this class may be split into more specialized implementations - for different configurations (e.g., with/without shared experts, gates, etc.). - """ - - @property - def do_naive_dispatch_combine(self) -> bool: - return ( - self.moe_config.dp_size > 1 and not self.quant_method.supports_internal_mk - ) - - def _maybe_dispatch( - self, - layer: torch.nn.Module, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - # For naive dispatch/combine Dp/Ep, dispatch the hidden states and - # router logits to all experts. - # NOTE: this will be removed once all kernels are migrated into the - # MoEKernel framework. - if self.do_naive_dispatch_combine: - res = get_ep_group().dispatch_router_logits( - hidden_states, - router_logits, - self.moe_config.is_sequence_parallel, - ) - assert len(res) == 2 - hidden_states, router_logits = res - - # NOTE: Similar with DP, PCP also needs dispatch and combine. For - # simplicity, AgRsAll2All was added separately for PCP here. Maybe - # we should modify All2AllManager abstraction to better support PCP. - if self.moe_config.pcp_size > 1: - hidden_states = get_pcp_group().all_gather( - hidden_states, - dim=0, - ) - router_logits = get_pcp_group().all_gather( - router_logits, - dim=0, - ) - - return hidden_states, router_logits - - def _maybe_combine( - self, - shared_output: torch.Tensor | None, - hidden_states: torch.Tensor, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]: - if self.do_naive_dispatch_combine: - hidden_states = get_ep_group().combine( - hidden_states, self.moe_config.is_sequence_parallel - ) - - if self.moe_config.pcp_size > 1: - hidden_states = get_pcp_group().reduce_scatter( - hidden_states, - dim=0, - ) - - if self.shared_experts is not None: - assert shared_output is not None - return shared_output, hidden_states - else: - return hidden_states - - def _forward_impl( - self, - layer: torch.nn.Module, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - # TODO(bnell): parts of the dispatch/combine steps will go away once - # #32567 lands and the remaining kernels are made MKs. The PCP - # code will probably remain - hidden_states, router_logits = self._maybe_dispatch( - layer, - hidden_states, - router_logits, - ) - - shared_output, hidden_states = self._apply_quant_method( - layer=layer, - hidden_states=hidden_states, - router_logits=router_logits, - shared_experts_input=shared_experts_input, - ) - - return self._maybe_combine( - shared_output, - hidden_states, - ) diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 199ceab0659..00be12780a1 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -1,44 +1,705 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from abc import ABC, abstractmethod +from collections.abc import Callable +from contextlib import nullcontext +from typing import TYPE_CHECKING import torch +import torch.nn.functional as F +from vllm.distributed import ( + get_ep_group, + get_pcp_group, + tensor_model_parallel_all_reduce, +) +from vllm.forward_context import ( + ForwardContext, + get_forward_context, + is_forward_context_available, +) +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, +) from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( FusedMoEMethodBase, ) +from vllm.model_executor.layers.fused_moe.router.fused_moe_router import ( + FusedMoERouter, +) +from vllm.model_executor.layers.fused_moe.router.zero_expert_router import ( + ZeroExpertRouter, +) +from vllm.model_executor.layers.fused_moe.runner.moe_runner_interface import ( + MoERunnerInterface, +) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, + SharedExpertsOrder, +) +from vllm.platforms import current_platform +from vllm.utils.torch_utils import ( + _USE_LAYERNAME, + LayerName, + direct_register_custom_op, ) -class MoERunner(ABC): - """ - Abstract base class for Mixture of Experts (MoE) runners. +def get_layer_from_name(layer_name: str) -> torch.nn.Module: + forward_context: ForwardContext = get_forward_context() + if not _USE_LAYERNAME and layer_name == "from_forward_context": + all_moe_layers = forward_context.all_moe_layers + assert all_moe_layers is not None + moe_layer_index = forward_context.moe_layer_index + if moe_layer_index >= len(all_moe_layers): + raise AssertionError( + "We expected the number of MOE layers in `all_moe_layers` " + "to be equal to the number of " + "{vllm.moe_forward, vllm.moe_forward_shared} calls." + ) + layer_name = all_moe_layers[moe_layer_index] + forward_context.moe_layer_index += 1 + return forward_context.no_compile_layers[layer_name] - This class defines the interface that all MoE runner implementations must follow. - MoE runners are responsible for executing the forward pass of MoE layers, handling - expert routing, and managing tensor parallel operations. + +# On torch >= 2.11, layer_name is a hoisted LayerName opaque object; +# on older versions it remains a plain str. +if TYPE_CHECKING: + from typing import TypeAlias + + _layer_name_type: TypeAlias = str | LayerName +else: + _layer_name_type = LayerName if _USE_LAYERNAME else str + + +@torch.compiler.assume_constant_result +def _resolve_layer_name(layer_name: str | LayerName) -> str: + from torch._library.fake_class_registry import FakeScriptObject + + if isinstance(layer_name, LayerName): + return layer_name.value + elif isinstance(layer_name, FakeScriptObject): + return layer_name.real_obj.value + return layer_name + + +# Note: _moe_forward and _moe_forward_shared should not contain any +# implementation details, They should merely pass along control to +# the runner's '_forward_impl' method. +# These functions should never be called directly since they do not +# include all the functionality of the MoE layer. +def _moe_forward( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + shared_experts_input: torch.Tensor | None, + layer_name: _layer_name_type, +) -> torch.Tensor: + layer = get_layer_from_name(_resolve_layer_name(layer_name)) + return layer.runner._forward_impl( + layer, + hidden_states, + router_logits, + shared_experts_input, + ) + + +def _moe_forward_fake( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + shared_experts_input: torch.Tensor | None, + layer_name: _layer_name_type, +) -> torch.Tensor: + return torch.empty_like(hidden_states) + + +def _moe_forward_shared( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + shared_experts_input: torch.Tensor | None, + layer_name: _layer_name_type, +) -> tuple[torch.Tensor, torch.Tensor]: + layer = get_layer_from_name(_resolve_layer_name(layer_name)) + return layer.runner._forward_impl( + layer, + hidden_states, + router_logits, + shared_experts_input, + ) + + +def _moe_forward_shared_fake( + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + shared_experts_input: torch.Tensor | None, + layer_name: _layer_name_type, +) -> tuple[torch.Tensor, torch.Tensor]: + # Output shapes: + # - fused_out: same as hidden_states (routed experts use transformed size) + # - shared_out: same as shared_experts_input if provided, else same as + # hidden_states + # (For latent MoE: shared experts use original hidden_size, not latent size) + fused_out = torch.empty_like(hidden_states) + if shared_experts_input is not None: + shared_out = torch.empty_like(shared_experts_input) + else: + shared_out = torch.empty_like(hidden_states) + return shared_out, fused_out + + +direct_register_custom_op( + op_name="moe_forward", + op_func=_moe_forward, + mutates_args=["hidden_states"], + fake_impl=_moe_forward_fake, + tags=(torch.Tag.needs_fixed_stride_order,), +) + + +direct_register_custom_op( + op_name="moe_forward_shared", + op_func=_moe_forward_shared, + fake_impl=_moe_forward_shared_fake, + tags=(torch.Tag.needs_fixed_stride_order,), +) + + +def _unpack( + result: torch.Tensor | tuple[torch.Tensor, torch.Tensor], +) -> tuple[torch.Tensor | None, torch.Tensor]: + if isinstance(result, tuple): + return result + else: + return (None, result) + + +class MoERunner(MoERunnerInterface): + """ + Standard MoE runner implementation for executing Mixture of Experts layers. + + This is the primary concrete implementation of MoE execution logic, providing + comprehensive support for standard MoE operations. It handles: + - Expert routing and token dispatching using various routing strategies + - Shared experts computation with optional parallel execution using CUDA streams + - Tensor model parallel and expert parallel operations + - Multiple quantization methods and optimized kernel selection + - Both monolithic and decomposed expert execution paths + - Integration with various parallel execution modes (TP, EP, DP) + + The runner orchestrates the complete MoE forward pass including routing tokens + to experts, executing expert computations in parallel, and combining results. + It supports advanced features like overlapped execution of shared experts, + optimized kernels for different parallel configurations, and seamless + integration with vLLM's distributed execution framework. + + Eventually, this class may be split into more specialized implementations + for different configurations (e.g., with/without shared experts, gates, etc.). """ - @abstractmethod + def __init__( + self, + layer_name: str, + moe_config: FusedMoEConfig, + router: FusedMoERouter, + routed_input_transform: torch.nn.Module | None, + gate: torch.nn.Module | None, + shared_experts: torch.nn.Module | None, + quant_method: FusedMoEMethodBase, + enable_dbo: bool, + routed_output_transform: torch.nn.Module | None = None, + routed_scaling_factor: float = 1.0, + ): + super().__init__() + self.moe_config = moe_config + self.router = router + self.routed_input_transform = routed_input_transform + self.routed_output_transform = routed_output_transform + self.routed_scaling_factor = routed_scaling_factor + self.gate = gate + self.quant_method = quant_method + self.enable_dbo = enable_dbo + + self._shared_experts: SharedExperts | None = None + if shared_experts is not None: + self._shared_experts = SharedExperts( + shared_experts, + moe_config=moe_config, + # Note: For now we must pass quant_method along to SharedExperts so it + # can property determine where the shared experts are supposed to be + # called, i.e. by a MK or by the MoERunner. + # Once the MK can be created upfront, we can just pass in the proper + # flags derived from the quant_method's MK. + quant_method=quant_method, + enable_dbo=enable_dbo, + ) + + # Needed for string -> FusedMoE layer lookup in custom ops. + self.layer_name = layer_name + + self._forward_entry = self._select_forward() + + def _select_forward(self) -> Callable: + if current_platform.is_tpu() or current_platform.is_cpu(): + # TODO: Once the OOM issue for the TPU backend is resolved, we + # will switch to using the moe_forward custom op. + # Note: CPU doesn't require wrapped _forward_impl. + return _moe_forward if self._shared_experts is None else _moe_forward_shared + + return ( + torch.ops.vllm.moe_forward + if self._shared_experts is None + else torch.ops.vllm.moe_forward_shared + ) + + @property + def shared_experts(self) -> SharedExperts | None: + return self._shared_experts + + # TODO(bnell): temporary hack, do not call this method. + def _replace_quant_method(self, quant_method: FusedMoEMethodBase): + if self._shared_experts is not None: + self._shared_experts._quant_method = quant_method + self.quant_method = quant_method + + def is_internal_router(self) -> bool: + return self.gate is not None + + def apply_routed_input_transform( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """Apply transform for routed experts (e.g., latent projection). + + This is called by FusedMoE.forward_native. The original hidden_states + is saved separately so shared experts get [S, hidden_size] while + routed experts get the transformed [S, moe_latent_size]. + + Returns (possibly transformed) hidden states and the input for shared + experts (or None if there are no shared experts). + """ + if self.routed_input_transform is not None: + result = self.routed_input_transform(hidden_states) + # ReplicatedLinear returns (output, extra_bias) tuple. + # We only need the output tensor; extra_bias is not used here. + if isinstance(result, tuple): + return result[0], hidden_states + return result, hidden_states + + return ( + hidden_states, + hidden_states if self._shared_experts is not None else None, + ) + + def apply_routed_output_transform( + self, + fused_output: torch.Tensor, + ) -> torch.Tensor: + """Apply transform to routed expert output (e.g., latent to full dim). + + Used by latent MoE models (e.g., NemotronH) where routed experts + operate in a compressed latent space and need projection back to + the full hidden dimension before combining with shared expert output. + """ + if self.routed_output_transform is not None: + r = self.routed_output_transform(fused_output) + fused_output = r[0] if isinstance(r, tuple) else r + return fused_output + + def _maybe_apply_routed_scale_to_output( + self, + shared_output: torch.Tensor | None, + fused_output: torch.Tensor, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + """Apply routed_scaling_factor to the output with FP16 overflow + protection. + + Scale the fused expert output by routed_scaling_factor. For FP16, + avoid overflow by dividing shared_output by the scale instead + (the decoder layer compensates with matching divisions). + """ + if self.routed_scaling_factor != 1.0: + if fused_output.dtype != torch.float16 or shared_output is None: + fused_output *= self.routed_scaling_factor + elif shared_output is not None: + shared_output *= 1.0 / self.routed_scaling_factor + return shared_output, fused_output + + @property + def _fused_output_is_reduced(self) -> bool: + return ( + self.quant_method.moe_kernel is not None + and self.quant_method.moe_kernel.output_is_reduced() + ) + + def _maybe_reduce_shared_expert_output( + self, + shared_output: torch.Tensor | None, + ) -> torch.Tensor | None: + """All-reduce shared expert output when the combine kernel already + reduced fused output. + + This is the "early" all-reduce path. When the combine kernel produces + already-reduced fused output, shared output must be reduced separately + to match. + """ + if shared_output is not None and self._fused_output_is_reduced: + shared_output = tensor_model_parallel_all_reduce(shared_output) + return shared_output + + def _maybe_reduce_final_output( + self, + states: torch.Tensor, + trunc_size: int, + ) -> torch.Tensor: + """Truncate padded dimensions and all-reduce the combined output. + + This is the "late" all-reduce path. When neither fused nor shared + output was individually reduced, the combined sum is all-reduced + here. Skipped when sequence-parallel is active (SP handles its + own reduction) or when the early path already reduced both outputs. + """ + # We don't need to reduce the final output if: + # - We are not running with TP or DP + # - The MK already reduced the fused output itself. + if ( + not self.moe_config.is_sequence_parallel + and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1) + and not self._fused_output_is_reduced + ): + states = tensor_model_parallel_all_reduce(states) + + return states[..., :trunc_size] + + def _encode_layer_name(self) -> str | LayerName: + if _USE_LAYERNAME: + return LayerName(self.layer_name) + # Can be unavailable or None in unittests + if ( + is_forward_context_available() + and get_forward_context().all_moe_layers is not None + ): + return "from_forward_context" + return self.layer_name + + def _maybe_pad_hidden_states( + self, + shared_experts_input: torch.Tensor | None, + hidden_states: torch.Tensor, + ) -> tuple[torch.Tensor, int]: + """Pad hidden_states to moe_config.hidden_dim and compute the + original dimension for later truncation. + + For latent MoE, the routed hidden_states may be smaller than + hidden_dim. Padding ensures uniform tensor sizes through the + fused MoE kernel. The returned trunc_size is used by + _maybe_reduce_final_output to strip the padding from the result. + """ + shared_experts_hidden_dim = ( + shared_experts_input.shape[-1] if shared_experts_input is not None else 0 + ) + transformed_hidden_dim = hidden_states.shape[-1] + if ( + not self.quant_method.skip_forward_padding + and self.moe_config.hidden_dim != transformed_hidden_dim + ): + hidden_states = F.pad( + hidden_states, + (0, self.moe_config.hidden_dim - transformed_hidden_dim), + mode="constant", + value=0.0, + ) + + if self.routed_output_transform is not None and shared_experts_hidden_dim > 0: + orig_hidden_dims = shared_experts_hidden_dim + else: + orig_hidden_dims = transformed_hidden_dim + + return hidden_states, orig_hidden_dims + + def _maybe_apply_shared_experts( + self, + shared_experts_input: torch.Tensor | None, + order: SharedExpertsOrder, + ): + if self._shared_experts is not None: + assert shared_experts_input is not None + self._shared_experts.apply(shared_experts_input, order) + + def _apply_quant_method( + self, + layer: torch.nn.Module, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + shared_experts_input: torch.Tensor | None, + ) -> tuple[torch.Tensor | None, torch.Tensor]: + """Run expert routing and the fused MoE kernel via the quant method. + + Orchestrates shared expert execution (before/after), expert selection + via the router, and the actual fused MoE computation. Returns + (shared_expert_output, fused_expert_output). + """ + self._maybe_apply_shared_experts( + shared_experts_input, SharedExpertsOrder.NO_OVERLAP + ) + + if self.quant_method.is_monolithic: + fused_out = self.quant_method.apply_monolithic( + layer=layer, + x=hidden_states, + router_logits=router_logits, + ) + else: + topk_weights, topk_ids = self.router.select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + ) + + # Passing shared_experts_input in case SharedExpertsOrder is + # MK_INTERNAL_OVERLAPPED. + fused_out = self.quant_method.apply( + layer=layer, + x=hidden_states, + topk_weights=topk_weights, + topk_ids=topk_ids, + shared_experts_input=shared_experts_input, + ) + + self._maybe_apply_shared_experts( + shared_experts_input, + SharedExpertsOrder.MULTI_STREAM_OVERLAPPED, + ) + + return ( + self._shared_experts.output if self._shared_experts is not None else None, + fused_out, + ) + + def _sequence_parallel_context(self): + """Return a context manager for sequence-parallel token + redistribution. + + When sequence parallelism is active, returns a context that handles + local size tracking for proper token scatter/gather. Otherwise + returns a no-op context. + """ + ctx = get_forward_context() + return ( + ctx.dp_metadata.sp_local_sizes(self.moe_config.sp_size) + if ctx.dp_metadata + else nullcontext() + ) + + def _maybe_sync_shared_experts_stream( + self, + shared_experts_input: torch.Tensor | None, + ): + # If router/gate provided, then apply it here. + # (Note: This code runs only when "overlapped mode" is on to allow + # parallel execution of shared experts with the FusedMoE via + # separate cuda stream) + if self._shared_experts is not None: + assert shared_experts_input is not None + self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input) + + def _maybe_add_zero_expert_output( + self, + result: torch.Tensor, + ) -> torch.Tensor: + """Add the zero expert's contribution to the final result. + + When a ZeroExpertRouter is used, it computes a bias-like output + from the "zero expert" that is added to the combined routed+shared + expert output. + """ + if isinstance(self.router, ZeroExpertRouter): + zero_expert_output = self.router.zero_expert_output + assert zero_expert_output is not None + result = result + zero_expert_output + return result + def forward( self, hidden_states: torch.Tensor, router_logits: torch.Tensor, ) -> torch.Tensor: - raise NotImplementedError + """Invoke the fused moe layer. - @abstractmethod - def is_internal_router(self) -> bool: - raise NotImplementedError + Input: + - hidden_states + - router_logits + + Output: + - The new hidden_states. + + Calling sequence + - forward + - self._forward_entry (_moe_forward or _moe_forward_shared custom op) + - _forward_impl + + Note: The existence of _moe_forward and _moe_forward_shared custom ops are due + to the following reason: + 1. pytorch cannot handle union types in custom op signatures so + _moe_forward and _moe_forward_shared must be split. + """ + + # Apply transform for routed experts (e.g., latent projection + # for latent MoE) + hidden_states, shared_experts_input = self.apply_routed_input_transform( + hidden_states + ) + + hidden_states, og_hidden_dim = self._maybe_pad_hidden_states( + shared_experts_input, + hidden_states, + ) + + result = self._forward_entry( + hidden_states, + router_logits, + shared_experts_input, + self._encode_layer_name(), + ) + + # + # Note: there are two all-reduce points below. They are mutually + # exclusive, controlled by _fused_output_is_reduced + # - When True: the combine kernel already reduced fused_output, + # so we reduce shared_output here to match, then skip the + # all-reduce in _maybe_reduce_final_output. + # - When False: neither output is reduced yet, so we combine + # them first and all-reduce the sum in _maybe_reduce_final_output. + + # Extract outputs from result + shared_output, fused_output = _unpack(result) + + # If combine kernel already reduced fused, reduce shared to match. + # See note above re: the two all-reduce points. + shared_output = self._maybe_reduce_shared_expert_output(shared_output) + + shared_output, fused_output = self._maybe_apply_routed_scale_to_output( + shared_output, fused_output + ) + + # Apply output transform (e.g. latent -> full dim) + fused_output = self.apply_routed_output_transform(fused_output) + + if shared_output is not None: + result = shared_output + fused_output + else: + result = fused_output + + result = self._maybe_reduce_final_output(result, og_hidden_dim) + + return self._maybe_add_zero_expert_output(result) @property - @abstractmethod - def shared_experts(self) -> SharedExperts | None: - raise NotImplementedError + def do_naive_dispatch_combine(self) -> bool: + return ( + self.moe_config.dp_size > 1 and not self.quant_method.supports_internal_mk + ) - # TODO(bnell): temporary hack, do not call this method. - @abstractmethod - def _replace_quant_method(self, quant_method: FusedMoEMethodBase): - raise NotImplementedError + def _maybe_dispatch( + self, + layer: torch.nn.Module, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + # For naive dispatch/combine Dp/Ep, dispatch the hidden states and + # router logits to all experts. + # NOTE: this will be removed once all kernels are migrated into the + # MoEKernel framework. + if self.do_naive_dispatch_combine: + result = get_ep_group().dispatch_router_logits( + hidden_states, + router_logits, + self.moe_config.is_sequence_parallel, + ) + assert len(result) == 2 + hidden_states, router_logits = result + + # NOTE: Similar with DP, PCP also needs dispatch and combine. For + # simplicity, AgRsAll2All was added separately for PCP here. Maybe + # we should modify All2AllManager abstraction to better support PCP. + if self.moe_config.pcp_size > 1: + hidden_states = get_pcp_group().all_gather( + hidden_states, + dim=0, + ) + router_logits = get_pcp_group().all_gather( + router_logits, + dim=0, + ) + + return hidden_states, router_logits + + def _maybe_combine( + self, + shared_output: torch.Tensor | None, + hidden_states: torch.Tensor, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]: + if self.do_naive_dispatch_combine: + hidden_states = get_ep_group().combine( + hidden_states, self.moe_config.is_sequence_parallel + ) + + if self.moe_config.pcp_size > 1: + hidden_states = get_pcp_group().reduce_scatter( + hidden_states, + dim=0, + ) + + if self.shared_experts is not None: + assert shared_output is not None + return shared_output, hidden_states + else: + return hidden_states + + def _forward_impl( + self, + layer: torch.nn.Module, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + shared_experts_input: torch.Tensor | None, + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + """Entry point called by the custom op to run the MoE computation. + + Handles pre-dispatch setup (gate application, external shared expert + triggering, quant config init) then performs the following steps + within the sequence-parallel context. + + - Performs expert routing + - fused MoE kernel execution + - shared expert computation. + + Returns a single tensor of combined fused and shared output (if present). + """ + # TODO(bnell): this can be removed after MK migration is complete. + layer.ensure_moe_quant_config_init() + + # Sync aux and main stream for shared expert multi-stream overlap. + self._maybe_sync_shared_experts_stream(shared_experts_input) + + # If the Runner holds the gate, apply it after the stream sync, + # so it can run overlapped with the + # NOTE: in future PR, MoE runner will always hold the gate. + if self.gate is not None: + router_logits, _ = self.gate(hidden_states) + + with self._sequence_parallel_context(): + # TODO(bnell): parts of the dispatch/combine steps will go away once + # #32567 lands and the remaining kernels are made MKs. The PCP + # code will probably remain + hidden_states, router_logits = self._maybe_dispatch( + layer, + hidden_states, + router_logits, + ) + + shared_output, hidden_states = self._apply_quant_method( + layer=layer, + hidden_states=hidden_states, + router_logits=router_logits, + shared_experts_input=shared_experts_input, + ) + + return self._maybe_combine( + shared_output, + hidden_states, + ) diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner_base.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner_base.py deleted file mode 100644 index 136e1b1f5b2..00000000000 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner_base.py +++ /dev/null @@ -1,639 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from abc import abstractmethod -from collections.abc import Callable -from contextlib import nullcontext -from typing import TYPE_CHECKING - -import torch -import torch.nn.functional as F - -from vllm.distributed import ( - tensor_model_parallel_all_reduce, -) -from vllm.forward_context import ( - ForwardContext, - get_forward_context, - is_forward_context_available, -) -from vllm.model_executor.layers.fused_moe.config import ( - FusedMoEConfig, -) -from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( - FusedMoEMethodBase, -) -from vllm.model_executor.layers.fused_moe.router.fused_moe_router import ( - FusedMoERouter, -) -from vllm.model_executor.layers.fused_moe.router.zero_expert_router import ( - ZeroExpertRouter, -) -from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner -from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( - SharedExperts, - SharedExpertsOrder, -) -from vllm.platforms import current_platform -from vllm.utils.torch_utils import ( - _USE_LAYERNAME, - LayerName, - direct_register_custom_op, -) - - -def get_layer_from_name(layer_name: str) -> torch.nn.Module: - forward_context: ForwardContext = get_forward_context() - if not _USE_LAYERNAME and layer_name == "from_forward_context": - all_moe_layers = forward_context.all_moe_layers - assert all_moe_layers is not None - moe_layer_index = forward_context.moe_layer_index - if moe_layer_index >= len(all_moe_layers): - raise AssertionError( - "We expected the number of MOE layers in `all_moe_layers` " - "to be equal to the number of " - "{vllm.moe_forward, vllm.moe_forward_shared} calls." - ) - layer_name = all_moe_layers[moe_layer_index] - forward_context.moe_layer_index += 1 - return forward_context.no_compile_layers[layer_name] - - -# On torch >= 2.11, layer_name is a hoisted LayerName opaque object; -# on older versions it remains a plain str. -if TYPE_CHECKING: - from typing import TypeAlias - - _layer_name_type: TypeAlias = str | LayerName -else: - _layer_name_type = LayerName if _USE_LAYERNAME else str - - -@torch.compiler.assume_constant_result -def _resolve_layer_name(layer_name: str | LayerName) -> str: - from torch._library.fake_class_registry import FakeScriptObject - - if isinstance(layer_name, LayerName): - return layer_name.value - elif isinstance(layer_name, FakeScriptObject): - return layer_name.real_obj.value - return layer_name - - -# Note: _moe_forward and _moe_forward_shared should not contain any -# implementation details, They should merely pass along control to -# the runner's '_forward_dispatch' method. -# These functions should never be called directly since they do not -# include all the functionality of the MoE layer. -def _moe_forward( - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - layer_name: _layer_name_type, -) -> torch.Tensor: - layer = get_layer_from_name(_resolve_layer_name(layer_name)) - return layer.runner._forward_dispatch( - layer, - hidden_states, - router_logits, - shared_experts_input, - ) - - -def _moe_forward_fake( - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - layer_name: _layer_name_type, -) -> torch.Tensor: - return torch.empty_like(hidden_states) - - -def _moe_forward_shared( - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - layer_name: _layer_name_type, -) -> tuple[torch.Tensor, torch.Tensor]: - layer = get_layer_from_name(_resolve_layer_name(layer_name)) - return layer.runner._forward_dispatch( - layer, - hidden_states, - router_logits, - shared_experts_input, - ) - - -def _moe_forward_shared_fake( - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - layer_name: _layer_name_type, -) -> tuple[torch.Tensor, torch.Tensor]: - # Output shapes: - # - fused_out: same as hidden_states (routed experts use transformed size) - # - shared_out: same as shared_experts_input if provided, else same as - # hidden_states - # (For latent MoE: shared experts use original hidden_size, not latent size) - fused_out = torch.empty_like(hidden_states) - if shared_experts_input is not None: - shared_out = torch.empty_like(shared_experts_input) - else: - shared_out = torch.empty_like(hidden_states) - return shared_out, fused_out - - -direct_register_custom_op( - op_name="moe_forward", - op_func=_moe_forward, - mutates_args=["hidden_states"], - fake_impl=_moe_forward_fake, - tags=(torch.Tag.needs_fixed_stride_order,), -) - - -direct_register_custom_op( - op_name="moe_forward_shared", - op_func=_moe_forward_shared, - fake_impl=_moe_forward_shared_fake, - tags=(torch.Tag.needs_fixed_stride_order,), -) - - -def _unpack( - result: torch.Tensor | tuple[torch.Tensor, torch.Tensor], -) -> tuple[torch.Tensor | None, torch.Tensor]: - if isinstance(result, tuple): - return result - else: - return (None, result) - - -class MoERunnerBase(MoERunner): - """ - Abstract base class providing common functionality for MoE runner implementations. - - This class serves as the foundation for concrete MoE runner implementations by - providing shared state management and common utilities. It handles: - - Common initialization and configuration management - - Shared expert output reduction logic for tensor parallel scenarios - - Base methods for tensor model parallel reductions - - Common properties and utility functions used across different runner types - - Concrete subclasses must implement the abstract methods to define their specific - execution strategies, such as standard execution, chunked processing, or other - specialized approaches. The base class provides the infrastructure while - allowing flexibility in the actual MoE computation implementation. - - Key abstract methods that subclasses must implement: - - _forward_impl: The core MoE computation logic specific to each runner type - """ - - def __init__( - self, - layer_name: str, - moe_config: FusedMoEConfig, - router: FusedMoERouter, - routed_input_transform: torch.nn.Module | None, - gate: torch.nn.Module | None, - shared_experts: torch.nn.Module | None, - quant_method: FusedMoEMethodBase, - enable_dbo: bool, - routed_output_transform: torch.nn.Module | None = None, - routed_scaling_factor: float = 1.0, - ): - super().__init__() - self.moe_config = moe_config - self.router = router - self.routed_input_transform = routed_input_transform - self.routed_output_transform = routed_output_transform - self.routed_scaling_factor = routed_scaling_factor - self.gate = gate - self.quant_method = quant_method - self.enable_dbo = enable_dbo - self._fused_output_is_reduced = ( - self.quant_method.moe_kernel is not None - and self.quant_method.moe_kernel.output_is_reduced() - ) - - self._shared_experts: SharedExperts | None = None - if shared_experts is not None: - self._shared_experts = SharedExperts( - shared_experts, - moe_config=moe_config, - # Note: For now we must pass quant_method along to SharedExperts so it - # can property determine where the shared experts are supposed to be - # called, i.e. by a MK or by the MoERunner. - # Once the MK can be created upfront, we can just pass in the proper - # flags derived from the quant_method's MK. - quant_method=quant_method, - enable_dbo=enable_dbo, - ) - - # Needed for string -> FusedMoE layer lookup in custom ops. - self.layer_name = layer_name - - self._forward_entry = self._select_forward() - - def _select_forward(self) -> Callable: - if current_platform.is_tpu() or current_platform.is_cpu(): - # TODO: Once the OOM issue for the TPU backend is resolved, we - # will switch to using the moe_forward custom op. - # Note: CPU doesn't require wrapped _forward_impl. - return _moe_forward if self._shared_experts is None else _moe_forward_shared - - return ( - torch.ops.vllm.moe_forward - if self._shared_experts is None - else torch.ops.vllm.moe_forward_shared - ) - - @property - def shared_experts(self) -> SharedExperts | None: - return self._shared_experts - - # TODO(bnell): temporary hack, do not call this method. - def _replace_quant_method(self, quant_method: FusedMoEMethodBase): - if self._shared_experts is not None: - self._shared_experts._quant_method = quant_method - self.quant_method = quant_method - - def is_internal_router(self) -> bool: - return self.gate is not None - - def apply_routed_input_transform( - self, hidden_states: torch.Tensor - ) -> tuple[torch.Tensor, torch.Tensor | None]: - """Apply transform for routed experts (e.g., latent projection). - - This is called by FusedMoE.forward_native. The original hidden_states - is saved separately so shared experts get [S, hidden_size] while - routed experts get the transformed [S, moe_latent_size]. - - Returns (possibly transformed) hidden states and the input for shared - experts (or None if there are no shared experts). - """ - if self.routed_input_transform is not None: - result = self.routed_input_transform(hidden_states) - # ReplicatedLinear returns (output, extra_bias) tuple. - # We only need the output tensor; extra_bias is not used here. - if isinstance(result, tuple): - return result[0], hidden_states - return result, hidden_states - - return ( - hidden_states, - hidden_states if self._shared_experts is not None else None, - ) - - def apply_routed_output_transform( - self, - fused_output: torch.Tensor, - ) -> torch.Tensor: - """Apply transform to routed expert output (e.g., latent to full dim). - - Used by latent MoE models (e.g., NemotronH) where routed experts - operate in a compressed latent space and need projection back to - the full hidden dimension before combining with shared expert output. - """ - if self.routed_output_transform is not None: - r = self.routed_output_transform(fused_output) - fused_output = r[0] if isinstance(r, tuple) else r - return fused_output - - def _maybe_apply_routed_scale_to_output( - self, - shared_output: torch.Tensor | None, - fused_output: torch.Tensor, - ) -> tuple[torch.Tensor | None, torch.Tensor]: - """Apply routed_scaling_factor to the output with FP16 overflow - protection. - - Scale the fused expert output by routed_scaling_factor. For FP16, - avoid overflow by dividing shared_output by the scale instead - (the decoder layer compensates with matching divisions). - """ - if self.routed_scaling_factor != 1.0: - if fused_output.dtype != torch.float16: - fused_output *= self.routed_scaling_factor - elif shared_output is not None: - shared_output *= 1.0 / self.routed_scaling_factor - return shared_output, fused_output - - def _maybe_reduce_shared_expert_output( - self, - shared_output: torch.Tensor | None, - ) -> torch.Tensor | None: - """All-reduce shared expert output when the combine kernel already - reduced fused output. - - This is the "early" all-reduce path. When the combine kernel produces - already-reduced fused output, shared output must be reduced separately - to match. - """ - if self._fused_output_is_reduced: - assert shared_output is not None - shared_output = tensor_model_parallel_all_reduce(shared_output) - return shared_output - - def _maybe_reduce_final_output( - self, - states: torch.Tensor, - trunc_size: int, - ) -> torch.Tensor: - """Truncate padded dimensions and all-reduce the combined output. - - This is the "late" all-reduce path. When neither fused nor shared - output was individually reduced, the combined sum is all-reduced - here. Skipped when sequence-parallel is active (SP handles its - own reduction) or when the early path already reduced both outputs. - """ - # We don't need to reduce the final output if: - # - We are not running with TP or DP - # - The MK already reduced the fused output itself. - if ( - not self.moe_config.is_sequence_parallel - and (self.moe_config.tp_size > 1 or self.moe_config.ep_size > 1) - and not self._fused_output_is_reduced - ): - states = tensor_model_parallel_all_reduce(states) - - return states[..., :trunc_size] - - def _encode_layer_name(self) -> str | LayerName: - if _USE_LAYERNAME: - return LayerName(self.layer_name) - # Can be unavailable or None in unittests - if ( - is_forward_context_available() - and get_forward_context().all_moe_layers is not None - ): - return "from_forward_context" - return self.layer_name - - def _maybe_pad_hidden_states( - self, - shared_experts_input: torch.Tensor | None, - hidden_states: torch.Tensor, - ) -> tuple[torch.Tensor, int]: - """Pad hidden_states to moe_config.hidden_dim and compute the - original dimension for later truncation. - - For latent MoE, the routed hidden_states may be smaller than - hidden_dim. Padding ensures uniform tensor sizes through the - fused MoE kernel. The returned trunc_size is used by - _maybe_reduce_final_output to strip the padding from the result. - """ - shared_experts_hidden_dim = ( - shared_experts_input.shape[-1] if shared_experts_input is not None else 0 - ) - transformed_hidden_dim = hidden_states.shape[-1] - if ( - not self.quant_method.skip_forward_padding - and self.moe_config.hidden_dim != transformed_hidden_dim - ): - hidden_states = F.pad( - hidden_states, - (0, self.moe_config.hidden_dim - transformed_hidden_dim), - mode="constant", - value=0.0, - ) - - if self.routed_output_transform is not None and shared_experts_hidden_dim > 0: - orig_hidden_dims = shared_experts_hidden_dim - else: - orig_hidden_dims = transformed_hidden_dim - - return hidden_states, orig_hidden_dims - - def _maybe_apply_shared_experts( - self, - shared_experts_input: torch.Tensor | None, - order: SharedExpertsOrder, - ): - if self._shared_experts is not None: - assert shared_experts_input is not None - self._shared_experts.apply(shared_experts_input, order) - - def _apply_quant_method( - self, - layer: torch.nn.Module, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - ) -> tuple[torch.Tensor | None, torch.Tensor]: - """Run expert routing and the fused MoE kernel via the quant method. - - Orchestrates shared expert execution (before/after), expert selection - via the router, and the actual fused MoE computation. Returns - (shared_expert_output, fused_expert_output). - """ - self._maybe_apply_shared_experts( - shared_experts_input, SharedExpertsOrder.NO_OVERLAP - ) - - if self.quant_method.is_monolithic: - fused_out = self.quant_method.apply_monolithic( - layer=layer, - x=hidden_states, - router_logits=router_logits, - ) - else: - topk_weights, topk_ids = self.router.select_experts( - hidden_states=hidden_states, - router_logits=router_logits, - ) - - # Passing shared_experts_input in case SharedExpertsOrder is - # MK_INTERNAL_OVERLAPPED. - fused_out = self.quant_method.apply( - layer=layer, - x=hidden_states, - topk_weights=topk_weights, - topk_ids=topk_ids, - shared_experts_input=shared_experts_input, - ) - - self._maybe_apply_shared_experts( - shared_experts_input, - SharedExpertsOrder.MULTI_STREAM_OVERLAPPED, - ) - - return ( - self._shared_experts.output if self._shared_experts is not None else None, - fused_out, - ) - - def _sequence_parallel_context(self): - """Return a context manager for sequence-parallel token - redistribution. - - When sequence parallelism is active, returns a context that handles - local size tracking for proper token scatter/gather. Otherwise - returns a no-op context. - """ - ctx = get_forward_context() - return ( - ctx.dp_metadata.sp_local_sizes(self.moe_config.sp_size) - if ctx.dp_metadata - else nullcontext() - ) - - def _maybe_sync_shared_experts_stream( - self, - shared_experts_input: torch.Tensor | None, - ): - # If router/gate provided, then apply it here. - # (Note: This code runs only when "overlapped mode" is on to allow - # parallel execution of shared experts with the FusedMoE via - # separate cuda stream) - if self._shared_experts is not None: - assert shared_experts_input is not None - self._shared_experts.maybe_sync_shared_experts_stream(shared_experts_input) - - def _maybe_add_zero_expert_output( - self, - result: torch.Tensor, - ) -> torch.Tensor: - """Add the zero expert's contribution to the final result. - - When a ZeroExpertRouter is used, it computes a bias-like output - from the "zero expert" that is added to the combined routed+shared - expert output. - """ - if isinstance(self.router, ZeroExpertRouter): - zero_expert_output = self.router.zero_expert_output - assert zero_expert_output is not None - result = result + zero_expert_output - return result - - def forward( - self, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - ) -> torch.Tensor: - """Invoke the fused moe layer. - - Input: - - hidden_states - - router_logits - - Output: - - The new hidden_states. - - Calling sequence - - forward - - self._forward_entry (_moe_forward or _moe_forward_shared custom op) - - _forward_dispatch - - _forward_impl - - Note: The existence of _moe_forward and _moe_forward_shared custom ops are due - to the following reasons: - 1. the chunking loop in ChunkingMoERunner._forward_impl cannot be compiled by - torch.compile - 2. pytorch cannot handle union types in custom op signatures so - _moe_forward and _moe_forward_shared must be split. - - If ChunkingMoERunner._forward_impl can be implemented via torch.scan we can - potentially get rid of _moe_forward and _moe_forward_shared and collapse the - whole sequence into the 'forward' method. - """ - - # Apply transform for routed experts (e.g., latent projection - # for latent MoE) - hidden_states, shared_experts_input = self.apply_routed_input_transform( - hidden_states - ) - - hidden_states, og_hidden_dim = self._maybe_pad_hidden_states( - shared_experts_input, - hidden_states, - ) - - result = self._forward_entry( - hidden_states, - router_logits, - shared_experts_input, - self._encode_layer_name(), - ) - - # - # Note: there are two all-reduce points below. They are mutually - # exclusive, controlled by _fused_output_is_reduced - # - When True: the combine kernel already reduced fused_output, - # so we reduce shared_output here to match, then skip the - # all-reduce in _maybe_reduce_final_output. - # - When False: neither output is reduced yet, so we combine - # them first and all-reduce the sum in _maybe_reduce_final_output. - - # Extract outputs from result - shared_output, fused_output = _unpack(result) - - # If combine kernel already reduced fused, reduce shared to match. - # See note above re: the two all-reduce points. - shared_output = self._maybe_reduce_shared_expert_output(shared_output) - - shared_output, fused_output = self._maybe_apply_routed_scale_to_output( - shared_output, fused_output - ) - - # Apply output transform (e.g. latent -> full dim) - fused_output = self.apply_routed_output_transform(fused_output) - - if shared_output is not None: - result = shared_output + fused_output - else: - result = fused_output - - result = self._maybe_reduce_final_output(result, og_hidden_dim) - - return self._maybe_add_zero_expert_output(result) - - def _forward_dispatch( - self, - layer: torch.nn.Module, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """Entry point called by the custom op to run the MoE computation. - - Handles pre-dispatch setup (gate application, external shared expert - triggering, quant config init) then delegates to _forward_impl within - the sequence-parallel context. - """ - # TODO(bnell): this can be removed after MK migration is complete. - layer.ensure_moe_quant_config_init() - - # Sync aux and main stream for shared expert multi-stream overlap. - self._maybe_sync_shared_experts_stream(shared_experts_input) - - # If the Runner holds the gate, apply it after the stream sync, - # so it can run overlapped with the - # NOTE: in future PR, MoE runner will always hold the gate. - if self.gate is not None: - router_logits, _ = self.gate(hidden_states) - - with self._sequence_parallel_context(): - return self._forward_impl( - layer, - hidden_states, - router_logits, - shared_experts_input, - ) - - @abstractmethod - def _forward_impl( - self, - layer: torch.nn.Module, - hidden_states: torch.Tensor, - router_logits: torch.Tensor, - shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - """Core MoE computation to be implemented by subclasses. - - Performs expert routing, fused MoE kernel execution, and shared - expert computation. Returns a single tensor (fused output only) - or a tuple of (shared_output, fused_output) when shared experts - are present. - """ - raise NotImplementedError diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner_factory.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner_factory.py deleted file mode 100644 index feb4614d837..00000000000 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner_factory.py +++ /dev/null @@ -1,47 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import torch - -from vllm.model_executor.layers.fused_moe.config import ( - FusedMoEConfig, -) -from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( - FusedMoEMethodBase, -) -from vllm.model_executor.layers.fused_moe.router.fused_moe_router import ( - FusedMoERouter, -) -from vllm.model_executor.layers.fused_moe.runner.default_moe_runner import ( - DefaultMoERunner, -) -from vllm.model_executor.layers.fused_moe.runner.moe_runner import MoERunner -from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( - SharedExperts, -) - - -def create_moe_runner( - layer_name: str, - moe_config: FusedMoEConfig, - router: FusedMoERouter, - routed_input_transform: torch.nn.Module | None, - gate: torch.nn.Module | None, - shared_experts: SharedExperts | None, - quant_method: FusedMoEMethodBase, - enable_dbo: bool, - routed_output_transform: torch.nn.Module | None = None, - routed_scaling_factor: float = 1.0, -) -> MoERunner: - return DefaultMoERunner( - layer_name, - moe_config, - router, - routed_input_transform, - gate, - shared_experts, - quant_method, - enable_dbo, - routed_output_transform=routed_output_transform, - routed_scaling_factor=routed_scaling_factor, - ) diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py new file mode 100644 index 00000000000..80bd83e3732 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner_interface.py @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from abc import ABC, abstractmethod + +import torch + +from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( + FusedMoEMethodBase, +) +from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( + SharedExperts, +) + + +class MoERunnerInterface(ABC): + """ + Abstract base class for Mixture of Experts (MoE) runners. + + This class defines the interface that all MoE runner implementations must follow. + MoE runners are responsible for executing the forward pass of MoE layers, handling + expert routing, and managing tensor parallel operations. + """ + + @abstractmethod + def forward( + self, + hidden_states: torch.Tensor, + router_logits: torch.Tensor, + ) -> torch.Tensor: + raise NotImplementedError + + @abstractmethod + def is_internal_router(self) -> bool: + raise NotImplementedError + + @property + @abstractmethod + def shared_experts(self) -> SharedExperts | None: + raise NotImplementedError + + # TODO(bnell): temporary hack, do not call this method. + @abstractmethod + def _replace_quant_method(self, quant_method: FusedMoEMethodBase): + raise NotImplementedError From 5f76b3fb3044785e628384f41fd3b32f1185b448 Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:53:30 -0400 Subject: [PATCH 041/153] [MoE] Convert CT W8A8 To Oracle Structure (#39187) Signed-off-by: Robert Shaw Co-authored-by: Claude --- .../layers/fused_moe/fused_batched_moe.py | 20 +- .../layers/fused_moe/fused_moe.py | 30 +-- .../layers/fused_moe/oracle/int8.py | 187 +++++++++++++++--- .../compressed_tensors_moe_w8a8_int8.py | 64 ++++-- .../layers/quantization/online/int8.py | 9 +- .../layers/quantization/utils/quant_utils.py | 3 + 6 files changed, 249 insertions(+), 64 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/fused_batched_moe.py index e2b5a8f6764..5554298bd09 100644 --- a/vllm/model_executor/layers/fused_moe/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_batched_moe.py @@ -928,16 +928,16 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular): p.is_cuda() and p.has_device_capability((8, 9)) ) - SUPPORTED_W_A_FP8 = [ - (kFp8Static128BlockSym, kFp8Dynamic128Sym), - (kFp8StaticChannelSym, kFp8DynamicTokenSym), - (kFp8StaticTensorSym, kFp8DynamicTokenSym), - (kFp8StaticTensorSym, kFp8StaticTensorSym), - (kFp8StaticTensorSym, kFp8DynamicTensorSym), - ] - return (weight_key, activation_key) == (None, None) or ( - device_supports_fp8 and (weight_key, activation_key) in SUPPORTED_W_A_FP8 - ) + supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)] + if device_supports_fp8: + supported += [ + (kFp8Static128BlockSym, kFp8Dynamic128Sym), + (kFp8StaticChannelSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8StaticTensorSym), + (kFp8StaticTensorSym, kFp8DynamicTensorSym), + ] + return (weight_key, activation_key) in supported @staticmethod def _supports_activation(activation: MoEActivation) -> bool: diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index 3b12f294939..a8c3b1d7713 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -46,6 +46,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt8DynamicTokenSym, + kInt8StaticChannelSym, ) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton @@ -1952,18 +1954,24 @@ class TritonExperts(mk.FusedMoEExpertsModular): weight_key: QuantKey | None, activation_key: QuantKey | None, ) -> bool: - if not current_platform.supports_fp8(): - return (weight_key, activation_key) == (None, None) + # INT8 requires at least 7.5 (Turing). + device_supports_int8 = ( + current_platform.is_cuda() + and current_platform.has_device_capability((7, 5)) + ) - SUPPORTED_W_A = [ - (None, None), - (kFp8Static128BlockSym, kFp8Dynamic128Sym), - (kFp8StaticChannelSym, kFp8DynamicTokenSym), - (kFp8StaticTensorSym, kFp8DynamicTokenSym), - (kFp8StaticTensorSym, kFp8StaticTensorSym), - (kFp8StaticTensorSym, kFp8DynamicTensorSym), - ] - return (weight_key, activation_key) in SUPPORTED_W_A + supported: list[tuple[QuantKey | None, QuantKey | None]] = [(None, None)] + if device_supports_int8: + supported.append((kInt8StaticChannelSym, kInt8DynamicTokenSym)) + if current_platform.supports_fp8(): + supported += [ + (kFp8Static128BlockSym, kFp8Dynamic128Sym), + (kFp8StaticChannelSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8DynamicTokenSym), + (kFp8StaticTensorSym, kFp8StaticTensorSym), + (kFp8StaticTensorSym, kFp8DynamicTensorSym), + ] + return (weight_key, activation_key) in supported @staticmethod def _supports_activation(activation: MoEActivation) -> bool: diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index 3ae9a491e9b..efa2792b420 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from enum import Enum + import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.config.kernel import MoEBackend from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe.all2all_utils import ( maybe_make_prepare_finalize, @@ -11,46 +14,165 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, + int8_w8a8_moe_quant_config, int8_w8a16_moe_quant_config, ) from vllm.model_executor.layers.fused_moe.runner.shared_experts import ( SharedExperts, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt8DynamicTokenSym, + kInt8StaticChannelSym, +) logger = init_logger(__name__) -def select_int8_moe_backend( - config: FusedMoEConfig, -) -> type[mk.FusedMoEExperts]: - from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts +class Int8MoeBackend(Enum): + TRITON = "TRITON" - supported, reason = TritonExperts.is_supported_config( - TritonExperts, - config, - None, - None, - mk.FusedMoEActivationFormat.Standard, - ) - if not supported: - raise ValueError( - f"INT8 Triton MoE backend does not support the " - f"deployment configuration: {reason}" + +def _get_priority_backends( + moe_config: FusedMoEConfig, +) -> list[Int8MoeBackend]: + """ + Get available backends in priority order based on platform and config. + """ + return [Int8MoeBackend.TRITON] + + +def backend_to_kernel_cls( + backend: Int8MoeBackend, +) -> list[type[mk.FusedMoEExperts]]: + if backend == Int8MoeBackend.TRITON: + from vllm.model_executor.layers.fused_moe.fused_moe import ( + TritonExperts, ) - logger.info_once("Using Triton INT8 MoE backend", scope="local") - return TritonExperts + return [TritonExperts] + + else: + raise ValueError(f"Unknown Int8 MoE backend: {backend.value}") + + +def map_int8_backend(runner_backend: MoEBackend) -> Int8MoeBackend: + """Map user's MoEBackend to Int8MoeBackend.""" + mapping = { + "triton": Int8MoeBackend.TRITON, + } + if backend := mapping.get(runner_backend): + return backend + raise ValueError( + f"moe_backend='{runner_backend}' is not supported for Int8 MoE. " + f"Expected one of {list(mapping.keys())}." + ) + + +def select_int8_moe_backend( + config: FusedMoEConfig, + weight_key: QuantKey | None = kInt8StaticChannelSym, + activation_key: QuantKey | None = kInt8DynamicTokenSym, +) -> tuple[Int8MoeBackend, type[mk.FusedMoEExperts]]: + """ + Select the primary Int8 MoE backend. + Note: Shape-specific fallbacks may still occur at runtime. + """ + + if config.is_lora_enabled: + return Int8MoeBackend.TRITON, backend_to_kernel_cls(Int8MoeBackend.TRITON)[0] + + AVAILABLE_BACKENDS = _get_priority_backends(config) + + activation_format = ( + mk.FusedMoEActivationFormat.BatchedExperts + if config.moe_parallel_config.use_batched_activation_format + else mk.FusedMoEActivationFormat.Standard + ) + + def _make_log_backend(backend: Int8MoeBackend) -> str: + available_backend_strs = [b.value for b in AVAILABLE_BACKENDS] + return ( + f"Using {backend.value} Int8 MoE backend out " + f"of potential backends: {available_backend_strs}." + ) + + def _make_log_unsupported(backend: Int8MoeBackend, reason: str | None) -> str: + if reason: + return ( + f"Int8 MoE backend {backend.value} does not support the " + f"deployment configuration since {reason}." + ) + else: + return ( + f"Int8 MoE backend '{backend.value}' does not support the " + "deployment configuration." + ) + + def _return_or_raise( + backend: Int8MoeBackend, + ) -> tuple[Int8MoeBackend, type[mk.FusedMoEExperts]]: + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, config, weight_key, activation_key, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend), scope="local") + return backend, k_cls + raise ValueError(_make_log_unsupported(backend, reason)) + + # Handle explicit moe_backend from user. + runner_backend = config.moe_backend + if runner_backend != "auto": + requested_backend = map_int8_backend(runner_backend) + return _return_or_raise(requested_backend) + + # Select kernels in order of backend. + for backend in AVAILABLE_BACKENDS: + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, + config, + weight_key, + activation_key, + activation_format, + ) + if supported: + logger.info_once(_make_log_backend(backend), scope="local") + return backend, k_cls + else: + logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + + raise NotImplementedError( + "No Int8 MoE backend supports the deployment configuration." + ) def make_int8_moe_quant_config( w1_scale: torch.Tensor, w2_scale: torch.Tensor, + a1_scale: torch.Tensor | None = None, + a2_scale: torch.Tensor | None = None, + per_act_token_quant: bool = False, ) -> FusedMoEQuantConfig: - return int8_w8a16_moe_quant_config( + assert (a1_scale is None and a2_scale is None) or ( + a1_scale is not None and a2_scale is not None + ), "a1_scale and a2_scale must both be provided or both be None" + + if a1_scale is None or a2_scale is None: + return int8_w8a16_moe_quant_config( + w1_scale=w1_scale, + w2_scale=w2_scale, + w1_zp=None, + w2_zp=None, + ) + + return int8_w8a8_moe_quant_config( w1_scale=w1_scale, w2_scale=w2_scale, - w1_zp=None, - w2_zp=None, + a1_scale=a1_scale, + a2_scale=a2_scale, + per_act_token_quant=per_act_token_quant, ) @@ -61,24 +183,39 @@ def make_int8_moe_kernel( routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, shared_experts: SharedExperts | None = None, ) -> mk.FusedMoEKernel: + # Create Prepare/Finalize. prepare_finalize = maybe_make_prepare_finalize( moe=moe_config, quant_config=moe_quant_config, routing_tables=routing_tables, allow_new_interface=True, + use_monolithic=issubclass(experts_cls, mk.FusedMoEExpertsMonolithic), ) assert prepare_finalize is not None logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local") - experts = experts_cls( - moe_config=moe_config, - quant_config=moe_quant_config, - ) + # Create Experts. + if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + max_num_tokens = prepare_finalize.max_num_tokens_per_rank() + assert max_num_tokens is not None + experts = experts_cls( + moe_config=moe_config, + quant_config=moe_quant_config, + max_num_tokens=max_num_tokens, + num_dispatchers=prepare_finalize.num_dispatchers(), + ) + else: + experts = experts_cls( + moe_config=moe_config, + quant_config=moe_quant_config, + ) - return mk.FusedMoEKernel( + kernel = mk.FusedMoEKernel( prepare_finalize, experts, shared_experts=shared_experts, inplace=not moe_config.disable_inplace, ) + + return kernel diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py index de155f9e179..bad5b3895b8 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w8a8_int8.py @@ -8,6 +8,7 @@ from compressed_tensors.quantization import ( QuantizationStrategy, ) +import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.logger import init_logger from vllm.model_executor.layers.fused_moe import ( FusedMoE, @@ -16,17 +17,27 @@ from vllm.model_executor.layers.fused_moe import ( from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, - int8_w8a8_moe_quant_config, +) +from vllm.model_executor.layers.fused_moe.oracle.int8 import ( + make_int8_moe_kernel, + make_int8_moe_quant_config, + select_int8_moe_backend, ) from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors_moe import ( # noqa E501 CompressedTensorsMoEMethod, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt8DynamicTokenSym, + kInt8StaticChannelSym, +) from vllm.model_executor.utils import set_weight_attrs logger = init_logger(__name__) class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): + """W8A8 Int8 MoE quantization using compressed tensors.""" + def __init__( self, weight_quant: QuantizationArgs, @@ -56,6 +67,13 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): "dynamic per token quantization. Found static input scales." ) + # Select Int8 MoE backend. + self.int8_backend, self.experts_cls = select_int8_moe_backend( + config=self.moe, + weight_key=kInt8StaticChannelSym, + activation_key=kInt8DynamicTokenSym, + ) + def create_weights( self, layer: torch.nn.Module, @@ -122,13 +140,28 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): layer.w13_input_scale = None layer.w2_input_scale = None - def process_weights_after_loading(self, layer: torch.nn.Module) -> None: - pass + def process_weights_after_loading(self, layer: FusedMoE) -> None: + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + assert self.experts_cls is not None + self.moe_kernel = make_int8_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + experts_cls=self.experts_cls, + routing_tables=layer._maybe_init_expert_routing_tables(), + shared_experts=layer.shared_experts, + ) - def get_fused_moe_quant_config( - self, layer: torch.nn.Module - ) -> FusedMoEQuantConfig | None: - return int8_w8a8_moe_quant_config( + def maybe_make_prepare_finalize( + self, + routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + ) -> mk.FusedMoEPrepareAndFinalizeModular | None: + raise ValueError( + f"{self.__class__.__name__} uses the new modular kernel initialization " + "logic. This function should not be called." + ) + + def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: + return make_int8_moe_quant_config( w1_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, a1_scale=layer.w13_input_scale, @@ -144,18 +177,17 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod): topk_ids: torch.Tensor, shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: - from vllm.model_executor.layers.fused_moe import fused_experts - - return fused_experts( - hidden_states=x, - w1=layer.w13_weight, - w2=layer.w2_weight, + assert not self.is_monolithic + assert self.moe_kernel is not None + return self.moe_kernel.apply( + x, + layer.w13_weight, + layer.w2_weight, topk_weights=topk_weights, topk_ids=topk_ids, - inplace=not self.moe.disable_inplace, activation=layer.activation, - apply_router_weight_on_input=layer.apply_router_weight_on_input, global_num_experts=layer.global_num_experts, expert_map=layer.expert_map, - quant_config=self.moe_quant_config, + apply_router_weight_on_input=layer.apply_router_weight_on_input, + shared_experts_input=shared_experts_input, ) diff --git a/vllm/model_executor/layers/quantization/online/int8.py b/vllm/model_executor/layers/quantization/online/int8.py index 18cc6aa1860..4b4c87fbce9 100644 --- a/vllm/model_executor/layers/quantization/online/int8.py +++ b/vllm/model_executor/layers/quantization/online/int8.py @@ -7,7 +7,6 @@ import torch from torch.nn import Module if TYPE_CHECKING: - import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, @@ -21,6 +20,10 @@ from vllm.model_executor.layers.fused_moe.oracle.int8 import ( from vllm.model_executor.layers.quantization.online.moe_base import ( OnlineMoEMethodBase, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + kInt8DynamicTokenSym, + kInt8StaticChannelSym, +) from vllm.model_executor.utils import replace_parameter @@ -35,8 +38,10 @@ class Int8OnlineMoEMethod(OnlineMoEMethodBase): layer: torch.nn.Module, ): super().__init__(layer.moe_config) - self.experts_cls: type[mk.FusedMoEExperts] = select_int8_moe_backend( + self.int8_backend, self.experts_cls = select_int8_moe_backend( config=self.moe, + weight_key=kInt8StaticChannelSym, + activation_key=kInt8DynamicTokenSym, ) def process_weights_after_loading(self, layer: Module) -> None: diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index f57eb39f42b..fedb9067207 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -170,6 +170,9 @@ kMxfp8Dynamic = QuantKey(FP8_DTYPE, scale=kMxfp8DynamicGroupScale, symmetric=Tru kMxfp4StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)) kMxfp4Static = QuantKey(FP4_DTYPE, scale=kMxfp4StaticGroupScale, symmetric=True) +kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True) +kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True) + def create_fp8_quant_key( static: bool, From d622e27d2be9cd4321d75073e4b7ef522204853d Mon Sep 17 00:00:00 2001 From: fxmarty-amd Date: Wed, 22 Apr 2026 17:58:54 +0200 Subject: [PATCH 042/153] [NVFP4] NVFP4 MOE emulation fallback for H100/MI300/MI350, standardize `TritonExperts` usage for OCP MX emulation (#35737) Signed-off-by: Felix Marty Signed-off-by: fxmarty-amd Co-authored-by: Kyle Sayers --- tests/evals/gsm8k/configs/models-mi3xx.txt | 2 + tests/models/quantization/test_nvfp4.py | 23 +++ vllm/config/kernel.py | 6 +- .../fused_moe/experts/nvfp4_emulation_moe.py | 164 +++++++++++++++ .../fused_moe/experts/ocp_mx_emulation_moe.py | 186 ++++++++++++++++++ .../fused_moe/experts/trtllm_nvfp4_moe.py | 6 +- .../layers/fused_moe/fused_moe.py | 58 ++---- .../layers/fused_moe/oracle/mxfp4.py | 22 +++ .../layers/fused_moe/oracle/nvfp4.py | 42 ++++ vllm/model_executor/layers/fused_moe/utils.py | 43 +++- .../layers/quantization/quark/quark_moe.py | 97 ++++----- .../utils/nvfp4_emulation_utils.py | 73 +++++-- 12 files changed, 601 insertions(+), 121 deletions(-) create mode 100644 vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py create mode 100644 vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py diff --git a/tests/evals/gsm8k/configs/models-mi3xx.txt b/tests/evals/gsm8k/configs/models-mi3xx.txt index 6cf833b6464..dfa4bc8eb53 100644 --- a/tests/evals/gsm8k/configs/models-mi3xx.txt +++ b/tests/evals/gsm8k/configs/models-mi3xx.txt @@ -2,3 +2,5 @@ DeepSeek-R1-TP_MI325.yaml DeepSeek-R1-DP_MI325.yaml DeepSeek-V3.2-TP_MI325.yaml DeepSeek-V3.2-DP_MI325.yaml +Qwen3-30B-A3B-NVFP4.yaml +Qwen3.5-35B-A3B-MXFP4-TP2.yaml \ No newline at end of file diff --git a/tests/models/quantization/test_nvfp4.py b/tests/models/quantization/test_nvfp4.py index 30f69f62130..afbcb1e5aec 100644 --- a/tests/models/quantization/test_nvfp4.py +++ b/tests/models/quantization/test_nvfp4.py @@ -120,3 +120,26 @@ def test_nvfp4(vllm_runner, model, eager, backend, monkeypatch): with vllm_runner(model, enforce_eager=eager) as llm: output = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2) assert output[0][1] == "1 2 3 4 5 6" + + +@pytest.mark.parametrize( + "model", + [ + "nvidia/Qwen3-30B-A3B-NVFP4", + "RedHatAI/Qwen3-30B-A3B-NVFP4", + ], +) +@pytest.mark.parametrize("backend", ["emulation"]) +@pytest.mark.skipif( + not current_platform.is_rocm(), + reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx", +) +def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch): + monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend) + with vllm_runner( + model, + moe_backend=backend, + load_format="dummy", + hf_overrides={"num_hidden_layers": 2}, + ) as llm: + _ = llm.generate_greedy(["1 2 3 4 5"], max_tokens=2) diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index f3ffbe4e8b1..8d8e37a0549 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -115,6 +115,7 @@ MoEBackend = Literal[ "flashinfer_cutedsl", "marlin", "aiter", + "emulation", ] @@ -142,7 +143,10 @@ class KernelConfig: - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only) - "marlin": Use Marlin kernels (weight-only quantization) - - "aiter": Use AMD AITer kernels (ROCm only)""" + - "aiter": Use AMD AITer kernels (ROCm only) + - "emulation": use BF16/FP16 GEMM, dequantizing weights and + running QDQ on activations. + """ @field_validator("moe_backend", mode="before") @classmethod diff --git a/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py b/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py new file mode 100644 index 00000000000..f1a0ee7ac52 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/nvfp4_emulation_moe.py @@ -0,0 +1,164 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +NVFP4 quantization emulation for MoE. + +This file implements NVFP4 emulation for NVFP4 MOE in case the hardware used does not +natively support NVFP4 MOE. + +Weights are dequantized on the fly during each forward, we fall back to calling +`TritonExperts` using BF16, and fake NVFP4 quantize-dequantize +is applied on `a13`, `a2`. +""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts +from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + dequantize_to_dtype, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kNvfp4Dynamic, + kNvfp4Static, +) + +logger = init_logger(__name__) + + +class Nvfp4QuantizationEmulationTritonExperts(TritonExperts): + """ + Extension of TritonExperts to support emulated NVFP4 MoE experts. + + It may be used for NVFP4 models when the device does not have + native support for this dtype. + """ + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + logger.warning_once( + "Using Nvfp4QuantizationEmulationTritonExperts MOE backend. This will" + " dequantize weights on the fly and may be slower than native" + " quantized MOE. Consider using a device with native quantization" + " support (e.g. Nvidia Blackwell) for better performance." + ) + + # `TritonExperts.apply` expects pre-dequantized weights, + # which we handle in `apply` below. + self.w1_scale_val = self.quant_config.w1_scale + self.w2_scale_val = self.quant_config.w2_scale + + self.quant_config._w1.scale = None + self.quant_config._w2.scale = None + + self.quantization_emulation = True + + @property + def quant_dtype(self) -> torch.dtype | str | None: + return "nvfp4" + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + return (weight_key, activation_key) == (kNvfp4Static, kNvfp4Dynamic) + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ): + """ + Apply emulated quantized MoE computation. + + This dequantizes the weights on the fly and calls fused_experts_impl + with activation quantization support. + """ + # Dequantize weights if they are quantized + # For NVFP4, weights are packed in uint8 format + # w1 shape: [num_experts, 2*intermediate_size, hidden_size//2] + # w2 shape: [num_experts, hidden_size, intermediate_size//2] + assert w1.dtype == torch.uint8 + assert w2.dtype == torch.uint8 + + # Dequantize w1 from packed NVFP4 to fp16/bf16 + w13_global_scale = self.quant_config.g1_alphas + + w1_dequant = dequantize_to_dtype( + tensor_fp4=w1, + tensor_sf=self.w1_scale_val, + global_scale=w13_global_scale, + dtype=hidden_states.dtype, + block_size=16, + swizzle=False, + ) + + # Dequantize w2 from packed NVFP4 to fp16/bf16 + w2_global_scale = self.quant_config.g2_alphas + + w2_dequant = dequantize_to_dtype( + tensor_fp4=w2, + tensor_sf=self.w2_scale_val, + global_scale=w2_global_scale, + dtype=hidden_states.dtype, + block_size=16, + swizzle=False, + ) + + hidden_states, _ = moe_kernel_quantize_input( + A=hidden_states, + A_scale=self.quant_config.a1_gscale, + quant_dtype="nvfp4", + per_act_token_quant=False, + quantization_emulation=True, + ) + + # Activation quantization/dequantization is deferred to + # `moe_kernel_quantize_input` in TritonExperts.apply. + super().apply( + output=output, + hidden_states=hidden_states, + w1=w1_dequant, + w2=w2_dequant, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=activation, + global_num_experts=global_num_experts, + expert_map=expert_map, + a1q_scale=None, + a2_scale=self.quant_config.a2_gscale, + workspace13=workspace13, + workspace2=workspace2, + expert_tokens_meta=expert_tokens_meta, + apply_router_weight_on_input=apply_router_weight_on_input, + ) diff --git a/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py b/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py new file mode 100644 index 00000000000..9fb163ef42a --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/experts/ocp_mx_emulation_moe.py @@ -0,0 +1,186 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +OCP MX quantization emulation for MoE. + +This file implements OCP MX (MXFP4/MXFP6) emulation for MoE in case the +hardware used does not natively support OCP MX MoE. + +Weights are dequantized on the fly during each forward, we fall back to calling +`TritonExperts` using BF16, and fake OCP MX quantize-dequantize +is applied on activations via `moe_kernel_quantize_input`. +""" + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts +from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input +from vllm.model_executor.layers.quantization.utils.mxfp4_utils import dequant_mxfp4 +from vllm.model_executor.layers.quantization.utils.mxfp6_utils import dequant_mxfp6 +from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import ( + OCP_MX_Scheme, +) + +logger = init_logger(__name__) + + +class OCP_MXQuantizationEmulationTritonExperts(TritonExperts): + """ + Extension of TritonExperts to support emulated OCP MX MoE experts. + + It may be used for OCP MX (MXFP4/MXFP6) models when the device does not + have native support for these dtypes. + """ + + def __init__( + self, + moe_config: FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + logger.warning_once( + "Using OCP_MXQuantizationEmulationTritonExperts MOE backend. This" + " will dequantize weights on the fly and may be slower than native" + " quantized MOE. Consider using a device with native OCP MX" + " quantization support for better performance." + ) + + self.ocp_mx_scheme = quant_config.ocp_mx_scheme + assert self.ocp_mx_scheme is not None, ( + "ocp_mx_scheme must be set in quant_config for" + " OCP_MXQuantizationEmulationTritonExperts" + ) + + # `TritonExperts.apply` expects pre-dequantized weights, + # which we handle in `apply` below. + self.w1_scale_val = self.quant_config.w1_scale + self.w2_scale_val = self.quant_config.w2_scale + + self.quant_config._w1.scale = None + self.quant_config._w2.scale = None + + self.quantization_emulation = True + + if self.ocp_mx_scheme in { + OCP_MX_Scheme.w_mxfp4_a_mxfp4, + }: + # Weight has to be dequantized for mxfp4 emulation. + self._quant_dtype = "mxfp4" + elif self.ocp_mx_scheme in [ + OCP_MX_Scheme.w_mxfp4_a_mxfp6_e3m2, + OCP_MX_Scheme.w_mxfp4_a_mxfp6_e2m3, + OCP_MX_Scheme.w_mxfp6_e3m2_a_mxfp6_e3m2, + OCP_MX_Scheme.w_mxfp6_e2m3_a_mxfp6_e2m3, + ]: + self._quant_dtype = "mxfp6" + elif self.ocp_mx_scheme in [ + OCP_MX_Scheme.w_mxfp4_a_fp8, + OCP_MX_Scheme.w_mxfp6_e3m2_a_fp8, + ]: + # TODO: double check this one + self._quant_dtype = "mxfp8" + + @property + def quant_dtype(self) -> torch.dtype | str | None: + return self._quant_dtype + + @property + def expects_unquantized_inputs(self) -> bool: + return True + + @staticmethod + def _supports_quant_scheme( + weight_key, + activation_key, + ) -> bool: + # This class is used for emulation only - the oracle selects it + # directly rather than via quant scheme matching. + return True + + def _dequantize_weights( + self, + w: torch.Tensor, + w_scale: torch.Tensor, + dtype: torch.dtype, + ) -> torch.Tensor: + """Dequantize weights based on the OCP MX scheme.""" + if self.ocp_mx_scheme.startswith("w_mxfp4"): # type: ignore[union-attr] + return dequant_mxfp4(w, w_scale, dtype) + elif self.ocp_mx_scheme.startswith("w_mxfp6_e3m2"): # type: ignore[union-attr] + return dequant_mxfp6(w, w_scale, quant_dtype="fp6_e3m2", float_dtype=dtype) + elif self.ocp_mx_scheme.startswith("w_mxfp6_e2m3"): # type: ignore[union-attr] + return dequant_mxfp6(w, w_scale, quant_dtype="fp6_e2m3", float_dtype=dtype) + else: + raise NotImplementedError(f"Unsupported ocp_mx_scheme={self.ocp_mx_scheme}") + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, + workspace2: torch.Tensor, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool, + ): + """ + Apply emulated quantized MoE computation. + + This dequantizes the weights on the fly and calls TritonExperts.apply + with activation quantization support. + """ + assert w1.dtype == torch.uint8 + assert w2.dtype == torch.uint8 + + # Dequantize w1 and w2 from packed OCP MX format to bf16/fp16 + w1_dequant = self._dequantize_weights( + w1, self.w1_scale_val, hidden_states.dtype + ) + w2_dequant = self._dequantize_weights( + w2, self.w2_scale_val, hidden_states.dtype + ) + + # Apply activation QDQ if needed by the OCP MX scheme + hidden_states, _ = moe_kernel_quantize_input( + A=hidden_states, + A_scale=None, + quant_dtype=self.quant_config.quant_dtype, + per_act_token_quant=False, + ocp_mx_scheme=self.ocp_mx_scheme, + quantization_emulation=True, + ) + + # Activation quantization/dequantization is deferred to + # `moe_kernel_quantize_input` in TritonExperts.apply. + super().apply( + output=output, + hidden_states=hidden_states, + w1=w1_dequant, + w2=w2_dequant, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=activation, + global_num_experts=global_num_experts, + expert_map=expert_map, + a1q_scale=None, + a2_scale=None, + workspace13=workspace13, + workspace2=workspace2, + expert_tokens_meta=expert_tokens_meta, + apply_router_weight_on_input=apply_router_weight_on_input, + ) diff --git a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py index fc30815f719..c6689bf9fed 100644 --- a/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/trtllm_nvfp4_moe.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import flashinfer + import torch import vllm.model_executor.layers.fused_moe.modular_kernel as mk @@ -188,6 +188,8 @@ class TrtLlmNvFp4ExpertsModular(TrtLlmNvFp4ExpertsBase, mk.FusedMoEExpertsModula expert_tokens_meta: mk.ExpertTokensMetadata | None, apply_router_weight_on_input: bool, ): + import flashinfer + assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL] assert a1q_scale is not None assert self.quant_config.w1_scale is not None @@ -306,6 +308,8 @@ class TrtLlmNvFp4ExpertsMonolithic( routed_scaling_factor: float | None = None, topk_group: int | None = None, ) -> torch.Tensor: + import flashinfer + assert activation in [MoEActivation.SILU, MoEActivation.RELU2_NO_MUL] assert a1q_scale is not None assert self.quant_config.w1_scale is not None diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index a8c3b1d7713..bf083eb9b55 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -36,8 +36,6 @@ from vllm.model_executor.layers.fused_moe.utils import ( disable_inplace, moe_kernel_quantize_input, ) -from vllm.model_executor.layers.quantization.utils.mxfp4_utils import dequant_mxfp4 -from vllm.model_executor.layers.quantization.utils.mxfp6_utils import dequant_mxfp6 from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, kFp8Dynamic128Sym, @@ -1708,22 +1706,18 @@ def fused_experts_impl( w1_bias: torch.Tensor | None = None, w2_bias: torch.Tensor | None = None, ) -> torch.Tensor: + if ocp_mx_scheme is not None: + raise NotImplementedError( + f"Using ocp_mx_scheme={ocp_mx_scheme} in functional fused_experts call is " + "deprecated. Please use OCP_MXQuantizationEmulationTritonExperts." + ) + # Convert string activation to enum for internal use activation_enum = MoEActivation.from_str(activation) # Check constraints. if use_int4_w4a16: assert hidden_states.size(1) // 2 == w1.size(2), "Hidden size mismatch" - elif ocp_mx_scheme is not None: - if ocp_mx_scheme.startswith("w_mxfp4"): - # 16bit activation and fp4x2 packed weight - assert hidden_states.size(1) == w1.size(2) * 2, "hidden size mismatch" - elif ocp_mx_scheme.startswith("w_mxfp6"): - assert hidden_states.size(1) == (w1.size(2) * 4) // 3, ( - "hidden size mismatch" - ) - else: - raise NotImplementedError(f"Unsupported ocp_mx_scheme={ocp_mx_scheme}") else: assert hidden_states.size(1) == w1.size(2), ( f"Hidden size mismatch {hidden_states.size(1)} != {w1.size(2)}" @@ -1748,7 +1742,6 @@ def fused_experts_impl( use_fp8_w8a8=use_fp8_w8a8, use_int8_w8a16=use_int8_w8a16, use_int4_w4a16=use_int4_w4a16, - ocp_mx_scheme=ocp_mx_scheme, dtype=hidden_states.dtype, ) @@ -1757,7 +1750,7 @@ def fused_experts_impl( quant_dtype = _get_config_quant_dtype( use_fp8_w8a8=use_fp8_w8a8, use_int8_w8a8=use_int8_w8a8, - ocp_mx_scheme=ocp_mx_scheme, + ocp_mx_scheme=None, ) get_config_func = functools.partial( @@ -1802,44 +1795,12 @@ def fused_experts_impl( out_hidden_states = hidden_states if inplace else torch.empty_like(hidden_states) - if ocp_mx_scheme is not None: - # TODO: On platforms for which `current_platform.supports_mx()` is True - # and for which we have a native OCP mx fused MOE kernel, - # this dequantization step should not be done. - if ocp_mx_scheme.startswith("w_mxfp4"): - # Weight has to be dequantized for mxfp4 emulation. - w1 = dequant_mxfp4(w1, w1_scale, hidden_states.dtype) - w1_scale = None - w2 = dequant_mxfp4(w2, w2_scale, hidden_states.dtype) - w2_scale = None - elif ocp_mx_scheme.startswith("w_mxfp6_e3m2"): - w1 = dequant_mxfp6( - w1, w1_scale, quant_dtype="fp6_e3m2", float_dtype=hidden_states.dtype - ) - w1_scale = None - w2 = dequant_mxfp6( - w2, w2_scale, quant_dtype="fp6_e3m2", float_dtype=hidden_states.dtype - ) - w2_scale = None - elif ocp_mx_scheme.startswith("w_mxfp6_e2m3"): - w1 = dequant_mxfp6( - w1, w1_scale, quant_dtype="fp6_e2m3", float_dtype=hidden_states.dtype - ) - w1_scale = None - w2 = dequant_mxfp6( - w2, w2_scale, quant_dtype="fp6_e2m3", float_dtype=hidden_states.dtype - ) - w2_scale = None - else: - raise NotImplementedError(f"Unsupported ocp_mx_scheme={ocp_mx_scheme}") - qhidden_states, a1q_scale = moe_kernel_quantize_input( A=hidden_states, A_scale=a1_scale, quant_dtype=quant_dtype, per_act_token_quant=per_channel_quant, block_shape=block_shape, - ocp_mx_scheme=ocp_mx_scheme, ) sorted_token_ids, expert_ids, num_tokens_post_padded = _prepare_expert_assignment( @@ -1889,7 +1850,6 @@ def fused_experts_impl( quant_dtype=quant_dtype, per_act_token_quant=per_channel_quant, block_shape=block_shape, - ocp_mx_scheme=ocp_mx_scheme, ) if expert_map is not None: @@ -1935,6 +1895,9 @@ class TritonExperts(mk.FusedMoEExpertsModular): moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig, ): + # Whether quantized MOE runs natively, or through + # higher-precision + activation QDQ. + self.quantization_emulation = False super().__init__(moe_config, quant_config) @staticmethod @@ -2144,6 +2107,7 @@ class TritonExperts(mk.FusedMoEExpertsModular): self.quant_dtype, self.per_act_token_quant, self.block_shape, + quantization_emulation=self.quantization_emulation, ) invoke_fused_moe_triton_kernel( diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 587954d5267..13d7a902c30 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -62,6 +62,8 @@ class Mxfp4MoeBackend(Enum): TRITON_UNFUSED = "TRITON_UNFUSED" # XPU XPU = "XPU" + # Emulation + EMULATION = "EMULATION" # Backends that share the same TRTLLM weight format @@ -143,6 +145,13 @@ def backend_to_kernel_cls( return [XPUExpertsMXFp4] + elif backend == Mxfp4MoeBackend.EMULATION: + from vllm.model_executor.layers.fused_moe.experts.ocp_mx_emulation_moe import ( + OCP_MXQuantizationEmulationTritonExperts, + ) + + return [OCP_MXQuantizationEmulationTritonExperts] + else: raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}") @@ -158,6 +167,7 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> Mxfp4MoeBackend: "marlin": Mxfp4MoeBackend.MARLIN, "aiter": Mxfp4MoeBackend.AITER, "xpu": Mxfp4MoeBackend.XPU, + "emulation": Mxfp4MoeBackend.EMULATION, } if backend := mapping.get(runner_backend): return backend @@ -181,6 +191,7 @@ def _get_priority_backends() -> list[Mxfp4MoeBackend]: Mxfp4MoeBackend.MARLIN, Mxfp4MoeBackend.BATCHED_MARLIN, Mxfp4MoeBackend.XPU, + Mxfp4MoeBackend.EMULATION, ] return _AVAILABLE_BACKENDS @@ -768,6 +779,17 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( w13_bias, w2_bias, ) + elif mxfp4_backend == Mxfp4MoeBackend.EMULATION: + # No additional transformation needed for emulation backend, + # weights are dequantized on the fly in the experts class. + return ( + w13_weight, + w2_weight, + w13_weight_scale, + w2_weight_scale, + w13_bias, + w2_bias, + ) else: raise ValueError( f"Unsupported mxfp4_backend: {mxfp4_backend}: " diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 597d784d3b6..6d0b66cb9f5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -45,6 +45,7 @@ class NvFp4MoeBackend(Enum): FLASHINFER_CUTEDSL_BATCHED = "FLASHINFER_CUTEDSL_BATCHED" VLLM_CUTLASS = "VLLM_CUTLASS" MARLIN = "MARLIN" + EMULATION = "EMULATION" FLASHINFER_NVFP4_MOE_BACKENDS = [ @@ -118,6 +119,12 @@ def backend_to_kernel_cls( ) return [MarlinExperts] + elif backend == NvFp4MoeBackend.EMULATION: + from vllm.model_executor.layers.fused_moe.experts.nvfp4_emulation_moe import ( + Nvfp4QuantizationEmulationTritonExperts, + ) + + return [Nvfp4QuantizationEmulationTritonExperts] else: raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}") @@ -130,6 +137,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend: "flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS, "flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL, "marlin": NvFp4MoeBackend.MARLIN, + "emulation": NvFp4MoeBackend.EMULATION, } if backend := mapping.get(runner_backend): return backend @@ -157,6 +165,7 @@ def select_nvfp4_moe_backend( NvFp4MoeBackend.FLASHINFER_CUTLASS, NvFp4MoeBackend.VLLM_CUTLASS, NvFp4MoeBackend.MARLIN, + NvFp4MoeBackend.EMULATION, ] # NOTE(rob): this is kind of a hack. We need to peak into @@ -372,6 +381,30 @@ def convert_to_nvfp4_moe_kernel_format( w2_scale_2=w2_scale_2, is_act_and_mul=is_act_and_mul, ) + elif nvfp4_backend == NvFp4MoeBackend.EMULATION: + if a13_scale is None or a2_scale is None: + raise ValueError( + "Activation global scales should not be None, got" + f" a13_scale={a13_scale}, a2_scale={a2_scale}" + ) + + if torch.unique(a13_scale).numel() != 1 or torch.unique(a2_scale).numel() != 1: + logger.warning_once( + "In NVFP4 linear, the activation global scale for inputs are different" + " for MOE w13 (gate_up_proj) layer or MOE w2 (down_proj). Using" + " a13_scale = a13_scale.max() and a2_scale = a2_scale.max()." + ) + + # 1. We take the max following e.g. quantization/utils/flashinfer_fp4_moe.py. + # 2. moe_kernel_quantize_input -> ref_nvfp4_quant_dequant + # use the inverse scale directly (large global scale). + # NOTE: Before this point, `a13_scale` and `a2_scale` are such that: + # `FP8_MAX = activation[expert_id].abs().max() * global_scale[expert_id]`, + # and `global_scale[expert_id]` are small (~1e-4). + # Taking the largest global scale likely results in overflowing the FP8 range + # for other experts - other selection strategies may be used. + a13_scale = 1.0 / a13_scale.max().to(torch.float32) + a2_scale = 1.0 / a2_scale.max().to(torch.float32) else: raise ValueError(f"Unknown NvFp4 backend for MoE: {nvfp4_backend}") @@ -403,6 +436,15 @@ def make_nvfp4_moe_quant_config( w1_scale=w13_scale, w2_scale=w2_scale, ) + elif backend == NvFp4MoeBackend.EMULATION: + return nvfp4_moe_quant_config( + g1_alphas=w13_scale_2, + g2_alphas=w2_scale_2, + a1_gscale=a13_scale, + a2_gscale=a2_scale, + w1_scale=w13_scale, + w2_scale=w2_scale, + ) # Pass w13_scale_2 / w2_scale_2 directly as g1/g2_alphas. # The expert's process_weights_after_loading will fuse activation diff --git a/vllm/model_executor/layers/fused_moe/utils.py b/vllm/model_executor/layers/fused_moe/utils.py index ce1e49bc4b0..d8e174051b5 100644 --- a/vllm/model_executor/layers/fused_moe/utils.py +++ b/vllm/model_executor/layers/fused_moe/utils.py @@ -22,6 +22,9 @@ from vllm.model_executor.layers.quantization.utils.mxfp6_utils import ( from vllm.model_executor.layers.quantization.utils.mxfp8_utils import ( mxfp8_e4m3_quantize, ) +from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import ( + ref_nvfp4_quant_dequant, +) from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( per_tensor_dequantize, ) @@ -253,6 +256,7 @@ def moe_kernel_quantize_input( block_shape: list[int] | None = None, is_fp4_scale_swizzled: bool = True, ocp_mx_scheme: str | None = None, + quantization_emulation: bool = False, ) -> tuple[torch.Tensor, torch.Tensor | None]: # Handle OCP MX scheme that requires QDQ (quantize-dequantize) for emulation if ocp_mx_scheme is not None: @@ -274,16 +278,41 @@ def moe_kernel_quantize_input( # activation quantization below. if quant_dtype == current_platform.fp8_dtype(): + if quantization_emulation: + raise NotImplementedError( + f"moe_kernel_quantize_input does not support quant_dtype={quant_dtype}" + " MOE quantization emulation. Please open an issue." + ) return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape) elif quant_dtype == torch.int8: + if quantization_emulation: + raise NotImplementedError( + "moe_kernel_quantize_input does not support quant_dtype=torch.int8" + " MOE quantization emulation. Please open an issue." + ) return _int8_quantize(A, A_scale, per_act_token_quant, block_shape) elif quant_dtype == "nvfp4": - return _nvfp4_quantize(A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled) + if not quantization_emulation: + return _nvfp4_quantize( + A, A_scale, is_sf_swizzled_layout=is_fp4_scale_swizzled + ) + else: + return ref_nvfp4_quant_dequant(A, A_scale, block_size=16) elif quant_dtype == "mxfp4": + if not quantization_emulation: + raise NotImplementedError( + "moe_kernel_quantize_input should not be used for native" + " quant_dtype='mxfp4' MOE. Please open an issue." + ) return _mxfp4_quantize(A, A_scale, per_act_token_quant, block_shape) elif quant_dtype == "mxfp8": # TODO: `quant_dtype == "mxfp8"` is ambiguous, # should be fp8_e4m3. OCP MX also defines `fp8_e5m2`. + if quantization_emulation: + raise NotImplementedError( + "moe_kernel_quantize_input does not support quant_dtype='mxfp8' MOE " + "quantization emulation. Please open an issue." + ) return _mxfp8_e4m3_quantize( A, A_scale, @@ -292,8 +321,20 @@ def moe_kernel_quantize_input( is_sf_swizzled_layout=is_fp4_scale_swizzled, ) elif quant_dtype == "mxfp6_e3m2": + if not quantization_emulation: + raise NotImplementedError( + "moe_kernel_quantize_input should not be used for native " + " quant_dtype='mxfp6_e3m2'MOE. Please open an issue." + ) + return _mxfp6_e3m2_quantize(A, A_scale, per_act_token_quant, block_shape) elif quant_dtype == "mxfp6_e2m3": + if not quantization_emulation: + raise NotImplementedError( + "moe_kernel_quantize_input should not be used for native" + " quant_dtype='mxfp6_e2m3' MOE. Please open an issue." + ) + return _mxfp6_e2m3_quantize(A, A_scale, per_act_token_quant, block_shape) else: return A, A_scale diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 2bab66709dd..64753a173df 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_m from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import ( TRITON_BACKENDS, Mxfp4MoeBackend, + backend_to_kernel_cls, convert_gpt_oss_weight_to_mxfp4_moe_kernel_format, make_mxfp4_moe_kernel, make_mxfp4_moe_quant_config, @@ -986,6 +987,8 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): f"Please check that the combination is supported in OCP_MX_Scheme." ) + # TODO(bowenbao): refactor and introduce backends for other OCP MX schemes, + # use kernel abstraction for all OCP MX MOE implementations. self.mxfp4_backend: Mxfp4MoeBackend = Mxfp4MoeBackend.NONE self.experts_cls: type[mk.FusedMoEExperts] | None = None self.moe_kernel: mk.FusedMoEKernel | None = None @@ -994,12 +997,6 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): self.w13_precision_config = None self.w2_precision_config = None - if self.ocp_mx_scheme == "w_mxfp4": - self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe) - elif self.ocp_mx_scheme.startswith("w_mxfp4"): - # TODO(bowenbao): refactor and introduce backends for other OCP MX schemes. - self.mxfp4_backend = Mxfp4MoeBackend.NONE - if self.input_quant is not None: self.static_input_scales = not self.input_quant.get("is_dynamic") else: @@ -1035,6 +1032,18 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): self.mxfp4_backend is Mxfp4MoeBackend.NONE or not self.use_rocm_aiter_moe ) + if self.ocp_mx_scheme == "w_mxfp4": + self.mxfp4_backend, self.experts_cls = select_gpt_oss_mxfp4_moe_backend(moe) + + if self.emulate: + # We use the same code path between MXFP4/MXFP6 emulation. + self.mxfp4_backend = Mxfp4MoeBackend.EMULATION + + # TODO: Remove `self.mxfp4_backend != Mxfp4MoeBackend.NONE` and make it so that + # all MXFP4 backends use the kernel abstraction. + if self.mxfp4_backend != Mxfp4MoeBackend.NONE: + self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0] + if self.emulate: logger.warning_once( f"The current mode (supports_mx={current_platform.supports_mx()}, " @@ -1063,7 +1072,12 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): act_dtype=act_dtype, moe_parallel_config=moe_parallel_config, ) - if self.mxfp4_backend is not None: + # In case quantization emulation backend is used, there is no need to apply + # MXFP4-specific padding logic as the compute happens in higher precision. + if ( + self.mxfp4_backend is not None + and self.mxfp4_backend != Mxfp4MoeBackend.EMULATION + ): hidden_size, intermediate_size_per_partition = ( mxfp4_round_up_hidden_size_and_intermediate_size( self.mxfp4_backend, hidden_size, intermediate_size_per_partition @@ -1237,7 +1251,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): ) # For w_mxfp4, use oracle functions - if ( + if self.emulate or ( self.ocp_mx_scheme == "w_mxfp4" and self.mxfp4_backend != Mxfp4MoeBackend.NONE ): @@ -1245,13 +1259,6 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): return # TODO(bowenbao): gradually migrate to oracles. - # secondly, process mxfp weights for other schemes - if self.emulate: - # Build quant config for emulation path - self.moe_quant_config = self.get_fused_moe_quant_config(layer) - torch.accelerator.empty_cache() - return - # Existing AITER path for w_mxfp4_a_mxfp4 and other schemes from aiter.utility.fp4_utils import e8m0_shuffle @@ -1345,9 +1352,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): self, layer: torch.nn.Module ) -> FusedMoEQuantConfig | None: # For w_mxfp4 with oracle backend, use oracle function - if ( - self.ocp_mx_scheme == "w_mxfp4" - and self.mxfp4_backend != Mxfp4MoeBackend.NONE + if self.ocp_mx_scheme == "w_mxfp4" and self.mxfp4_backend not in ( + Mxfp4MoeBackend.NONE, + Mxfp4MoeBackend.EMULATION, ): w1_scale = layer.w13_weight_scale w2_scale = layer.w2_weight_scale @@ -1362,9 +1369,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): w2_bias=getattr(layer, "w2_bias", None), ) - # Existing code for other schemes - # TODO(bowenbao): kept for emulation fallback, to be refactored into - # dedicated emulation backend. + # Emulation and other schemes if self.ocp_mx_scheme == "w_mxfp4": return mxfp4_w4a16_moe_quant_config( w1_scale=layer.w13_weight_scale, @@ -1414,7 +1419,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): topk_ids: torch.Tensor, shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: - # For w_mxfp4 with oracle kernel + # For oracle kernel or emulation kernel if self.moe_kernel is not None: return self.moe_kernel.apply( hidden_states=x, @@ -1429,39 +1434,23 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): shared_experts_input=shared_experts_input, ) - # Existing code for emulation/AITER paths - if not self.emulate: - from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( - rocm_aiter_fused_experts, - ) + # AITER path + # TODO: Refactor this to use modular MOE kernel as well. + from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( + rocm_aiter_fused_experts, + ) - return rocm_aiter_fused_experts( - x, - layer.w13_weight, - layer.w2_weight, - topk_weights=topk_weights, - topk_ids=topk_ids, - activation=layer.activation, - quant_config=self.moe_quant_config, - moe_config=layer.moe_config, - expert_map=layer.expert_map, - ) - else: - from vllm.model_executor.layers.fused_moe import fused_experts - - return fused_experts( - x, - layer.w13_weight, - layer.w2_weight, - topk_weights=topk_weights, - topk_ids=topk_ids, - inplace=not self.moe.disable_inplace, - activation=layer.activation, - global_num_experts=layer.global_num_experts, - apply_router_weight_on_input=layer.apply_router_weight_on_input, - expert_map=layer.expert_map, - quant_config=self.moe_quant_config, - ) + return rocm_aiter_fused_experts( + x, + layer.w13_weight, + layer.w2_weight, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=layer.activation, + quant_config=self.moe_quant_config, + moe_config=layer.moe_config, + expert_map=layer.expert_map, + ) def apply_monolithic( self, diff --git a/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py b/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py index 9a0c52b62c1..af5c6f2a7ab 100644 --- a/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py +++ b/vllm/model_executor/layers/quantization/utils/nvfp4_emulation_utils.py @@ -53,26 +53,52 @@ def convert_swizzled_to_linear(a_sf_swizzled: torch.Tensor, m, k, block_size): def dequantize_to_dtype( tensor_fp4: torch.Tensor, tensor_sf: torch.Tensor, - global_scale: torch.Tensor | float, + global_scale: torch.Tensor, dtype: torch.dtype, block_size: int = 16, swizzle: bool | None = True, ): - """Dequantize the fp4 tensor back to high precision.""" + """Dequantize the fp4 tensor back to high precision. + + Supports both 2D and 3D inputs: + - 2D: [m, packed_k] -> [m, k] + - 3D: [dim0, m, packed_k] -> [dim0, m, k] + """ # Two fp4 values are packed into one uint8. assert tensor_fp4.dtype == torch.uint8 - m, packed_k = tensor_fp4.shape + + # We handle 3D tensors reshaping them to 2D. + is_3d = tensor_fp4.ndim == 3 + + if is_3d: + dim0, m, packed_k = tensor_fp4.shape + tensor_fp4 = tensor_fp4.reshape(-1, packed_k) + tensor_sf = tensor_sf.reshape(-1, tensor_sf.shape[-1]) + global_scale = global_scale[:, None, None] + else: + m, packed_k = tensor_fp4.shape + k = packed_k * 2 tensor_f32 = break_fp4_bytes(tensor_fp4, torch.float32) - tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size) + tensor_f32 = tensor_f32.reshape(-1, k // block_size, block_size) tensor_sf = tensor_sf.view(torch.float8_e4m3fn) if swizzle: - tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size) + tensor_sf = convert_swizzled_to_linear( # noqa: E501 + tensor_sf, tensor_f32.size(0), k, block_size + ) + + if is_3d: + tensor_sf = tensor_sf.reshape(dim0, m, k // block_size) tensor_sf_dtype = tensor_sf.to(torch.float32) * global_scale + if is_3d: + tensor_f32 = tensor_f32.reshape(dim0, m, -1, block_size) + # scale the tensor - out = (tensor_f32 * tensor_sf_dtype.unsqueeze(-1)).reshape(m, k) + out = tensor_f32 * tensor_sf_dtype.unsqueeze(-1) + out = out.reshape(*out.shape[:-2], -1) + return out.to(dtype) @@ -117,6 +143,28 @@ def ref_nvfp4_quant(x, global_scale, block_size): return cast_to_fp4(clipped_x), scale.squeeze(-1) +def ref_nvfp4_quant_dequant( + x: torch.Tensor, global_scale: torch.Tensor, block_size: int +) -> tuple[torch.Tensor, None]: + """ + NVFP4 quantize-dequantize operation. + + `global_scale` is expected to have a single element. + """ + x_m, x_k = x.shape + output_dtype = x.dtype + + # quantize input to (FP4 and interleaved block scale) + x_fp4, x_blockscale = ref_nvfp4_quant(x, global_scale, block_size) + + # dequantize input + x_fp4 = x_fp4.reshape(x_m, x_k // block_size, block_size) + x_blockscale = x_blockscale.unsqueeze(-1) / global_scale + x_dq = (x_fp4 * x_blockscale).reshape(x_m, x_k).to(output_dtype) + + return x_dq, None + + def run_nvfp4_emulations( x: torch.Tensor, input_global_scale: torch.Tensor, @@ -125,18 +173,10 @@ def run_nvfp4_emulations( weight_global_scale: torch.Tensor, swizzle: bool | None = True, ): - group_size = 16 - x_m, x_k = x.shape output_dtype = x.dtype + group_size = 16 - # quantize input to (FP4 and interleaved block scale) - x_fp4, x_blockscale = ref_nvfp4_quant(x, input_global_scale, group_size) - - # dequantize input - x_fp4 = x_fp4.reshape(x_m, x_k // group_size, group_size) - x_blockscale = x_blockscale.unsqueeze(-1) / input_global_scale - x_dq = (x_fp4 * x_blockscale).reshape(x_m, x_k).to(output_dtype) - del x_fp4, x_blockscale + x_dq, _ = ref_nvfp4_quant_dequant(x, input_global_scale, block_size=group_size) # dequantize weight w_fp4 = weight.data.view(torch.uint8) @@ -151,5 +191,4 @@ def run_nvfp4_emulations( # matmul out = torch.matmul(x_dq, w_dq.t()) - del w_dq, x_dq return out From eb6661d52260a3a43e40dae6c808db49487e87c4 Mon Sep 17 00:00:00 2001 From: Angela Yi Date: Wed, 22 Apr 2026 12:31:41 -0700 Subject: [PATCH 043/153] Fix test_startup.py for torch 2.12 (#40636) Signed-off-by: Angela Yi --- tests/compile/h100/test_startup.py | 14 ++++++-------- tests/compile/test_dynamic_shapes_compilation.py | 2 +- vllm/env_override.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/compile/h100/test_startup.py b/tests/compile/h100/test_startup.py index 6a94322b1b6..1e1c93217f9 100644 --- a/tests/compile/h100/test_startup.py +++ b/tests/compile/h100/test_startup.py @@ -135,10 +135,9 @@ MODEL_SPECS = [ model="deepseek-ai/DeepSeek-V3.2", hf_overrides=_SMALL_MOE_OVERRIDES, cold_artifacts_saved=4, - # TODO: https://github.com/vllm-project/vllm/issues/38051 - # We shouldn't be saving any artifacts on warm start. - warm_artifacts_saved=4, - warm_artifacts_loaded=0, + # https://github.com/vllm-project/vllm/issues/38051 + warm_artifacts_saved=0 if is_torch_equal_or_newer("2.12.0") else 4, + warm_artifacts_loaded=4 if is_torch_equal_or_newer("2.12.0") else 0, ), id="deepseek_v3.2", ), @@ -147,10 +146,9 @@ MODEL_SPECS = [ model="moonshotai/Kimi-K2.5", hf_overrides={"text_config": _SMALL_MOE_OVERRIDES}, cold_artifacts_saved=4, - # TODO: https://github.com/vllm-project/vllm/issues/38051 - # We shouldn't be saving any artifacts on warm start. - warm_artifacts_saved=4, - warm_artifacts_loaded=0, + # https://github.com/vllm-project/vllm/issues/38051 + warm_artifacts_saved=0 if is_torch_equal_or_newer("2.12.0") else 4, + warm_artifacts_loaded=4 if is_torch_equal_or_newer("2.12.0") else 0, ), id="kimi_k2.5", ), diff --git a/tests/compile/test_dynamic_shapes_compilation.py b/tests/compile/test_dynamic_shapes_compilation.py index 1775b2c9deb..c5b7d783af0 100644 --- a/tests/compile/test_dynamic_shapes_compilation.py +++ b/tests/compile/test_dynamic_shapes_compilation.py @@ -28,7 +28,7 @@ def get_test_models(): "Qwen/Qwen2-7B-Instruct", "meta-llama/Llama-3.1-8B", ] - if is_torch_equal_or_newer("2.12.0"): + if is_torch_equal_or_newer("2.12.0.dev"): models.append("Qwen/Qwen3-4B-Instruct-2507") return models diff --git a/vllm/env_override.py b/vllm/env_override.py index a19b6e25541..f0dc91e11b4 100644 --- a/vllm/env_override.py +++ b/vllm/env_override.py @@ -553,7 +553,7 @@ def _apply_constrain_to_fx_strides_patch(): _lowering.constrain_to_fx_strides = _patched -if is_torch_equal_or_newer("2.10.0") and not is_torch_equal_or_newer("2.12.0"): +if is_torch_equal_or_newer("2.10.0") and not is_torch_equal_or_newer("2.12.0.dev"): import builtins as _builtins import pickle From 29f64c5f5e635e5071fe23ef6ed36edb520b9ae8 Mon Sep 17 00:00:00 2001 From: Fynn Schmitt-Ulms Date: Wed, 22 Apr 2026 16:22:57 -0400 Subject: [PATCH 044/153] FlexAttention non-causal support (#40394) Signed-off-by: Fynn Schmitt-Ulms Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/v1/attention/test_attention_backends.py | 77 ++++++++++++++++++- tests/v1/attention/utils.py | 23 +++++- vllm/v1/attention/backends/flex_attention.py | 39 +++++++--- 3 files changed, 122 insertions(+), 17 deletions(-) diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index 06095b87e59..41218c41f4f 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -315,6 +315,7 @@ def _test_backend_correctness( backend_to_test: list[AttentionBackendEnum | str], mask_mod, *, + causal: bool = True, attn_type: AttentionType = AttentionType.DECODER, block_size: int = 16, atol: float = 1e-2, @@ -370,7 +371,7 @@ def _test_backend_correctness( ) device = torch.device(f"{DEVICE_TYPE}:0") - kv_cache_spec = create_standard_kv_cache_spec(vllm_config) + kv_cache_spec = create_standard_kv_cache_spec(vllm_config, attn_type) # 1. Setup batch_size = batch_spec.batch_size @@ -453,9 +454,7 @@ def _test_backend_correctness( common_attn_metadata = create_common_attn_metadata( batch_spec, vllm_config.cache_config.block_size, device ) - if attn_type == AttentionType.ENCODER_ONLY: - # For encoder-only, all tokens are prefill tokens - common_attn_metadata.causal = False + common_attn_metadata.causal = causal # 3. Simulate Paged KV Cache and a realistic slot_mapping kv_cache = create_and_prepopulate_kv_cache( @@ -736,6 +735,76 @@ def test_sliding_window_encoder_backend_correctness( model, SLIDING_WINDOW_BACKENDS_TO_TEST, sliding_window_mask_mod_fn, + causal=False, attn_type=AttentionType.ENCODER_ONLY, tensor_parallel_size=tensor_parallel_size, ) + + +NON_CAUSAL_BACKENDS_TO_TEST = [ + AttentionBackendEnum.FLASH_ATTN, + AttentionBackendEnum.FLEX_ATTENTION, + "FLEX_ATTENTION_SLOW", +] + +if current_platform.is_rocm(): + NON_CAUSAL_BACKENDS_TO_TEST = [ + x + for x in NON_CAUSAL_BACKENDS_TO_TEST + if x is not AttentionBackendEnum.FLASH_ATTN + ] + + +@pytest.mark.parametrize( + "batch_spec_name", + [ + "small_decode", + "small_prefill", + "mixed_small", + ], +) +@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"]) +def test_non_causal_backend_correctness( + default_vllm_config, batch_spec_name: str, model: str +): + """Test backend's correctness with non-causal (bidirectional) decoder + attention, as used by DFlash speculative decoding.""" + + def bidirectional_mask_mod( + b: torch.Tensor, + h: torch.Tensor, + q_idx: torch.Tensor, + kv_idx: torch.Tensor, + *, + context_len: int, + ): + return q_idx >= 0 # Always True + + batch_spec = BATCH_SPECS[batch_spec_name] + LARGE_BLOCK_BACKENDS = ( + [AttentionBackendEnum.FLEX_ATTENTION] + if is_torch_equal_or_newer("2.9.0.dev0") + else [] + ) + + SMALL_BLOCK_BACKENDS = [ + x for x in NON_CAUSAL_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS + ] + + _test_backend_correctness( + batch_spec, + model, + SMALL_BLOCK_BACKENDS, + bidirectional_mask_mod, + causal=False, + ) + + if LARGE_BLOCK_BACKENDS: + _test_backend_correctness( + batch_spec, + model, + LARGE_BLOCK_BACKENDS, + bidirectional_mask_mod, + causal=False, + block_size=128, + ) diff --git a/tests/v1/attention/utils.py b/tests/v1/attention/utils.py index 91decf6658a..aac4a46be3b 100644 --- a/tests/v1/attention/utils.py +++ b/tests/v1/attention/utils.py @@ -21,10 +21,11 @@ from vllm.config.model import ModelDType from vllm.v1.attention.backend import ( AttentionImpl, AttentionMetadataBuilder, + AttentionType, CommonAttentionMetadata, ) from vllm.v1.attention.backends.registry import AttentionBackendEnum -from vllm.v1.kv_cache_interface import FullAttentionSpec +from vllm.v1.kv_cache_interface import EncoderOnlyAttentionSpec, FullAttentionSpec @dataclass @@ -142,8 +143,24 @@ def try_backend_includes_kv_cache_update( raise AssertionError("unreachable") from None -def create_standard_kv_cache_spec(vllm_config: VllmConfig) -> FullAttentionSpec: - """Create a FullAttentionSpec from ModelParams only.""" +def create_standard_kv_cache_spec( + vllm_config: VllmConfig, + attn_type: AttentionType = AttentionType.DECODER, +) -> FullAttentionSpec | EncoderOnlyAttentionSpec: + """Create an AttentionSpec from VllmConfig. + + Returns an EncoderOnlyAttentionSpec for encoder-only attention (no KV + cache), and a FullAttentionSpec otherwise. + """ + if attn_type == AttentionType.ENCODER_ONLY: + return EncoderOnlyAttentionSpec( + block_size=vllm_config.cache_config.block_size, + num_kv_heads=vllm_config.model_config.get_num_kv_heads( + vllm_config.parallel_config + ), + head_size=vllm_config.model_config.get_head_size(), + dtype=vllm_config.model_config.dtype, + ) return FullAttentionSpec( block_size=vllm_config.cache_config.block_size, num_kv_heads=vllm_config.model_config.get_num_kv_heads( diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index 3c5b99904b6..a027fe52441 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -36,7 +36,7 @@ from vllm.v1.attention.backend import ( AttentionType, CommonAttentionMetadata, ) -from vllm.v1.kv_cache_interface import AttentionSpec +from vllm.v1.kv_cache_interface import AttentionSpec, EncoderOnlyAttentionSpec logger = init_logger(__name__) @@ -90,6 +90,10 @@ class FlexAttentionBackend(AttentionBackend): def get_name() -> str: return "FLEX_ATTENTION" + @classmethod + def supports_non_causal(cls) -> bool: + return True + @classmethod def supports_attn_type(cls, attn_type: str) -> bool: """FlexAttention supports both decoder and encoder-only attention.""" @@ -294,6 +298,12 @@ def causal_mask_mod( return q_idx >= kv_idx +def bidirectional_mask_mod( + b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor +): + return q_idx >= 0 + + # Type alias for the block sparsity hint callable signature. _block_sparsity_hint_signature = Callable[ [torch.Tensor, torch.Tensor, int], torch.Tensor @@ -364,6 +374,7 @@ class FlexAttentionMetadata: block_mask: BlockMask | None = None score_mod: _score_mod_signature | None = None logical_mask_mod: _mask_mod_signature = causal_mask_mod + uses_paged_kv: bool = True doc_ids: torch.Tensor | None = None direct_build: bool = True q_block_size: int = 16 @@ -497,7 +508,7 @@ class FlexAttentionMetadata: False, ) - return final_mask_mod if self.causal else sliding_window_mask_mod + return final_mask_mod if self.uses_paged_kv else sliding_window_mask_mod def get_prefix_lm_mask_mod(self) -> _mask_mod_signature: """Creates the prefix LM mask_mod function for FlexAttention.""" @@ -541,8 +552,7 @@ class FlexAttentionMetadata: def get_mask_mod(self): # Stage-1: initialize the base mask_mod # (causal mask for decoder or bidirectional mask for encoder) - has_custom_mask = self.logical_mask_mod is not causal_mask_mod - if self.causal or has_custom_mask: + if self.uses_paged_kv: mask_mod = self.get_paged_mask_mod() else: mask_mod = self.get_bidirectional_mask_mod() @@ -595,7 +605,7 @@ class FlexAttentionMetadata: return transformed_score_mod def _build_block_mask_direct(self) -> BlockMask: - """Direct block mask construction for standard causal attention. + """Direct block mask construction for paged KV cache attention. This method constructs the block mask directly using BlockMask.from_kv_blocks which is much more efficient than the @@ -693,7 +703,9 @@ class FlexAttentionMetadata: def build_block_mask(self) -> BlockMask: mask_mod = self.get_mask_mod() - kv_len = self.total_cache_tokens if self.causal else self.num_actual_tokens + kv_len = ( + self.total_cache_tokens if self.uses_paged_kv else self.num_actual_tokens + ) return create_block_mask_compiled( mask_mod, None, @@ -842,8 +854,16 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat offset_tensor = common_attn_metadata.compute_num_computed_tokens() offset_tensor = copy_to_persistent(self.persistent_offset_tensor, offset_tensor) + uses_paged_kv = not isinstance(self.kv_cache_spec, EncoderOnlyAttentionSpec) + logical_mask_mod = ( + bidirectional_mask_mod + if uses_paged_kv and not common_attn_metadata.causal + else causal_mask_mod + ) + out = FlexAttentionMetadata( causal=common_attn_metadata.causal, + logical_mask_mod=logical_mask_mod, num_actual_tokens=num_actual_tokens, max_query_len=max_query_len, query_start_loc=query_start_loc, @@ -863,10 +883,11 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat total_cache_tokens=total_cache_tokens, decode_offset=offset_tensor, num_blocks_per_seq=num_blocks_per_seq, + uses_paged_kv=uses_paged_kv, # FIXME(Isotr0py): direct build has issue to build bidirectional # attention block mask for encoder-only models, disable it temporarily. # see: https://github.com/vllm-project/vllm/pull/27329#issuecomment-3431484053 - direct_build=(self.direct_build and common_attn_metadata.causal), + direct_build=self.direct_build and uses_paged_kv, q_block_size=self.q_block_size, kv_block_size=self.kv_block_size, persistent_kv_indices=self.persistent_kv_indices, @@ -1055,9 +1076,7 @@ class FlexAttentionImpl(AttentionImpl): else: attn_metadata.block_mask = attn_metadata.build_block_mask() - if not attn_metadata.causal: - assert self.attn_type == AttentionType.ENCODER_ONLY - + if self.attn_type == AttentionType.ENCODER_ONLY: query, key_tensor, value_tensor = map( lambda x: self.view_as_4d(x).permute(0, 2, 1, 3), (query, key, value), From cfa49213d778f56364ea1312fa6b6b61a6d386ef Mon Sep 17 00:00:00 2001 From: Doug Smith Date: Wed, 22 Apr 2026 16:35:00 -0400 Subject: [PATCH 045/153] [Bugfix][Parser] Fix Mistral pre-v11 tool parser failing on trailing model output (#40531) Signed-off-by: dougbtv Signed-off-by: Doug Smith Co-authored-by: Claude Opus 4.6 Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../tool_parsers/test_mistral_tool_parser.py | 58 ++++++++++++++++--- vllm/tool_parsers/mistral_tool_parser.py | 26 ++++++--- 2 files changed, 66 insertions(+), 18 deletions(-) diff --git a/tests/tool_parsers/test_mistral_tool_parser.py b/tests/tool_parsers/test_mistral_tool_parser.py index 473eb716266..42e8cf138b9 100644 --- a/tests/tool_parsers/test_mistral_tool_parser.py +++ b/tests/tool_parsers/test_mistral_tool_parser.py @@ -24,7 +24,6 @@ from mistral_common.protocol.instruct.tool_calls import ( ToolChoiceEnum as MistralToolChoiceEnum, ) from partial_json_parser.core.options import Allow -from pydantic import ValidationError from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, @@ -250,6 +249,7 @@ def test_extract_tool_calls_no_tools(parser_fixture, request): "argument_before_name_and_name_in_argument", "multiple_tools", "content_before_tool", + "trailing_data_after_json", ], argnames=["model_output", "expected_tool_calls", "expected_content"], argvalues=[ @@ -338,6 +338,24 @@ def test_extract_tool_calls_no_tools(parser_fixture, request): ], "Hello", ), + ( + """[TOOL_CALLS] [{"name": "get_current_weather", "arguments":{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\nextra trailing data""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + { + "city": "Dallas", + "state": "TX", + "unit": "fahrenheit", + } + ), + ) + ) + ], + None, + ), ], ) def test_extract_tool_calls_pre_v11_tokenizer( @@ -366,19 +384,22 @@ def test_extract_tool_calls_pre_v11_multiple_bot_tokens_raises( ) -def test_extract_tool_calls_pre_v11_regex_fallback_raises( +def test_extract_tool_calls_pre_v11_regex_fallback( mistral_pre_v11_tool_parser, ): - """The regex fallback path finds valid JSON but does not re-serialize - the `arguments` dict to a string, causing a Pydantic - `ValidationError` when constructing `FunctionCall`.""" + """The regex fallback path finds valid JSON via regex when the primary + raw_decode fails on leading junk. It should re-serialize arguments + and return a valid tool call.""" model_output = ( '[TOOL_CALLS] junk [{"name": "add", "arguments":{"a": 1, "b": 2}}] trail' ) - with pytest.raises(ValidationError): - mistral_pre_v11_tool_parser.extract_tool_calls( - model_output, request=_DUMMY_REQUEST - ) + result = mistral_pre_v11_tool_parser.extract_tool_calls( + model_output, request=_DUMMY_REQUEST + ) + assert result.tools_called + assert len(result.tool_calls) == 1 + assert result.tool_calls[0].function.name == "add" + assert result.tool_calls[0].function.arguments == json.dumps({"a": 1, "b": 2}) def test_extract_tool_calls_pre_v11_regex_fallback_fails( @@ -579,6 +600,7 @@ def _test_extract_tool_calls_streaming( "argument_before_name", "argument_before_name_and_name_in_argument", "multiple_tools", + "trailing_data_after_json", ], argnames=["model_output", "expected_tool_calls", "expected_content"], argvalues=[ @@ -668,6 +690,24 @@ def _test_extract_tool_calls_streaming( ], "", ), + ( + """[TOOL_CALLS] [{"name": "get_current_weather", "arguments":{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}}]\nextra trailing data""", # noqa: E501 + [ + ToolCall( + function=FunctionCall( + name="get_current_weather", + arguments=json.dumps( + { + "city": "Dallas", + "state": "TX", + "unit": "fahrenheit", + } + ), + ) + ) + ], + "\nextra trailing data", + ), ], ) def test_extract_tool_calls_streaming_pre_v11_tokenizer( diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py index 1d2613104fc..5170f6eb097 100644 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ b/vllm/tool_parsers/mistral_tool_parser.py @@ -479,21 +479,28 @@ class MistralToolParser(ToolParser): ) stringified_tool_calls = raw_tool_calls[0].strip() try: - tool_calls = json.loads(stringified_tool_calls) + # Use raw_decode to parse the first valid JSON value, + # ignoring trailing tokens the model may emit after + # the tool call array. + tool_calls, _ = json.JSONDecoder().raw_decode(stringified_tool_calls) except json.JSONDecodeError: - # use a regex to find the part corresponding to the tool call. - # NOTE: This use case should not happen if the model is trained - # correctly. It's an easy possible fix so it's included, but - # can be brittle for very complex / highly nested tool calls try: raw_tool_call = self.tool_call_regex.findall( stringified_tool_calls )[0] tool_calls = json.loads(raw_tool_call) + tool_calls = [ + { + "name": tool_call["name"], + "arguments": json.dumps( + tool_call.get("arguments", {}), + ensure_ascii=False, + ), + } + for tool_call in tool_calls + ] except (IndexError, json.JSONDecodeError): logger.exception("Error in extracting tool call from response.") - # If raw decoding and decoding post regex rule fails, then just - # return content. return ExtractedToolCallInformation( tools_called=False, tool_calls=[], @@ -504,7 +511,8 @@ class MistralToolParser(ToolParser): { "name": tool_call["name"], "arguments": json.dumps( - tool_call["arguments"], ensure_ascii=False + tool_call.get("arguments", {}), + ensure_ascii=False, ), } for tool_call in tool_calls @@ -515,7 +523,7 @@ class MistralToolParser(ToolParser): type="function", function=FunctionCall( name=tool_call["name"], - arguments=tool_call["arguments"], + arguments=tool_call.get("arguments", "{}"), ), ) for tool_call in tool_calls From 8f87eb4622cf7516ce34c1f9703fee43fb0101e1 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Wed, 22 Apr 2026 16:42:43 -0400 Subject: [PATCH 046/153] [Refactor] Clean up log once `scope="local"` (#40540) Signed-off-by: yewentao256 Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- vllm/compilation/backends.py | 16 +++------------- vllm/compilation/decorators.py | 1 - vllm/compilation/monitor.py | 3 +-- vllm/config/scheduler.py | 1 - vllm/config/vllm.py | 11 +---------- vllm/distributed/device_communicators/pynccl.py | 4 +--- vllm/engine/arg_utils.py | 3 --- vllm/lora/model_manager.py | 1 - .../layers/attention/attention.py | 1 - .../layers/attention/mla_attention.py | 17 +++++------------ .../layers/attention/mm_encoder_attention.py | 4 +--- vllm/model_executor/layers/batch_invariant.py | 2 +- .../fused_moe/experts/batched_deep_gemm_moe.py | 1 - .../layers/fused_moe/fused_moe.py | 1 - .../fused_moe/nixl_ep_prepare_finalize.py | 1 - .../layers/fused_moe/oracle/fp8.py | 14 ++++++-------- .../layers/fused_moe/oracle/int8.py | 8 ++++---- .../layers/fused_moe/oracle/mxfp4.py | 8 ++++---- .../layers/fused_moe/oracle/nvfp4.py | 10 ++++------ .../layers/fused_moe/oracle/unquantized.py | 14 ++++++-------- .../fused_moe/prepare_finalize/deepep_ll.py | 1 - .../layers/fused_moe/runner/shared_experts.py | 6 ++---- .../layers/mamba/gdn_linear_attn.py | 5 ++--- .../compressed_tensors_moe_w4a4_mxfp4.py | 4 ++-- .../compressed_tensors_moe_wna16_marlin.py | 1 - .../model_executor/layers/quantization/mxfp4.py | 2 -- .../quantization/utils/flashinfer_utils.py | 4 ---- vllm/model_executor/model_loader/base_loader.py | 1 - .../model_loader/default_loader.py | 1 - .../model_loader/sharded_state_loader.py | 1 - vllm/model_executor/offloader/base.py | 6 ++---- vllm/platforms/cuda.py | 2 -- vllm/profiler/wrapper.py | 12 ++++-------- vllm/utils/deep_gemm.py | 8 +++----- vllm/utils/import_utils.py | 2 -- vllm/v1/attention/backends/fa_utils.py | 1 - vllm/v1/attention/backends/flash_attn.py | 1 - vllm/v1/core/kv_cache_utils.py | 6 +----- vllm/v1/engine/core.py | 3 --- vllm/v1/executor/multiproc_executor.py | 1 - vllm/v1/worker/dp_utils.py | 1 - vllm/v1/worker/gpu/eplb_utils.py | 4 +--- vllm/v1/worker/gpu_model_runner.py | 7 +------ vllm/v1/worker/gpu_worker.py | 3 +-- 44 files changed, 56 insertions(+), 148 deletions(-) diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index c3900ffc67d..501436275a0 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -292,7 +292,6 @@ class CompilerManager: "from the cache, took %.3f s", str(compile_range), elapsed, - scope="local", ) return compiled_graph @@ -377,7 +376,6 @@ class CompilerManager: logger.info_once( "Cache the graph of compile range %s for later use", str(compile_range), - scope="local", ) logger.debug_once( "Store the %s-th graph for compile range%s from %s via handle %s", @@ -385,7 +383,6 @@ class CompilerManager: str(compile_range), self.compiler.name, handle, - scope="local", ) # after compiling the last graph, record the end time @@ -399,7 +396,6 @@ class CompilerManager: "Compiling a graph for compile range %s takes %.2f s", str(compile_range), elapsed, - scope="local", ) return compiled_graph @@ -1072,12 +1068,11 @@ class VllmBackend: disable_cache = disable_cache or is_ngram_gpu_enabled if disable_cache: - logger.info_once("vLLM's torch.compile cache is disabled.", scope="local") + logger.info_once("vLLM's torch.compile cache is disabled.") else: logger.info_once( "Using cache directory: %s for vLLM's torch.compile", local_cache_dir, - scope="local", ) self.compiler_manager.initialize_cache( @@ -1134,9 +1129,7 @@ class VllmBackend: from .monitor import torch_compile_start_time dynamo_time = time.perf_counter() - torch_compile_start_time - logger.info_once( - "Dynamo bytecode transform time: %.2f s", dynamo_time, scope="local" - ) + logger.info_once("Dynamo bytecode transform time: %.2f s", dynamo_time) if self.is_encoder: self.compilation_config.encoder_compilation_time += dynamo_time else: @@ -1215,7 +1208,6 @@ class VllmBackend: logger.info_once( "Saved compiler manager cache in %.2f seconds.", elapsed, - scope="local", ) from torch._guards import detect_fake_mode @@ -1254,9 +1246,7 @@ class VllmBackend: with open(graph_path, "w") as f: f.write(src) - logger.debug_once( - "Computation graph saved to %s", graph_path, scope="local" - ) + logger.debug_once("Computation graph saved to %s", graph_path) self._called = True graph_to_serialize = ( diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 79daf00de66..a9ecb321cb3 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -665,7 +665,6 @@ def _support_torch_compile( logger.info_once( "saved AOT compiled function to %s", self._aot_compilation_path, - scope="local", ) except Exception as e: logger.warning( diff --git a/vllm/compilation/monitor.py b/vllm/compilation/monitor.py index f584f526f08..a15f1d5fe73 100644 --- a/vllm/compilation/monitor.py +++ b/vllm/compilation/monitor.py @@ -45,7 +45,7 @@ def monitor_torch_compile( else: total_compile_time = time.perf_counter() - torch_compile_start_time if compilation_config.mode == CompilationMode.VLLM_COMPILE: - logger.info_once(message, total_compile_time, scope="local") + logger.info_once(message, total_compile_time) finally: if depyf_cm is not None: try: @@ -76,7 +76,6 @@ def monitor_profiling_run() -> Generator[None, None, None]: logger.info_once( "Initial profiling/warmup run took %.2f s", elapsed, - scope="local", ) diff --git a/vllm/config/scheduler.py b/vllm/config/scheduler.py index b9a48144ded..fb6951ea7dd 100644 --- a/vllm/config/scheduler.py +++ b/vllm/config/scheduler.py @@ -239,7 +239,6 @@ class SchedulerConfig: logger.info_once( "Chunked prefill is enabled with max_num_batched_tokens=%d.", self.max_num_batched_tokens, - scope="local", ) if self.max_num_partial_prefills > 1: diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 26506642561..0726e93d2fe 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -716,9 +716,7 @@ class VllmConfig: self.instance_id = f"{time.time_ns()}" if self.performance_mode != "balanced": - logger.info_once( - "Performance mode set to '%s'.", self.performance_mode, scope="local" - ) + logger.info_once("Performance mode set to '%s'.", self.performance_mode) self.try_verify_and_update_config() @@ -818,7 +816,6 @@ class VllmConfig: "Async scheduling not supported with %s-based " "speculative decoding and will be disabled.", self.speculative_config.method, - scope="local", ) self.scheduler_config.async_scheduling = False elif ( @@ -828,7 +825,6 @@ class VllmConfig: logger.warning_once( "Async scheduling is not compatible with " "disable_padded_drafter_batch=True and will be disabled.", - scope="local", ) self.scheduler_config.async_scheduling = False elif not executor_supports_async_sched: @@ -836,7 +832,6 @@ class VllmConfig: "Async scheduling will be disabled because it is not supported " "with the `%s` distributed executor backend. ", executor_backend, - scope="local", ) self.scheduler_config.async_scheduling = False else: @@ -855,7 +850,6 @@ class VllmConfig: logger.info_once( "Disabling NCCL for DP synchronization " "when using async scheduling.", - scope="local", ) self.parallel_config.disable_nccl_for_dp_synchronization = True else: @@ -870,7 +864,6 @@ class VllmConfig: logger.warning_once( "Disabling cascade attention (not yet compatible with " "async speculative decoding).", - scope="local", ) self.model_config.disable_cascade_attn = True @@ -1231,7 +1224,6 @@ class VllmConfig: self.model_config.disable_cascade_attn = True logger.warning_once( "Disabling cascade attention when VLLM_BATCH_INVARIANT is enabled.", - scope="local", ) if self.parallel_config.use_ubatching: @@ -1418,7 +1410,6 @@ class VllmConfig: " performance. Consider increasing max_num_batched_tokens to" " accommodate the additional draft token slots, or decrease" " num_speculative_tokens or max_num_seqs.", - scope="local", ) max_num_scheduled_tokens = self.scheduler_config.max_num_scheduled_tokens diff --git a/vllm/distributed/device_communicators/pynccl.py b/vllm/distributed/device_communicators/pynccl.py index 6ac3b9ea3c7..990c808a983 100644 --- a/vllm/distributed/device_communicators/pynccl.py +++ b/vllm/distributed/device_communicators/pynccl.py @@ -108,9 +108,7 @@ class PyNcclCommunicator: if self.rank == 0: # get the unique id from NCCL self.unique_id = self.nccl.ncclGetUniqueId() - logger.info_once( - "vLLM is using nccl==%s", self.nccl.ncclGetVersion(), scope="local" - ) + logger.info_once("vLLM is using nccl==%s", self.nccl.ncclGetVersion()) else: # construct an empty unique id self.unique_id = ncclUniqueId() diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 7028b12dab3..e6528849b21 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -2254,7 +2254,6 @@ class EngineArgs: "This model does not officially support disabling chunked prefill. " "Disabling this manually may cause the engine to crash " "or produce incorrect outputs.", - scope="local", ) elif ( model_config.runner_type == "pooling" @@ -2265,7 +2264,6 @@ class EngineArgs: "This model does not officially support chunked prefill. " "Enabling this manually may cause the engine to crash " "or produce incorrect outputs.", - scope="local", ) if self.enable_prefix_caching is None: @@ -2284,7 +2282,6 @@ class EngineArgs: "This model does not officially support prefix caching. " "Enabling this manually may cause the engine to crash " "or produce incorrect outputs.", - scope="local", ) # Disable chunked prefill and prefix caching for: diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index 3b58031dcba..52ff8ebc91f 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -387,7 +387,6 @@ class LoRAModelManager: "LoRA is not supported for non-gated MoE gate module." " %s will be ignored.", module_name, - scope="local", ) continue diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index d229e32be75..61fb687e463 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -332,7 +332,6 @@ class Attention(nn.Module, AttentionLayerBase): logger.warning_once( "Disabling prefix caching for FLASHINFER/TRITON_MLA " "with batch invariance, as it is not yet supported.", - scope="local", ) cache_config.enable_prefix_caching = False diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 9d6ae6bf601..5c7dc60fe15 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -427,7 +427,6 @@ class MLAAttention(nn.Module, AttentionLayerBase): logger.warning_once( "Disabling prefix caching for TRITON_MLA / FLASHINFER " "with batch invariance, as it is not yet supported.", - scope="local", ) cache_config.enable_prefix_caching = False @@ -1523,9 +1522,7 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): if use_fp8: fp8_dtype = current_platform.fp8_dtype() - logger.info_once( - "FP8 prefill attention enabled: query data type is FP8", scope="local" - ) + logger.info_once("FP8 prefill attention enabled: query data type is FP8") return fp8_dtype elif vllm_config.attention_config.use_prefill_query_quantization: logger.info_once( @@ -1533,7 +1530,6 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): " use_prefill_query_quantization is enabled. Please" " ensure that --kv-cache-dtype is set to fp8 and your prefill" " backend is compatible with FP8 attention.", - scope="local", ) return model_dtype elif ( @@ -1547,7 +1543,6 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): "prefill latency. To enable, add: " '--attention-config \'{"use_prefill_query_quantization"' ": true}'", - scope="local", ) return model_dtype @@ -2225,21 +2220,19 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): ) if use_trtllm_ragged_deepseek_prefill(): - logger.info_once( - "Using TRT-LLM ragged DeepSeek prefill for MLA", scope="local" - ) + logger.info_once("Using TRT-LLM ragged DeepSeek prefill for MLA") self._run_prefill_context_chunk = ( self._run_prefill_context_chunk_trtllm_ragged ) self._run_prefill_new_tokens = self._run_prefill_new_tokens_trtllm_ragged self._pad_v = False elif use_flashinfer_prefill(): - logger.info_once("Using FlashInfer prefill for MLA", scope="local") + logger.info_once("Using FlashInfer prefill for MLA") self._run_prefill_context_chunk = self._run_prefill_context_chunk_fi self._run_prefill_new_tokens = self._run_prefill_new_tokens_fi self._pad_v = False elif use_cudnn_prefill(): - logger.info_once("Using CUDNN prefill for MLA", scope="local") + logger.info_once("Using CUDNN prefill for MLA") self._run_prefill_context_chunk = self._run_prefill_context_chunk_cudnn self._run_prefill_new_tokens = self._run_prefill_new_tokens_cudnn self._pad_v = False @@ -2250,7 +2243,7 @@ class MLACommonImpl(MLAAttentionImpl[M], Generic[M]): "available. Please install flash_attn or use " "--attention-backend ROCM_AITER_MLA." ) - logger.info_once("Using FlashAttention prefill for MLA", scope="local") + logger.info_once("Using FlashAttention prefill for MLA") self._run_prefill_context_chunk = self._run_prefill_context_chunk_fa self._run_prefill_new_tokens = self._run_prefill_new_tokens_fa diff --git a/vllm/model_executor/layers/attention/mm_encoder_attention.py b/vllm/model_executor/layers/attention/mm_encoder_attention.py index 6755e9af9e6..46d461c38b3 100644 --- a/vllm/model_executor/layers/attention/mm_encoder_attention.py +++ b/vllm/model_executor/layers/attention/mm_encoder_attention.py @@ -227,9 +227,7 @@ class MMEncoderAttention(CustomOp): if self.attn_backend == AttentionBackendEnum.FLASHINFER: _get_flashinfer_workspace_buffer() - logger.info_once( - f"Using {self.attn_backend} for MMEncoderAttention.", scope="local" - ) + logger.info_once(f"Using {self.attn_backend} for MMEncoderAttention.") @classmethod def enabled(cls) -> bool: diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 4a88421e3b5..152333beecb 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -1020,7 +1020,7 @@ def override_envs_for_invariance( "You are using a non-decode-invariant form of batch invariance. " "This will not be invariant between prefill and decode." ) - logger.warning_once(warning, scope="local") + logger.warning_once(warning) os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0" os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" diff --git a/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py b/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py index fad39b3e9d4..7bd383b9cda 100644 --- a/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/batched_deep_gemm_moe.py @@ -369,7 +369,6 @@ class BatchedDeepGemmExperts(mk.FusedMoEExpertsModular): logger.warning_once( "DPMetadata unavailable. Defaulting expected_m to " f"{max_tokens_per_expert}.", - scope="local", ) return max_tokens_per_expert diff --git a/vllm/model_executor/layers/fused_moe/fused_moe.py b/vllm/model_executor/layers/fused_moe/fused_moe.py index bf083eb9b55..cf53907e2c3 100644 --- a/vllm/model_executor/layers/fused_moe/fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_moe.py @@ -1091,7 +1091,6 @@ def get_moe_configs( "Using default MoE config. Performance might be sub-optimal! " "Config file not found at %s", ", ".join(config_file_paths), - scope="local", ) return None diff --git a/vllm/model_executor/layers/fused_moe/nixl_ep_prepare_finalize.py b/vllm/model_executor/layers/fused_moe/nixl_ep_prepare_finalize.py index dbc54e2c9de..a1068a75242 100644 --- a/vllm/model_executor/layers/fused_moe/nixl_ep_prepare_finalize.py +++ b/vllm/model_executor/layers/fused_moe/nixl_ep_prepare_finalize.py @@ -123,7 +123,6 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): "NixlEPPrepareAndFinalize is setup to dispatch raw/unquantized " f"activations despite ({fused_experts.__class__.__name__}) being able " "to support quantized activations.", - scope="local", ) def num_dispatchers(self) -> int: diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 4420bb38731..584c2bf7928 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -266,7 +266,7 @@ def select_fp8_moe_backend( k_cls, config, weight_key, activation_key, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) @@ -337,12 +337,10 @@ def select_fp8_moe_backend( ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once( - _make_log_unsupported(backend, reason), scope="local" - ) + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no " @@ -396,10 +394,10 @@ def select_fp8_moe_backend( activation_format, ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + logger.debug_once(_make_log_unsupported(backend, reason)) # TODO(rob): per discussion with TPU team, we need a way to register # MoE backends by OOT plugins, rather than having an explicit list @@ -580,7 +578,7 @@ def make_fp8_moe_kernel( ) assert prepare_finalize is not None - logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local") + logger.info_once("Using %s", prepare_finalize.__class__.__name__) # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: diff --git a/vllm/model_executor/layers/fused_moe/oracle/int8.py b/vllm/model_executor/layers/fused_moe/oracle/int8.py index efa2792b420..cdb1be108b5 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/int8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/int8.py @@ -117,7 +117,7 @@ def select_int8_moe_backend( k_cls, config, weight_key, activation_key, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) @@ -138,10 +138,10 @@ def select_int8_moe_backend( activation_format, ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "No Int8 MoE backend supports the deployment configuration." @@ -193,7 +193,7 @@ def make_int8_moe_kernel( ) assert prepare_finalize is not None - logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local") + logger.info_once("Using %s", prepare_finalize.__class__.__name__) # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 13d7a902c30..6306d0e2e9d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -269,7 +269,7 @@ def select_gpt_oss_mxfp4_moe_backend( k_cls, config, weight_key, activation_key, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) @@ -363,10 +363,10 @@ def select_gpt_oss_mxfp4_moe_backend( k_cls, config, kMxfp4Static, activation_key, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + logger.debug_once(_make_log_unsupported(backend, reason)) if current_platform.is_xpu(): backend = Mxfp4MoeBackend.XPU @@ -861,7 +861,7 @@ def make_mxfp4_moe_kernel( ) assert prepare_finalize is not None - logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local") + logger.info_once("Using %s", prepare_finalize.__class__.__name__) # Create Experts. if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 6d0b66cb9f5..724f6d5399b 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -252,12 +252,10 @@ def select_nvfp4_moe_backend( activation_format, ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once( - _make_log_unsupported(backend, reason), scope="local" - ) + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no " @@ -282,10 +280,10 @@ def select_nvfp4_moe_backend( ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "No NvFp4 MoE backend supports the deployment configuration." diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index af7cb7baf96..cdfd6bb8c02 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -210,7 +210,7 @@ def select_unquantized_moe_backend( k_cls, config, None, None, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls raise ValueError(_make_log_unsupported(backend, reason)) @@ -271,12 +271,10 @@ def select_unquantized_moe_backend( k_cls, moe_config, None, None, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls else: - logger.debug_once( - _make_log_unsupported(backend, reason), scope="local" - ) + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no " @@ -298,10 +296,10 @@ def select_unquantized_moe_backend( k_cls, moe_config, None, None, activation_format ) if supported: - logger.info_once(_make_log_backend(backend), scope="local") + logger.info_once(_make_log_backend(backend)) return backend, k_cls - logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + logger.debug_once(_make_log_unsupported(backend, reason)) raise NotImplementedError( "No Unquantized MoE backend supports the deployment configuration." @@ -355,7 +353,7 @@ def make_unquantized_moe_kernel( ) assert prepare_finalize is not None - logger.info_once("Using %s", prepare_finalize.__class__.__name__, scope="local") + logger.info_once("Using %s", prepare_finalize.__class__.__name__) # Create Experts if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_ll.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_ll.py index 0c6e32ae4a5..058d09d23bf 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_ll.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/deepep_ll.py @@ -135,7 +135,6 @@ class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): "DeepEPLLPrepareAndFinalize is setup to dispatch raw/unquantized " f"activations despite ({fused_experts.__class__.__name__}) being able " "to support quantized activations.", - scope="local", ) def num_dispatchers(self) -> int: diff --git a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py index c105badabcb..227014e2397 100644 --- a/vllm/model_executor/layers/fused_moe/runner/shared_experts.py +++ b/vllm/model_executor/layers/fused_moe/runner/shared_experts.py @@ -69,16 +69,14 @@ class SharedExperts: # TODO: Remove this after more extensive testings with TP/DP # and other execution modes if envs.VLLM_DISABLE_SHARED_EXPERTS_STREAM: - logger.debug_once("Disabling MoE shared_experts cuda stream", scope="local") + logger.debug_once("Disabling MoE shared_experts cuda stream") self._stream = None else: # TODO(rob): enable shared expert overlap with non-cuda-alike. # aux_stream() returns None on non-cuda-alike platforms. self._stream = aux_stream() if self._stream is not None: - logger.debug_once( - "Enabled separate cuda stream for MoE shared_experts", scope="local" - ) + logger.debug_once("Enabled separate cuda stream for MoE shared_experts") @property def _disable_shared_experts_overlap(self) -> bool: diff --git a/vllm/model_executor/layers/mamba/gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn_linear_attn.py index c74ca13024a..7a0b54335ba 100644 --- a/vllm/model_executor/layers/mamba/gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn_linear_attn.py @@ -143,15 +143,14 @@ class ChunkGatedDeltaRule(CustomOp): use_flashinfer = supports_flashinfer if use_flashinfer: - logger.info_once("Using FlashInfer GDN prefill kernel", scope="local") + logger.info_once("Using FlashInfer GDN prefill kernel") logger.info_once( "FlashInfer GDN prefill kernel is JIT-compiled; first run may " "take a while to compile. Set `--gdn-prefill-backend triton` to " "avoid JIT compile time.", - scope="local", ) else: - logger.info_once("Using Triton/FLA GDN prefill kernel", scope="local") + logger.info_once("Using Triton/FLA GDN prefill kernel") self._forward_method = ( self.forward_cuda if use_flashinfer else self.forward_native diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py index 57ebb961d48..9d3e0e7a787 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py @@ -44,10 +44,10 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod): self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device() self.experts_cls: type[mk.FusedMoEExperts] if self.use_cutlass_mxfp4: - logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE", scope="local") + logger.info_once("Using CutlassExpertsMxfp4 for MXFP4 MoE") self.experts_cls = CutlassExpertsMxfp4 else: - logger.info_once("Using MarlinExperts for MXFP4 MoE", scope="local") + logger.info_once("Using MarlinExperts for MXFP4 MoE") self.experts_cls = MarlinExperts def create_weights( diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py index 216eed6372a..81b7efaa6d7 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py @@ -87,7 +87,6 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod): logger.info_once( f"Using {self.kernel_backend} backend for WNA16 MoE " f"(group_size={self.group_size}, num_bits={self.num_bits})", - scope="local", ) def get_weight_shape( diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 019bb45d65d..b53c7cc9ac1 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -83,7 +83,6 @@ class Mxfp4Config(QuantizationConfig): logger.debug_once( "MXFP4 linear layer is not implemented - falling back to " "UnquantizedLinearMethod.", - scope="local", ) return UnquantizedLinearMethod() elif isinstance(layer, FusedMoE): @@ -92,7 +91,6 @@ class Mxfp4Config(QuantizationConfig): logger.debug_once( "MXFP4 attention layer is not implemented. " "Skipping quantization for this layer.", - scope="local", ) return None diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 32c7a772f3f..973f759698f 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -111,7 +111,6 @@ def get_flashinfer_moe_backend() -> FlashinferMoeBackend: logger.info_once( "Flashinfer TRTLLM MOE backend is only supported on " "SM100 and later, using CUTLASS backend instead", - scope="local", ) return FlashinferMoeBackend.CUTLASS return backend_map[flashinfer_moe_backend] @@ -239,7 +238,6 @@ def align_fp4_moe_weights_for_fi( "Padding intermediate size from %d to %d for up/down projection weights.", intermediate, padded_intermediate, - scope="local", ) up_mult = 2 if is_act_and_mul else 1 @@ -285,7 +283,6 @@ def align_trtllm_fp4_moe_hidden_dim_for_fi( "performance degradation.", hidden_size, padded_hidden_size, - scope="local", ) padded_w13 = w13.new_zeros((num_experts, gate_up_dim, padded_hidden_size // 2)) @@ -331,7 +328,6 @@ def align_fp8_moe_weights_for_fi( "Padding intermediate size from %d to %d for up/down projection weights.", intermediate, padded_intermediate, - scope="local", ) up_mult = 2 if is_act_and_mul else 1 diff --git a/vllm/model_executor/model_loader/base_loader.py b/vllm/model_executor/model_loader/base_loader.py index d6c38664fde..fb2f77d1b11 100644 --- a/vllm/model_executor/model_loader/base_loader.py +++ b/vllm/model_executor/model_loader/base_loader.py @@ -70,7 +70,6 @@ class BaseModelLoader(ABC): logger.debug_once( "Peak GPU memory after loading weights: %s GiB", format_gib(peak_memory), - scope="local", ) # Process weights into kernel format. Note that when using online diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index 5c9c97f4b64..037195b9063 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -384,7 +384,6 @@ class DefaultModelLoader(BaseModelLoader): logger.info_once( "Loading weights took %.2f seconds", self.counter_after_loading_weights - self.counter_before_loading_weights, - scope="local", ) # We only enable strict check for non-quantized models # that have loaded weights tracking currently. diff --git a/vllm/model_executor/model_loader/sharded_state_loader.py b/vllm/model_executor/model_loader/sharded_state_loader.py index a87731e8bc0..87b4b72db2a 100644 --- a/vllm/model_executor/model_loader/sharded_state_loader.py +++ b/vllm/model_executor/model_loader/sharded_state_loader.py @@ -157,7 +157,6 @@ class ShardedStateLoader(BaseModelLoader): logger.info_once( "Loading weights took %.2f seconds", counter_after_loading_weights - counter_before_loading_weights, - scope="local", ) if state_dict: raise ValueError(f"Missing keys {tuple(state_dict)} in loaded state!") diff --git a/vllm/model_executor/offloader/base.py b/vllm/model_executor/offloader/base.py index b8c1b6cfa48..ceff60cd4cd 100644 --- a/vllm/model_executor/offloader/base.py +++ b/vllm/model_executor/offloader/base.py @@ -118,11 +118,9 @@ def set_offloader(instance: BaseOffloader) -> None: global _instance _instance = instance if isinstance(instance, NoopOffloader): - logger.debug_once( - "Offloader set to NoopOffloader (no offloading).", scope="local" - ) + logger.debug_once("Offloader set to NoopOffloader (no offloading).") else: - logger.info_once("Offloader set to %s", type(instance).__name__, scope="local") + logger.info_once("Offloader set to %s", type(instance).__name__) def create_offloader(offload_config: "OffloadConfig") -> BaseOffloader: diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index d79d3191820..4f9b9d7bf23 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -369,7 +369,6 @@ class CudaPlatformBase(Platform): "Using %s attention backend out of potential backends: %s.", selected_backend.name, "[" + ", ".join(f"'{b[0].name}'" for b in valid_backends_priorities) + "]", - scope="local", ) return selected_backend.get_path() @@ -423,7 +422,6 @@ class CudaPlatformBase(Platform): if is_backend_supported: logger.info_once( f"Using backend {vit_attn_backend} for vit attention", - scope="local", ) return vit_attn_backend except ImportError: diff --git a/vllm/profiler/wrapper.py b/vllm/profiler/wrapper.py index 7cd4d8874df..201b4507849 100644 --- a/vllm/profiler/wrapper.py +++ b/vllm/profiler/wrapper.py @@ -63,7 +63,7 @@ class WorkerProfiler(ABC): """Call _stop with error handling but no safeguards.""" try: self._stop() - logger.info_once("Profiler stopped successfully.", scope="local") + logger.info_once("Profiler stopped successfully.") except Exception as e: logger.warning("Failed to stop profiler: %s", e) self._running = False # Always mark as not running, assume stop worked @@ -93,7 +93,7 @@ class WorkerProfiler(ABC): and self._delay_iters > 0 and self._active_iteration_count == self._delay_iters ): - logger.info_once("Starting profiler after delay...", scope="local") + logger.info_once("Starting profiler after delay...") self._call_start() # Call profiler step for schedule-based profiling @@ -109,9 +109,7 @@ class WorkerProfiler(ABC): # Automatically stop the profiler after max iters # will be marked as not running, but leave as active so that stop # can clean up properly - logger.info_once( - "Max profiling iterations reached. Stopping profiler...", scope="local" - ) + logger.info_once("Max profiling iterations reached. Stopping profiler...") self._call_stop() return @@ -141,7 +139,7 @@ class WorkerProfiler(ABC): def shutdown(self) -> None: """Ensure profiler is stopped when shutting down.""" - logger.info_once("Shutting down profiler", scope="local") + logger.info_once("Shutting down profiler") if self._running: self.stop() @@ -176,7 +174,6 @@ class TorchProfilerWrapper(WorkerProfiler): logger.info_once( "Torch profiling enabled. Traces will be saved to: %s", torch_profiler_trace_dir, - scope="local", ) logger.debug( "Profiler config: record_shapes=%s," @@ -216,7 +213,6 @@ class TorchProfilerWrapper(WorkerProfiler): profiler_config.wait_iterations, profiler_config.warmup_iterations, profiler_config.active_iterations, - scope="local", ) self.profiler = torch.profiler.profile( diff --git a/vllm/utils/deep_gemm.py b/vllm/utils/deep_gemm.py index a2e10ea3951..637e9ec37e0 100644 --- a/vllm/utils/deep_gemm.py +++ b/vllm/utils/deep_gemm.py @@ -106,16 +106,14 @@ def is_deep_gemm_e8m0_used() -> bool: _lazy_init() if _fp8_gemm_nt_impl is None: - logger.info_once( - "DeepGEMM E8M0 disabled: _fp8_gemm_nt_impl not found", scope="local" - ) + logger.info_once("DeepGEMM E8M0 disabled: _fp8_gemm_nt_impl not found") return False if envs.VLLM_USE_DEEP_GEMM_E8M0: - logger.info_once("DeepGEMM E8M0 enabled on current platform.", scope="local") + logger.info_once("DeepGEMM E8M0 enabled on current platform.") return True - logger.info_once("DeepGEMM E8M0 disabled on current configuration.", scope="local") + logger.info_once("DeepGEMM E8M0 disabled on current configuration.") return False diff --git a/vllm/utils/import_utils.py b/vllm/utils/import_utils.py index 31b63d1e6b4..6cf57c6894a 100644 --- a/vllm/utils/import_utils.py +++ b/vllm/utils/import_utils.py @@ -66,14 +66,12 @@ def import_triton_kernels(): logger.debug_once( f"Loading module triton_kernels from {triton_kernels.__file__}.", - scope="local", ) elif _has_module("vllm.third_party.triton_kernels"): import vllm.third_party.triton_kernels as triton_kernels logger.debug_once( f"Loading module triton_kernels from {triton_kernels.__file__}.", - scope="local", ) sys.modules["triton_kernels"] = triton_kernels else: diff --git a/vllm/v1/attention/backends/fa_utils.py b/vllm/v1/attention/backends/fa_utils.py index db8cafeb748..76f98965623 100644 --- a/vllm/v1/attention/backends/fa_utils.py +++ b/vllm/v1/attention/backends/fa_utils.py @@ -118,7 +118,6 @@ def get_flash_attn_version( logger.warning_once( "Cannot use FA version 4 with batch invariance, " "defaulting to FA version 2.", - scope="local", ) fa_version = 2 diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 6af0fa7c496..4926851903b 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -637,7 +637,6 @@ class FlashAttentionImpl(AttentionImpl): logger.info_once( "Using FlashAttention version %s", self.vllm_flash_attn_version, - scope="local", ) # Cache the batch invariant result for use in forward passes self.batch_invariant_enabled = envs.VLLM_BATCH_INVARIANT diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 3f6999b82a4..8f4963fcc87 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1334,7 +1334,7 @@ def _report_kv_cache_config( dcp_size, ) num_tokens_str = f"{num_tokens:,}" - logger.info_once("GPU KV cache size: %s tokens", num_tokens_str, scope="local") + logger.info_once("GPU KV cache size: %s tokens", num_tokens_str) max_model_len_str = f"{vllm_config.model_config.max_model_len:,}" max_concurrency = get_max_concurrency_for_kv_cache_config( vllm_config, kv_cache_config @@ -1343,7 +1343,6 @@ def _report_kv_cache_config( "Maximum concurrency for %s tokens per request: %.2fx", max_model_len_str, max_concurrency, - scope="local", ) @@ -1445,7 +1444,6 @@ def _auto_fit_max_model_len( "Auto-fit max_model_len: attention-free model, " "using derived max_model_len=%d", original_max, - scope="local", ) return @@ -1472,7 +1470,6 @@ def _auto_fit_max_model_len( "Auto-fit max_model_len: full model context length %d fits in " "available GPU memory", original_max, - scope="local", ) else: # Need to reduce max_model_len to fit in memory @@ -1483,7 +1480,6 @@ def _auto_fit_max_model_len( original_max, auto_fit_max, format_gib(limiting_worker_mem), - scope="local", ) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index c2c1a239adb..6bf6910cc6f 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -293,7 +293,6 @@ class EngineCore: compile_time + encoder_compile_time, compile_time, encoder_compile_time, - scope="local", ) elif compile_time > 0: logger.info_once( @@ -301,13 +300,11 @@ class EngineCore: "%.2f s (compilation: %.2f s)", elapsed, compile_time, - scope="local", ) else: logger.info_once( "init engine (profile, create kv cache, warmup model) took %.2f s", elapsed, - scope="local", ) return scheduler_kv_cache_config diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 52969783f09..db21d7cee77 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -1032,7 +1032,6 @@ def set_multiprocessing_worker_envs(): "external environment to tune this value as needed.", current_parallelism, default_omp_num_threads, - scope="local", ) os.environ["OMP_NUM_THREADS"] = str(default_omp_num_threads) torch.set_num_threads(default_omp_num_threads) diff --git a/vllm/v1/worker/dp_utils.py b/vllm/v1/worker/dp_utils.py index 051fe42155e..fbc88f81db8 100644 --- a/vllm/v1/worker/dp_utils.py +++ b/vllm/v1/worker/dp_utils.py @@ -29,7 +29,6 @@ def _get_device_and_group(parallel_config: ParallelConfig): if parallel_config.disable_nccl_for_dp_synchronization: logger.info_once( "Using CPU all reduce to synchronize DP padding between ranks.", - scope="local", ) device = "cpu" group = get_dp_group().cpu_group diff --git a/vllm/v1/worker/gpu/eplb_utils.py b/vllm/v1/worker/gpu/eplb_utils.py index 61d70fafea3..4ffb081ca30 100644 --- a/vllm/v1/worker/gpu/eplb_utils.py +++ b/vllm/v1/worker/gpu/eplb_utils.py @@ -92,9 +92,7 @@ class EPLBController: if not is_mixture_of_experts(model): return False - logger.info_once( - "EPLB is enabled for model %s.", model_config.model, scope="local" - ) + logger.info_once("EPLB is enabled for model %s.", model_config.model) assert self.state is not None self.state.add_model(model, model_config) self._has_registered_models = True diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index b6bc942fc85..386db4fecd4 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -4855,7 +4855,6 @@ class GPUModelRunner( "Model loading took %s GiB memory and %.6f seconds", format_gib(self.model_memory_usage), time_after_load - time_before_load, - scope="local", ) if not load_dummy_weights: prepare_communication_buffer_for_model(self.model) @@ -4989,7 +4988,7 @@ class GPUModelRunner( ) # begin loading weights - logger.info_once("Reloading weights inplace...", scope="local") + logger.info_once("Reloading weights inplace...") if is_checkpoint_format: # load weights from checkpoint/ original model format initialize_layerwise_reload(model) @@ -5001,7 +5000,6 @@ class GPUModelRunner( logger.warning_once( "Reloading with `is_checkpoint_format=True` requires that " "weights be in kernel format and already sharded", - scope="local", ) loaded_weights = set() for name, loaded_weight in weights_iterator: @@ -5015,7 +5013,6 @@ class GPUModelRunner( logger.info_once( "Reloading and processing weights took %.2f seconds", diff_seconds, - scope="local", ) if self.model_config.quantization is None and loaded_weights is not None: weights_not_loaded = weights_to_load - loaded_weights @@ -5802,7 +5799,6 @@ class GPUModelRunner( encoder_budget, max_mm_items_per_batch, dummy_modality, - scope="local", ) # Create dummy batch of multimodal inputs. @@ -6099,7 +6095,6 @@ class GPUModelRunner( "Graph capturing finished in %.0f secs, took %.2f GiB", elapsed_time, cuda_graph_size / (1 << 30), - scope="local", ) return cuda_graph_size diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 98f3212bae0..afbee95c4d7 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -269,7 +269,7 @@ class Worker(WorkerBase): ) if self.use_v2_model_runner: - logger.info_once("Using V2 Model Runner", scope="local") + logger.info_once("Using V2 Model Runner") # Set random seed. set_random_seed(self.model_config.seed) @@ -440,7 +440,6 @@ class Worker(WorkerBase): logger.info_once( "Available KV cache memory: %s GiB", format_gib(self.available_kv_cache_memory_bytes), - scope="local", ) if cudagraph_memory_estimate > 0: From 22fa63cfe849c38bbdb590163043230a1a1a6148 Mon Sep 17 00:00:00 2001 From: Lucas Kabela Date: Wed, 22 Apr 2026 13:48:55 -0700 Subject: [PATCH 047/153] [Bugfix][Torch 2.12] Fix batch_invariant test with allow_override for torch 2.12 upgrade (#40562) Signed-off-by: Lucas Kabela Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/layers/batch_invariant.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 152333beecb..08756ee04de 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -963,8 +963,12 @@ def enable_batch_invariant_mode(): _batch_invariant_LIB.impl("aten::_softmax", softmax_batch_invariant, "CUDA") _batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, "CUDA") - # Also monkeypatch torch.bmm directly as a fallback - _batch_invariant_LIB.impl("aten::bmm", bmm_batch_invariant, "CUDA") + # torch 2.12+ registers a built-in Triton bmm kernel for CUDA + # (torch._native.ops.bmm_outer_product), so we need allow_override + # to replace it at the dispatcher level. + _batch_invariant_LIB.impl( + "aten::bmm", bmm_batch_invariant, "CUDA", allow_override=True + ) _original_torch_bmm = torch.bmm torch.bmm = bmm_batch_invariant From 9c271f94039341101aa144fccc857230a86ac575 Mon Sep 17 00:00:00 2001 From: Honglin Cao Date: Wed, 22 Apr 2026 17:31:00 -0400 Subject: [PATCH 048/153] [gRPC] Add standard gRPC health checking (grpc.health.v1) for Kubernetes native probes (#38016) Signed-off-by: Honglin Cao --- docs/deployment/k8s.md | 44 ++++++++ setup.py | 2 +- tests/entrypoints/test_grpc_health.py | 143 ++++++++++++++++++++++++++ vllm/entrypoints/grpc_server.py | 11 ++ 4 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 tests/entrypoints/test_grpc_health.py diff --git a/docs/deployment/k8s.md b/docs/deployment/k8s.md index dbcb277278c..7a92c99b2c4 100644 --- a/docs/deployment/k8s.md +++ b/docs/deployment/k8s.md @@ -4,6 +4,7 @@ Deploying vLLM on Kubernetes is a scalable and efficient way to serve machine le - [Deployment with CPUs](#deployment-with-cpus) - [Deployment with GPUs](#deployment-with-gpus) +- [Serving with gRPC](#serving-with-grpc) - [Troubleshooting](#troubleshooting) - [Startup Probe or Readiness Probe Failure, container log contains "KeyboardInterrupt: terminated"](#startup-probe-or-readiness-probe-failure-container-log-contains-keyboardinterrupt-terminated) - [Conclusion](#conclusion) @@ -387,6 +388,49 @@ INFO: Uvicorn running on http://0.0.0.0:8000 (Press CTRL+C to quit) If the service is correctly deployed, you should receive a response from the vLLM model. +## Serving with gRPC + +vLLM can serve models over gRPC instead of HTTP by passing the `--grpc` flag. This requires the optional gRPC dependencies: + +```bash +pip install vllm[grpc] +``` + +When using `--grpc`, the server exposes the standard [gRPC Health Checking Protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) (`grpc.health.v1.Health`), which integrates with Kubernetes [native gRPC probes](https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#define-a-grpc-liveness-probe) (available since Kubernetes 1.24). + +To deploy with gRPC, change the `vllm serve` command to include `--grpc` and replace `httpGet` probes with `grpc` probes: + +```yaml +containers: +- name: mistral-7b + image: vllm/vllm-openai:latest + command: ["/bin/sh", "-c"] + args: [ + "pip install vllm[grpc] && vllm serve mistralai/Mistral-7B-Instruct-v0.3 --grpc --port 50051 --trust-remote-code" + ] + ports: + - containerPort: 50051 + livenessProbe: + grpc: + port: 50051 + initialDelaySeconds: 120 + periodSeconds: 10 + readinessProbe: + grpc: + port: 50051 + initialDelaySeconds: 120 + periodSeconds: 5 +``` + +!!! note + The gRPC health service checks the engine status on every probe. If the engine is unhealthy or the server is shutting down, the probe returns `NOT_SERVING`. + +You can also verify the health service manually with `grpcurl`: + +```bash +grpcurl -plaintext localhost:50051 grpc.health.v1.Health/Check +``` + ## Troubleshooting ### Startup Probe or Readiness Probe Failure, container log contains "KeyboardInterrupt: terminated" diff --git a/setup.py b/setup.py index bb2d6ac545d..c05280e40e7 100644 --- a/setup.py +++ b/setup.py @@ -1107,7 +1107,7 @@ setup( # - .buildkite/test-amd.yaml "helion": ["helion==1.0.0"], # Optional deps for gRPC server (vllm serve --grpc) - "grpc": ["smg-grpc-servicer[vllm] >= 0.5.0"], + "grpc": ["smg-grpc-servicer[vllm] >= 0.5.2"], # Optional deps for OpenTelemetry tracing "otel": [ "opentelemetry-sdk>=1.26.0", diff --git a/tests/entrypoints/test_grpc_health.py b/tests/entrypoints/test_grpc_health.py new file mode 100644 index 00000000000..d63b8294c5f --- /dev/null +++ b/tests/entrypoints/test_grpc_health.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +grpc = pytest.importorskip("grpc") +health_pb2 = pytest.importorskip("grpc_health.v1.health_pb2") +VllmHealthServicer = pytest.importorskip( + "smg_grpc_servicer.vllm.health_servicer" +).VllmHealthServicer + +SERVING = health_pb2.HealthCheckResponse.SERVING +NOT_SERVING = health_pb2.HealthCheckResponse.NOT_SERVING +SERVICE_UNKNOWN = health_pb2.HealthCheckResponse.SERVICE_UNKNOWN + + +@pytest.fixture +def async_llm(): + mock = MagicMock() + mock.check_health = AsyncMock() + return mock + + +@pytest.fixture +def context(): + return MagicMock(spec=grpc.aio.ServicerContext) + + +@pytest.fixture +def servicer(async_llm): + return VllmHealthServicer(async_llm) + + +@pytest.fixture +def request_msg(): + msg = MagicMock() + msg.service = "" + return msg + + +# -- Check() tests -- + + +@pytest.mark.asyncio +async def test_check_serving_overall(servicer, request_msg, context, async_llm): + request_msg.service = "" + response = await servicer.Check(request_msg, context) + assert response.status == SERVING + async_llm.check_health.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_check_serving_vllm_service(servicer, request_msg, context, async_llm): + request_msg.service = "vllm.grpc.engine.VllmEngine" + response = await servicer.Check(request_msg, context) + assert response.status == SERVING + async_llm.check_health.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_check_not_serving_engine_errored( + servicer, request_msg, context, async_llm +): + async_llm.check_health = AsyncMock(side_effect=Exception("engine dead")) + request_msg.service = "" + response = await servicer.Check(request_msg, context) + assert response.status == NOT_SERVING + + +@pytest.mark.asyncio +async def test_check_not_serving_shutting_down( + servicer, request_msg, context, async_llm +): + servicer.set_not_serving() + request_msg.service = "" + response = await servicer.Check(request_msg, context) + assert response.status == NOT_SERVING + async_llm.check_health.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_check_unknown_service_status(servicer, request_msg, context): + request_msg.service = "nonexistent.Service" + response = await servicer.Check(request_msg, context) + assert response.status == SERVICE_UNKNOWN + + +@pytest.mark.asyncio +async def test_check_unknown_service_grpc_code(servicer, request_msg, context): + request_msg.service = "fake.Svc" + await servicer.Check(request_msg, context) + context.set_code.assert_called_once_with(grpc.StatusCode.NOT_FOUND) + context.set_details.assert_called_once() + details_arg = context.set_details.call_args[0][0] + assert "fake.Svc" in details_arg + + +@pytest.mark.asyncio +@patch("smg_grpc_servicer.vllm.health_servicer.logger") +async def test_check_logs_exception_on_error( + mock_logger, servicer, request_msg, context, async_llm +): + async_llm.check_health = AsyncMock(side_effect=Exception("engine exploded")) + request_msg.service = "" + await servicer.Check(request_msg, context) + mock_logger.exception.assert_called_once() + log_args = mock_logger.exception.call_args + assert "service" in str(log_args).lower() + + +# -- Watch() tests -- + + +@pytest.mark.asyncio +async def test_watch_yields_serving(servicer, request_msg, context, async_llm): + request_msg.service = "" + watch_iter = servicer.Watch(request_msg, context) + first = await anext(watch_iter.__aiter__()) + assert first.status == SERVING + + +@pytest.mark.asyncio +async def test_watch_yields_not_serving(servicer, request_msg, context, async_llm): + async_llm.check_health = AsyncMock(side_effect=Exception("engine down")) + request_msg.service = "" + watch_iter = servicer.Watch(request_msg, context) + first = await anext(watch_iter.__aiter__()) + assert first.status == NOT_SERVING + + +@pytest.mark.asyncio +async def test_watch_unknown_service(servicer, request_msg, context): + request_msg.service = "fake.Service" + results = [] + async for response in servicer.Watch(request_msg, context): + results.append(response) + assert len(results) == 1 + assert results[0].status == SERVICE_UNKNOWN + # Watch returns SERVICE_UNKNOWN in the response body (not as a gRPC error + # code) so the stream terminates normally -- unlike Check, which sets + # NOT_FOUND on the gRPC context for unknown services. + context.set_code.assert_not_called() diff --git a/vllm/entrypoints/grpc_server.py b/vllm/entrypoints/grpc_server.py index ddd8a5c50e4..b9173b302ca 100644 --- a/vllm/entrypoints/grpc_server.py +++ b/vllm/entrypoints/grpc_server.py @@ -26,8 +26,10 @@ import time try: import grpc + from grpc_health.v1 import health_pb2_grpc from grpc_reflection.v1alpha import reflection from smg_grpc_proto import vllm_engine_pb2, vllm_engine_pb2_grpc + from smg_grpc_servicer.vllm.health_servicer import VllmHealthServicer from smg_grpc_servicer.vllm.servicer import VllmEngineServicer except ImportError as e: raise ImportError( @@ -98,9 +100,14 @@ async def serve_grpc(args: argparse.Namespace): # Add servicer to server vllm_engine_pb2_grpc.add_VllmEngineServicer_to_server(servicer, server) + # Add standard gRPC health service for Kubernetes probes + health_servicer = VllmHealthServicer(async_llm) + health_pb2_grpc.add_HealthServicer_to_server(health_servicer, server) + # Enable reflection for grpcurl and other tools service_names = ( vllm_engine_pb2.DESCRIPTOR.services_by_name["VllmEngine"].full_name, + "grpc.health.v1.Health", reflection.SERVICE_NAME, ) reflection.enable_server_reflection(service_names, server) @@ -147,6 +154,10 @@ async def serve_grpc(args: argparse.Namespace): logger.info("Shutting down vLLM gRPC server...") if stats_task is not None: stats_task.cancel() + try: + health_servicer.set_not_serving() + except Exception: # broad: must not prevent server.stop() / shutdown() + logger.warning("Failed to set health status to NOT_SERVING", exc_info=True) await server.stop(grace=5.0) logger.info("gRPC server stopped") async_llm.shutdown() From b8401a9bf462cbbdcd99a1c6521af8301ad8fa1b Mon Sep 17 00:00:00 2001 From: Lucas Kabela Date: Wed, 22 Apr 2026 15:04:42 -0700 Subject: [PATCH 049/153] [Bugfix] Fix RMS norm + quant fusion on DeepGEMM UE8M0 path for B200 (#40552) Signed-off-by: Lucas Kabela --- tests/compile/passes/test_fusion.py | 21 +++++++++++++++++++++ tests/utils.py | 1 + 2 files changed, 22 insertions(+) diff --git a/tests/compile/passes/test_fusion.py b/tests/compile/passes/test_fusion.py index 79e63efdfe4..32803aad8c1 100644 --- a/tests/compile/passes/test_fusion.py +++ b/tests/compile/passes/test_fusion.py @@ -51,6 +51,7 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import ( ) from vllm.platforms import current_platform from vllm.utils.deep_gemm import ( + is_deep_gemm_e8m0_used, is_deep_gemm_supported, ) @@ -317,6 +318,26 @@ def test_fusion_rmsnorm_quant( ): pytest.skip("Unsupported group shape 64 for CUTLASS/DeepGemm") + # TODO(quant-rms-fusion): DeepGEMM UE8M0 activation quant on B200 lowers + # to a packed int32-scale op (per_token_group_quant_fp8_packed_for_deepgemm), + # but the rms+quant fusion pattern only matches the fp32-scale variant, so + # the fused output gets a mismatched scale layout and produces NaN. Only + # reproduces on bf16 (DeepGEMM UE8M0 on B200 is bf16-only). + # To re-enable: make rms_norm_per_block_quant emit packed UE8M0 scales + # and extend the fusion pattern to rewrite the packed activation quant. + deepgemm_kernels = ( + DeepGemmFp8BlockScaledMMKernel, + FlashInferFp8DeepGEMMDynamicBlockScaledKernel, + ) + if ( + dtype == torch.bfloat16 + and force_kernel in deepgemm_kernels + and is_deep_gemm_e8m0_used() + ): + pytest.skip( + "rms+quant fusion does not yet match the packed UE8M0 DeepGEMM path" + ) + custom_ops = [] if enable_rms_norm_custom_op: custom_ops.append("+rms_norm") diff --git a/tests/utils.py b/tests/utils.py index d35555d37c6..5ccdaa0d64e 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1826,6 +1826,7 @@ class TestFP8Layer(torch.nn.Module): self.weight = torch.rand(weight_shape).to(dtype=FP8_DTYPE) self.input_scale = None self.weight_scale = None + self.weight_block_size = [block_size, block_size] if transpose_weights: self.weight = self.weight.t() else: From ac58e2a1704ba18db3c18748cee2fb6c874496d6 Mon Sep 17 00:00:00 2001 From: Simon Danielsson <70206058+simondanielsson@users.noreply.github.com> Date: Thu, 23 Apr 2026 01:06:31 +0200 Subject: [PATCH 050/153] [Fix][MoRI] Align MoRI-IO message format with P2pNcclConnector and vllm-router (#39565) Signed-off-by: simondanielsson Co-authored-by: Matvei Pashkovskii --- .../moriio_toy_proxy_server.py | 155 ++++++++++-------- .../kv_connector/v1/moriio/moriio_common.py | 77 ++++++++- .../v1/moriio/moriio_connector.py | 60 ++++--- 3 files changed, 202 insertions(+), 90 deletions(-) diff --git a/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py b/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py index e2a0bfc7c9b..de4757f36b7 100644 --- a/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py +++ b/examples/online_serving/disaggregated_serving/moriio_toy_proxy_server.py @@ -10,7 +10,6 @@ import uuid import aiohttp import msgpack -import regex as re import zmq from quart import Quart, Request, make_response, request @@ -25,32 +24,10 @@ decode_instances: list[dict] = [] request_nums = 0 app = Quart(__name__) -IP_PORT_PATTERN = re.compile(r"//(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)") - TRANSFER_TYPE = None -def _append_whole_dict_unique(target_list, data_dict): - new_filtered = {k: v for k, v in data_dict.items() if k != "index"} - for existed in target_list: - existed_filtered = {k: v for k, v in existed.items() if k != "index"} - if existed_filtered == new_filtered: - return False - print("!!APPEND!!", data_dict) - target_list.append(data_dict) - transfer_mode = data_dict.get("transfer_mode", "unknown") - global TRANSFER_TYPE - - if TRANSFER_TYPE is None: - TRANSFER_TYPE = transfer_mode - logger.info("SET TRANSFER TYPE TO %s", TRANSFER_TYPE) - elif transfer_mode != TRANSFER_TYPE: - raise ValueError(f"mismatched transfer mode {TRANSFER_TYPE} vs {transfer_mode}") - - return True - - _list_lock = threading.RLock() @@ -68,23 +45,81 @@ def _listen_for_register(hostname, port): if router_socket in socks: remote_addr, msg = router_socket.recv_multipart() data = msgpack.loads(msg) - if data["type"] == "HELLO": + if data.get("type") == "HELLO": pass - elif ( - data["type"] == "register" - and data["role"] == "P" - and data["request_address"] not in prefill_instances - ): - with _list_lock: - _append_whole_dict_unique(prefill_instances, data) + elif data.get("type") in ("P", "D"): + role = data["type"] + required_keys = { + "http_address", + "zmq_address", + "dp_size", + "tp_size", + "transfer_mode", + } + missing = required_keys - data.keys() + if missing: + logger.error( + "Registration message missing required keys %s; skipping", + missing, + ) + continue + # Derive request_address from http_address + # api path suffix is appended at request time + instance = { + "role": role, + "request_address": f"http://{data['http_address']}/v1", + "http_address": data["http_address"], + "zmq_address": data["zmq_address"], + "dp_size": data["dp_size"], + "tp_size": data["tp_size"], + "transfer_mode": data["transfer_mode"], + } + # zmq_address format: "host:IP,handshake:PORT,notify:PORT" + # Stored verbatim; embedded into the request_id by handle_request. - elif ( - data["type"] == "register" - and data["role"] == "D" - and data["request_address"] not in decode_instances - ): + global TRANSFER_TYPE + transfer_mode = instance["transfer_mode"] + target_list = prefill_instances if role == "P" else decode_instances with _list_lock: - _append_whole_dict_unique(decode_instances, data) + if TRANSFER_TYPE is None: + TRANSFER_TYPE = transfer_mode + logger.info("SET TRANSFER TYPE TO %s", TRANSFER_TYPE) + elif transfer_mode != TRANSFER_TYPE: + logger.error( + "Mismatched transfer mode: expected %s, got %s;" + " skipping registration of %s", + TRANSFER_TYPE, + transfer_mode, + data["http_address"], + ) + continue + existing_idx = next( + ( + idx + for idx, i in enumerate(target_list) + if i.get("http_address") == data["http_address"] + ), + None, + ) + if existing_idx is not None: + target_list[existing_idx] = instance + logger.info( + "Updated existing %s instance: %s", + "Prefill" if role == "P" else "Decode", + instance, + ) + else: + target_list.append(instance) + logger.info( + "Registered %s instance: %s", + "Prefill" if role == "P" else "Decode", + instance, + ) + else: + logger.warning( + "Received message with unrecognized type %r; ignoring", + data.get("type"), + ) def start_service_discovery(hostname, port): @@ -101,7 +136,7 @@ def start_service_discovery(hostname, port): async def send_request_to_prefill( - endpoint, req_data, request_id, d_endpoint, dip, dport, selected_prefill_dp_rank + endpoint, req_data, request_id, selected_prefill_dp_rank ): req_data_copy = req_data @@ -109,12 +144,8 @@ async def send_request_to_prefill( { "do_remote_decode": True, "do_remote_prefill": False, - "remote_handshake_port": d_endpoint["handshake_port"], - "remote_notify_port": d_endpoint["notify_port"], "remote_engine_id": None, "remote_block_ids": None, - "remote_host": dip, - "remote_port": dport, } ) req_data_copy["stream"] = False @@ -197,14 +228,7 @@ async def handle_request(api: str, request: Request): global request_nums request_nums += 1 - def extract_ip_port_fast(url): - match = IP_PORT_PATTERN.search(url) - if not match: - raise ValueError(f"Invalid URL format: {url}") - return match.groups() - req_data = await request.get_json() - request_id = str(uuid.uuid4()) prefill_instance_endpoint = None decode_instance_endpoint = None @@ -230,7 +254,14 @@ async def handle_request(api: str, request: Request): prefill_instance_endpoint["dp_size"], ) - dip, dport = extract_ip_port_fast(decode_instance_endpoint["request_address"]) + # Embed both zmq_addresses in the request_id so the connector can parse + # the peer's host/ports from it, similar to P2P-NCCL + uid = str(uuid.uuid4()).replace("-", "") + request_id = ( + f"___prefill_addr_{prefill_instance_endpoint['zmq_address']}" + f"___decode_addr_{decode_instance_endpoint['zmq_address']}" + f"_{uid}" + ) transfer_id = f"{MoRIIOConstants.TRANSFER_PREFIX}-{str(uuid.uuid4())}" @@ -251,35 +282,30 @@ async def handle_request(api: str, request: Request): prefill_request_url, req_data_to_prefill, request_id, - decode_instance_endpoint, - dip, - dport, selected_prefill_dp_rank, ) ) - ip, port = extract_ip_port_fast(prefill_request_url) req_data["max_tokens"] -= 1 req_data["kv_transfer_params"] = { "do_remote_decode": False, "do_remote_prefill": True, - "remote_handshake_port": prefill_instance_endpoint["handshake_port"], - "remote_notify_port": prefill_instance_endpoint["notify_port"], "remote_engine_id": None, "remote_block_ids": None, - "remote_host": ip, - "remote_port": port, + "transfer_id": transfer_id, } if TRANSFER_TYPE == "READ": # In read mode, prefill and decode are executed serially. prefill_response = await send_prefill_task - req_data["kv_transfer_params"]["remote_engine_id"] = prefill_response[ - "kv_transfer_params" - ]["remote_engine_id"] - req_data["kv_transfer_params"]["remote_block_ids"] = prefill_response[ - "kv_transfer_params" - ]["remote_block_ids"] + prefill_kv = prefill_response["kv_transfer_params"] + req_data["kv_transfer_params"]["remote_engine_id"] = prefill_kv[ + "remote_engine_id" + ] + req_data["kv_transfer_params"]["remote_block_ids"] = prefill_kv[ + "remote_block_ids" + ] + req_data["kv_transfer_params"]["transfer_id"] = prefill_kv["transfer_id"] req_data["kv_transfer_params"]["remote_dp_size"] = prefill_instance_endpoint[ "dp_size" @@ -290,7 +316,6 @@ async def handle_request(api: str, request: Request): if selected_prefill_dp_rank is not None: req_data["kv_transfer_params"]["remote_dp_rank"] = selected_prefill_dp_rank - req_data["kv_transfer_params"]["transfer_id"] = transfer_id decode_request_url = decode_instance_endpoint["request_address"] + api decode_request_task = asyncio.create_task( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py index f3b2ce3b5be..b843c5b5930 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_common.py @@ -8,6 +8,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import msgspec +import regex as re import torch import zmq @@ -239,7 +240,7 @@ class MoRIIOConstants: COMPLETION_PREFIX = "cmpl" TRANSFER_PREFIX = "tx" - PING_INTERVAL = 5 + PING_INTERVAL = 3 MAX_PING_RETRIES = 100 DEFAULT_HANDSHAKE_PORT = "6301" DEFAULT_NOTIFY_PORT = "61005" @@ -247,6 +248,64 @@ class MoRIIOConstants: VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT = 3600 +# The router embeds both zmq_addresses in the request_id (similar to P2pNcclConnector): +# "___prefill_addr_{zmq}___decode_addr_{zmq}_{32-hex-uuid}" +# MoRIIO zmq_address format: "host:IP,handshake:PORT,notify:PORT" +# +# This lets each connector side parse the peer's connection info without +# requiring the router to pass it explicitly in kv_transfer_params. +_PREFILL_ZMQ_RE = re.compile(r"___prefill_addr_(.+?)___decode_addr_") +# vLLM wraps the router's X-Request-Id as "cmpl---" so there may +# be a trailing "--" suffix after the 32-char UUID. Allow it. +_DECODE_ZMQ_RE = re.compile(r"___decode_addr_(.+)_[0-9a-f]{32}(?:-.*)?$") + + +def parse_moriio_zmq_address( + zmq_address: str, +) -> tuple[str, int, int]: + """Parse the MoRI-IO zmq address into its components. + + Parses ``"host:IP,handshake:PORT,notify:PORT"`` into + (host, handshake_port, notify_port). + + Each key-value pair is split on the *first* colon so that IPv6 addresses + (e.g. ``host:::1``) are handled correctly. Raises ``ValueError`` if any + of ``host``, ``handshake``, or ``notify`` keys are absent or if the port + values are non-numeric. + """ + parts: dict[str, str] = {} + for segment in zmq_address.split(","): + key, _, val = segment.partition(":") + parts[key.strip()] = val.strip() + try: + host = parts["host"] + handshake_port = int(parts["handshake"]) + notify_port = int(parts["notify"]) + except (KeyError, ValueError) as e: + raise ValueError( + f"Malformed zmq_address {zmq_address!r}: expected " + f"'host:IP,handshake:PORT,notify:PORT' format" + ) from e + return host, handshake_port, notify_port + + +def get_peer_zmq_from_request_id(request_id: str, is_producer: bool) -> str: + """Extract the *peer's* zmq_address from the vLLM router request_id. + + The producer (prefill) needs the decode's address; the consumer (decode) + needs the prefill's address. + """ + if is_producer: + m = _DECODE_ZMQ_RE.search(request_id) + else: + m = _PREFILL_ZMQ_RE.search(request_id) + if m is None: + raise ValueError( + f"Cannot parse peer zmq_address from request_id: {request_id!r}" + ) + return m.group(1) + + @dataclass class ReqMeta: """Metadata for a single request.""" @@ -286,15 +345,23 @@ class MoRIIOConnectorMetadata(KVConnectorMetadata): write_mode=False, ): transfer_id = kv_transfer_params["transfer_id"] + + # Parse host/ports from the request_id. The router embeds both zmq_addresses + # in the request_id + peer_zmq = get_peer_zmq_from_request_id(request_id, is_producer=write_mode) + remote_host, remote_handshake_port, remote_notify_port = ( + parse_moriio_zmq_address(peer_zmq) + ) + _req = ReqMeta( transfer_id=transfer_id, local_block_ids=local_block_ids, remote_block_ids=kv_transfer_params["remote_block_ids"], remote_engine_id=kv_transfer_params["remote_engine_id"], - remote_host=kv_transfer_params["remote_host"], - remote_port=kv_transfer_params["remote_port"], - remote_handshake_port=kv_transfer_params["remote_handshake_port"], - remote_notify_port=kv_transfer_params["remote_notify_port"], + remote_host=remote_host, + remote_port=remote_handshake_port, + remote_handshake_port=remote_handshake_port, + remote_notify_port=remote_notify_port, tp_size=kv_transfer_params.get("tp_size", 1), remote_dp_size=kv_transfer_params.get("remote_dp_size", 1), ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py index 0fd6d81f23e..15aca3e571c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/moriio/moriio_connector.py @@ -35,8 +35,10 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( TransferId, WriteTask, get_moriio_mode, + get_peer_zmq_from_request_id, get_port_offset, get_role, + parse_moriio_zmq_address, set_role, zmq_ctx, ) @@ -379,13 +381,12 @@ class MoRIIOConnectorScheduler: if params is not None and params.get("do_remote_prefill"): if self.mode == MoRIIOMode.READ: if remote_block_ids := params.get("remote_block_ids"): - if all( - p in params - for p in ("remote_engine_id", "remote_host", "remote_port") - ): - # If remote_blocks and num_external_tokens = 0, we + # remote_engine_id is returned by the prefill's request_finished. + # host/ports come from the request_id (parsed in add_new_req). + if "remote_engine_id" in params: + # If remote_blocks and num_external_tokens = 0, we have # a full prefix cache hit on the D worker. We need to call - # send_notif in _read_blocks to free the memory on the P. + # send_notify in _read_blocks to free the memory on the P. # Get unhashed blocks to pull from remote. local_block_ids = blocks.get_block_ids()[0] @@ -407,22 +408,30 @@ class MoRIIOConnectorScheduler: ) else: + # WRITE mode: prefill scheduler notifies the decode side that + # blocks are ready. Parse the decode's host/notify_port from + # the request_id assert request.kv_transfer_params is not None, ( "kv_transfer_params should not be None" ) remote_dp_rank = request.kv_transfer_params.get("remote_dp_rank", 0) + peer_zmq = get_peer_zmq_from_request_id( + request.request_id, is_producer=True + ) + remote_host, _, remote_notify_port = parse_moriio_zmq_address(peer_zmq) + for tp_index in range(self.tp_size): - target_port = request.kv_transfer_params[ - "remote_notify_port" - ] + get_port_offset(remote_dp_rank, tp_index) + target_port = remote_notify_port + get_port_offset( + remote_dp_rank, tp_index + ) self.send_notify_block( req_id=request.request_id, transfer_id=request.kv_transfer_params["transfer_id"], block_notify_list=blocks.get_block_ids()[0], - host=params.get("remote_host"), + host=remote_host, port=target_port, ) @@ -584,15 +593,15 @@ class MoRIIOConnectorScheduler: + MoRIIOConstants.VLLM_MORI_READ_ABORT_REQUEST_TIMEOUT ) - # If we execute in P-D serial mode, no notification port is needed. + # Return KV transfer params forwarded verbatim to the decode instance by + # the router. return delay_free_blocks, dict( do_remote_prefill=True, do_remote_decode=False, remote_block_ids=computed_block_ids, remote_engine_id=self.engine_id, - remote_host=self.host_ip, - remote_port=self.handshake_port, tp_size=self.vllm_config.parallel_config.tensor_parallel_size, + transfer_id=params["transfer_id"], ) @@ -846,7 +855,15 @@ class MoRIIOConnectorWorker: ] def _ping(self, zmq_context): - http_request_address = f"http://{self.request_address}/v1" + # Use host:port format for http_address (compatible with official router) + http_address = f"{self.request_address}" + # Include host so the router embeds it in the request_id; the connector + # on the other side parses host/ports from there. + zmq_address = ( + f"host:{self.local_ip}," + f"handshake:{self.handshake_port}," + f"notify:{self.notify_port}" + ) role = "P" if self.is_producer else "D" retry_count = 0 @@ -857,14 +874,17 @@ class MoRIIOConnectorWorker: while True: try: data = { - "type": "register", - "role": role, - "index": str(index), - "request_address": http_request_address, - "handshake_port": self.handshake_port, - "notify_port": self.notify_port, + "type": role, # "P" or "D" + "http_address": http_address, + "zmq_address": zmq_address, + # dp_size/tp_size are not used by the official vLLM router + # (routing operates at the http_address level); they are + # consumed only by the toy proxy server. "dp_size": self.moriio_config.dp_size, "tp_size": self.moriio_config.tp_size, + # transfer_mode is included so the router can distinguish + # READ (prefill-then-decode, sequential) from WRITE (concurrent) + # scheduling. "transfer_mode": self.mode.name, } From 0283f303d8c6f19f670306c50dba00e897ae94f9 Mon Sep 17 00:00:00 2001 From: Lucas Kabela Date: Wed, 22 Apr 2026 17:12:08 -0700 Subject: [PATCH 051/153] [BE] Fix compile time message to be consistent (use monitoring) (#40641) Signed-off-by: Lucas Kabela --- vllm/compilation/backends.py | 17 ++++------------- vllm/compilation/decorators.py | 7 +++++-- vllm/compilation/monitor.py | 5 +++++ 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index 501436275a0..8045e296e13 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -283,10 +283,6 @@ class CompilerManager: # after loading the last graph for this shape, record the time. # there can be multiple graphs due to piecewise compilation. elapsed = time.perf_counter() - compilation_start_time - if is_encoder: - compilation_config.encoder_compilation_time += elapsed - else: - compilation_config.compilation_time += elapsed logger.info_once( "Directly load the compiled graph(s) for compile range %s " "from the cache, took %.3f s", @@ -388,10 +384,6 @@ class CompilerManager: # after compiling the last graph, record the end time if graph_index == num_graphs - 1: elapsed = time.perf_counter() - compilation_start_time - if is_encoder: - compilation_config.encoder_compilation_time += elapsed - else: - compilation_config.compilation_time += elapsed logger.info_once( "Compiling a graph for compile range %s takes %.2f s", str(compile_range), @@ -1129,11 +1121,10 @@ class VllmBackend: from .monitor import torch_compile_start_time dynamo_time = time.perf_counter() - torch_compile_start_time - logger.info_once("Dynamo bytecode transform time: %.2f s", dynamo_time) - if self.is_encoder: - self.compilation_config.encoder_compilation_time += dynamo_time - else: - self.compilation_config.compilation_time += dynamo_time + logger.info_once( + "Dynamo bytecode transform time: %.2f s", + dynamo_time, + ) # Record Dynamo time in tracing if available start_time = int(torch_compile_start_time * 1e9) diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index a9ecb321cb3..90b5c0c44ed 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -285,7 +285,7 @@ def _try_load_aot_compiled_fn( Re-raises on failure when ``VLLM_FORCE_AOT_LOAD`` is set. """ try: - with monitor_torch_compile(model.vllm_config): + with monitor_torch_compile(model.vllm_config, is_encoder=model._is_encoder): with ( set_current_vllm_config(model.vllm_config), open(aot_compilation_path, "rb") as f, @@ -617,7 +617,9 @@ def _support_torch_compile( # store the path for saving after warmup self._aot_compilation_path = aot_compilation_path self._aot_cache_dir = cache_dir - with monitor_torch_compile(self.vllm_config): + with monitor_torch_compile( + self.vllm_config, is_encoder=self._is_encoder + ): self.aot_compiled_fn = self.aot_compile(*args, **kwargs) compilation_counter.num_aot_compiles += 1 # All compilation is done at this point, save the @@ -631,6 +633,7 @@ def _support_torch_compile( self.vllm_config, "torch.compile and initial profiling/warmup " "run together took %.2f s in total", + is_encoder=self._is_encoder, ): output = TorchCompileWithNoGuardsWrapper.__call__( self, # type: ignore[arg-type] diff --git a/vllm/compilation/monitor.py b/vllm/compilation/monitor.py index a15f1d5fe73..c23a8f67228 100644 --- a/vllm/compilation/monitor.py +++ b/vllm/compilation/monitor.py @@ -18,6 +18,7 @@ torch_compile_start_time: float = 0.0 def monitor_torch_compile( vllm_config: VllmConfig, message: str = "torch.compile took %.2f s in total", + is_encoder: bool = False, ) -> Generator[None, None, None]: """Context manager that times torch.compile and manages depyf debugging. @@ -45,6 +46,10 @@ def monitor_torch_compile( else: total_compile_time = time.perf_counter() - torch_compile_start_time if compilation_config.mode == CompilationMode.VLLM_COMPILE: + if is_encoder: + compilation_config.encoder_compilation_time += total_compile_time + else: + compilation_config.compilation_time += total_compile_time logger.info_once(message, total_compile_time) finally: if depyf_cm is not None: From ccaf5ffaa3e1fb2a081b2c9e403ac0e4dfc142c8 Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Thu, 23 Apr 2026 10:07:45 +0800 Subject: [PATCH 052/153] [XPU] disable fusion pattern support on XPU platform (#39789) Signed-off-by: Chaojun Zhang Co-authored-by: Kunshang Ji --- vllm/platforms/xpu.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vllm/platforms/xpu.py b/vllm/platforms/xpu.py index d52ba23243f..bd9006f3f8f 100644 --- a/vllm/platforms/xpu.py +++ b/vllm/platforms/xpu.py @@ -213,6 +213,26 @@ class XPUPlatform(Platform): "falling back to PIECEWISE graph mode on XPU platform." ) + # Disable fusion passes not yet supported on XPU. + pass_config = compilation_config.pass_config + fusion_passes_to_disable = { + "enable_sp": "Sequence parallelism", + "fuse_gemm_comms": "Async TP", + "fuse_allreduce_rms": "AllReduce + RMSNorm fusion", + "fuse_norm_quant": "RMSNorm + quant fusion", + "fuse_act_quant": "Activation + quant fusion", + "fuse_attn_quant": "Attention + quant fusion", + "fuse_act_padding": "Activation + padding fusion", + "fuse_rope_kvcache": "RoPE + KV cache fusion", + } + for flag, feature_name in fusion_passes_to_disable.items(): + if getattr(pass_config, flag): + logger.warning( + "Feature %r is not yet supported on XPU and will be disabled.", + feature_name, + ) + setattr(pass_config, flag, False) + # check and update parallel config parallel_config = vllm_config.parallel_config # Only override worker_cls if it's still the default "auto" From fe9c3d6c5f66c873d196800384ed6880687b9e52 Mon Sep 17 00:00:00 2001 From: Julian Huang Date: Thu, 23 Apr 2026 12:35:24 +0800 Subject: [PATCH 053/153] [TurboQuant] enable FA3/FA4 for prefill paths (#40092) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 墨楼 Co-authored-by: 墨楼 Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Codex --- .../gsm8k/configs/Qwen3-4B-TQ-k3v4nc.yaml | 2 +- .../evals/gsm8k/configs/Qwen3-4B-TQ-k8v4.yaml | 2 +- .../evals/gsm8k/configs/Qwen3-4B-TQ-t3nc.yaml | 2 +- .../evals/gsm8k/configs/Qwen3-4B-TQ-t4nc.yaml | 2 +- vllm/v1/attention/backends/flash_attn.py | 9 +++- vllm/v1/attention/backends/turboquant_attn.py | 53 +++++++++++++++---- 6 files changed, 55 insertions(+), 15 deletions(-) diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc.yaml index fedb7416960..b9f9a7944f2 100644 --- a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc.yaml +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k3v4nc.yaml @@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B" accuracy_threshold: 0.78 num_questions: 1319 num_fewshot: 5 -server_args: "--kv-cache-dtype turboquant_k3v4_nc --enforce-eager --max-model-len 4096" +server_args: "--kv-cache-dtype turboquant_k3v4_nc --max-model-len 4096" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4.yaml index 9717333582b..200b570e23d 100644 --- a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4.yaml +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-k8v4.yaml @@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B" accuracy_threshold: 0.80 num_questions: 1319 num_fewshot: 5 -server_args: "--kv-cache-dtype turboquant_k8v4 --enforce-eager --max-model-len 4096" +server_args: "--kv-cache-dtype turboquant_k8v4 --max-model-len 4096" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc.yaml index 8ece1852625..1c833fe7bf2 100644 --- a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc.yaml +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t3nc.yaml @@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B" accuracy_threshold: 0.75 num_questions: 1319 num_fewshot: 5 -server_args: "--kv-cache-dtype turboquant_3bit_nc --enforce-eager --max-model-len 4096" +server_args: "--kv-cache-dtype turboquant_3bit_nc --max-model-len 4096" diff --git a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc.yaml b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc.yaml index 9b3a14f9b95..6a7f82b6609 100644 --- a/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc.yaml +++ b/tests/evals/gsm8k/configs/Qwen3-4B-TQ-t4nc.yaml @@ -2,4 +2,4 @@ model_name: "Qwen/Qwen3-4B" accuracy_threshold: 0.80 num_questions: 1319 num_fewshot: 5 -server_args: "--kv-cache-dtype turboquant_4bit_nc --enforce-eager --max-model-len 4096" +server_args: "--kv-cache-dtype turboquant_4bit_nc --max-model-len 4096" diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 4926851903b..19bcdfdc98e 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -255,11 +255,16 @@ class FlashAttentionMetadata: def _get_sliding_window_configs( vllm_config: VllmConfig, ) -> set[tuple[int, int] | None]: - """Get the set of all sliding window configs used in the model.""" + """Get the set of all sliding window configs used in the model. + + Only inspects FlashAttentionImpl layers. Other backends (e.g. + TurboQuant, MLA) use their own metadata builders and are skipped. + """ sliding_window_configs: set[tuple[int, int] | None] = set() layers = get_layers_from_vllm_config(vllm_config, Attention) for layer in layers.values(): - assert isinstance(layer.impl, FlashAttentionImpl) + if not isinstance(layer.impl, FlashAttentionImpl): + continue sliding_window_configs.add(layer.impl.sliding_window) return sliding_window_configs diff --git a/vllm/v1/attention/backends/turboquant_attn.py b/vllm/v1/attention/backends/turboquant_attn.py index e7baff9d899..a0bcc252d85 100644 --- a/vllm/v1/attention/backends/turboquant_attn.py +++ b/vllm/v1/attention/backends/turboquant_attn.py @@ -39,6 +39,7 @@ from vllm.v1.attention.backend import ( MultipleOf, ) from vllm.v1.attention.backends.fa_utils import ( + get_flash_attn_version, is_flash_attn_varlen_func_available, ) from vllm.v1.attention.backends.utils import split_decodes_and_prefills @@ -271,6 +272,9 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): self._val_data_bytes = math.ceil(head_size * cfg.effective_value_quant_bits / 8) self._n_centroids = cfg.n_centroids if not cfg.key_fp8 else 1 + # Detect flash-attn version (FA2/3/4) for prefill paths. + self.fa_version = get_flash_attn_version(head_size=head_size) + # Fixed NUM_KV_SPLITS (grid dims must be constant for cudagraph, # and benchmarks show no regression vs dynamic in eager mode). vllm_config = get_current_vllm_config() @@ -278,6 +282,43 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): vllm_config.attention_config.tq_max_kv_splits_for_cuda_graph ) + def _flash_attn_varlen( + self, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + cu_seqlens_q: torch.Tensor, + cu_seqlens_k: torch.Tensor, + max_seqlen_q: int, + max_seqlen_k: int, + ) -> torch.Tensor: + # fa_utils.get_flash_attn_version() returns None on backends that + # should not pass an explicit fa_version kwarg. + if self.fa_version is None: + return flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + softmax_scale=self.scale, + causal=True, + ) + return flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=cu_seqlens_q, + cu_seqlens_k=cu_seqlens_k, + max_seqlen_q=max_seqlen_q, + max_seqlen_k=max_seqlen_k, + softmax_scale=self.scale, + causal=True, + fa_version=self.fa_version, + ) + def _ensure_on_device(self, layer, device): """One-time derivation of TQ buffers (rotation matrix, midpoints). @@ -503,7 +544,7 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): # max_query_len == max_seq_len means no request has prior cached KV. # Both are Python ints — no GPU sync. if _HAS_FLASH_ATTN and attn_metadata.max_query_len == attn_metadata.max_seq_len: - return flash_attn_varlen_func( + return self._flash_attn_varlen( q=query, k=key, v=value, @@ -511,8 +552,6 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): cu_seqlens_k=attn_metadata.query_start_loc, max_seqlen_q=attn_metadata.max_query_len, max_seqlen_k=attn_metadata.max_query_len, - softmax_scale=self.scale, - causal=True, ) # Continuation or no flash_attn: per-request attention. @@ -552,7 +591,7 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): if _HAS_FLASH_ATTN: _cu_2[1] = q_len cu = _cu_2 - out = flash_attn_varlen_func( + out = self._flash_attn_varlen( q=q_seq, k=k_seq, v=v_seq, @@ -560,8 +599,6 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): cu_seqlens_k=cu, max_seqlen_q=q_len, max_seqlen_k=q_len, - softmax_scale=self.scale, - causal=True, ) else: q_t = q_seq.transpose(0, 1).contiguous() @@ -726,7 +763,7 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): if _HAS_FLASH_ATTN: cu_seqlens_q = torch.tensor([0, q_len], device=device, dtype=torch.int32) cu_seqlens_k = torch.tensor([0, seq_len], device=device, dtype=torch.int32) - return flash_attn_varlen_func( + return self._flash_attn_varlen( q=query, k=k_full, v=v_full, @@ -734,8 +771,6 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): cu_seqlens_k=cu_seqlens_k, max_seqlen_q=q_len, max_seqlen_k=seq_len, - softmax_scale=self.scale, - causal=True, ) else: # SDPA fallback: expand KV for GQA, build causal mask From 342c58bc548f6dd38c1039fdc1c5af014ee9a268 Mon Sep 17 00:00:00 2001 From: Kunshang Ji Date: Thu, 23 Apr 2026 13:04:41 +0800 Subject: [PATCH 054/153] [BugFix]fix Qwen3 MoE call gate twice (#40664) Signed-off-by: Kunshang Ji --- vllm/model_executor/models/qwen3_moe.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index 6f080d07795..520126718fd 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -231,11 +231,19 @@ class Qwen3MoeSparseMoeBlock(nn.Module): if self.is_sequence_parallel: hidden_states = sequence_parallel_chunk(hidden_states) - # router_logits: (num_tokens, n_experts) - router_logits, _ = self.gate(hidden_states) - final_hidden_states = self.experts( - hidden_states=hidden_states, router_logits=router_logits - ) + if self.experts.is_internal_router: + # In this case, the gate/router runs inside the FusedMoE class + final_hidden_states = self.experts( + hidden_states=hidden_states, router_logits=hidden_states + ) + else: + # Actually this will be dead code, since we always pass gate into + # FusedMoE in the current implementation. But we keep this code + # here for clarity and future flexibility. + router_logits, _ = self.gate(hidden_states) + final_hidden_states = self.experts( + hidden_states=hidden_states, router_logits=router_logits + ) if self.is_sequence_parallel: final_hidden_states = tensor_model_parallel_all_gather( From e4ee48da2d24c502a7e16606f871e12ef1e1fa3d Mon Sep 17 00:00:00 2001 From: Kunshang Ji Date: Thu, 23 Apr 2026 13:21:47 +0800 Subject: [PATCH 055/153] [MoE refactor] refactor GPTQMarlinMoEMethod with MK (#37990) Signed-off-by: Kunshang Ji Signed-off-by: Robert Shaw Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Robert Shaw --- .../layers/fused_moe/fused_marlin_moe.py | 4 + .../layers/fused_moe/oracle/int_wna16.py | 445 ++++++++++++++++++ .../layers/quantization/gptq_marlin.py | 348 +++++--------- .../layers/quantization/utils/quant_utils.py | 8 + 4 files changed, 577 insertions(+), 228 deletions(-) create mode 100644 vllm/model_executor/layers/fused_moe/oracle/int_wna16.py diff --git a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py b/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py index daae5b6bd16..6143c3d0adc 100644 --- a/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_marlin_moe.py @@ -40,6 +40,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kFp8Static128BlockSym, kFp8StaticChannelSym, kFp8StaticTensorSym, + kInt4Static, + kInt8Static, kMxfp4Static, kMxfp8Static, kNvfp4Static, @@ -585,6 +587,8 @@ class MarlinExpertsBase(mk.FusedMoEExpertsModular): kMxfp4Static, kMxfp8Static, kNvfp4Static, + kInt4Static, + kInt8Static, ] return weight_key in SUPPORTED_W diff --git a/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py new file mode 100644 index 00000000000..5503d233f12 --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/oracle/int_wna16.py @@ -0,0 +1,445 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from enum import Enum +from typing import TYPE_CHECKING + +import torch + +import vllm._custom_ops as ops +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + BatchedMarlinExperts, + MarlinExperts, +) +from vllm.model_executor.layers.quantization.base_config import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.marlin_utils import ( + marlin_act_int8_process_scales, + marlin_moe_permute_scales, + marlin_permute_bias, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, +) + +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.gptq_marlin import GPTQMarlinConfig + +logger = init_logger(__name__) + + +class WNA16MoEBackend(Enum): + MARLIN = "MARLIN" + BATCHED_MARLIN = "BATCHED_MARLIN" + + +def backend_to_kernel_cls( + backend: WNA16MoEBackend, +) -> list[type[mk.FusedMoEExperts]]: + """Return the experts class for the given backend, or None for NONE.""" + if backend == WNA16MoEBackend.MARLIN: + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + MarlinExperts, + ) + + return [MarlinExperts] + + elif backend == WNA16MoEBackend.BATCHED_MARLIN: + from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( + BatchedMarlinExperts, + ) + + return [BatchedMarlinExperts] + + else: + raise ValueError(f"Unknown WNA16 MoE backend: {backend.value}") + + +def _get_priority_backends() -> list[WNA16MoEBackend]: + """ + Get available backends in priority order based on platform and config. + """ + _AVAILABLE_BACKENDS = [ + WNA16MoEBackend.MARLIN, + WNA16MoEBackend.BATCHED_MARLIN, + ] + return _AVAILABLE_BACKENDS + + +def select_wna16_moe_backend( + config: FusedMoEConfig, + weight_key: QuantKey, + weight_bits: int, +) -> tuple[WNA16MoEBackend, type[mk.FusedMoEExperts]]: + """Select the WNA16 MoE backend. + + Args: + config: the shared ``FusedMoEConfig`` for this layer. + weight_bits: quantization bit-width (4 or 8). 8-bit weights are not + supported by the modular Marlin kernel, so ``NONE`` is returned. + + Returns: + A tuple of (``WNA16MoEBackend``, experts class or ``None``). + """ + + activation_format = ( + mk.FusedMoEActivationFormat.BatchedExperts + if config.moe_parallel_config.use_batched_activation_format + else mk.FusedMoEActivationFormat.Standard + ) + + def _make_log_backend(backend: WNA16MoEBackend): + return f"Using '{backend.value}' WNA16 MoE backend." + + def _make_log_unsupported(backend: WNA16MoEBackend, reason: str | None) -> str: + if reason: + return ( + f"WNA16 MoE backend '{backend.value}' does not support the " + f"deployment configuration since {reason}." + ) + return ( + f"WNA16 MoE backend '{backend.value}' does not support the " + "deployment configuration." + ) + + def _return_or_raise( + backend: WNA16MoEBackend, + config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, + activation_format: mk.FusedMoEActivationFormat, + ) -> tuple[WNA16MoEBackend, type[mk.FusedMoEExperts]]: + reason: str | None = None + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, config, weight_key, activation_key, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend), scope="local") + return backend, k_cls + raise ValueError(_make_log_unsupported(backend, reason)) + + # Select kernels in order of backend. + AVAILABLE_BACKENDS = _get_priority_backends() + + for backend in AVAILABLE_BACKENDS: + activation_key = None # always BF16 activation for WNA16 MoE + for k_cls in backend_to_kernel_cls(backend): + supported, reason = k_cls.is_supported_config( + k_cls, config, weight_key, activation_key, activation_format + ) + if supported: + logger.info_once(_make_log_backend(backend), scope="local") + return backend, k_cls + else: + logger.debug_once(_make_log_unsupported(backend, reason), scope="local") + + raise NotImplementedError( + "No WNA16 MoE backend supports the deployment configuration." + ) + + +def make_wna16_moe_kernel( + moe_quant_config: FusedMoEQuantConfig, + moe_config: FusedMoEConfig, + experts_cls: type[mk.FusedMoEExperts] | None, + layer: torch.nn.Module, + is_k_full: bool, + w13_g_idx: torch.Tensor | None, + w2_g_idx: torch.Tensor | None, + w13_g_idx_sort_indices: torch.Tensor | None, + w2_g_idx_sort_indices: torch.Tensor | None, + routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, + shared_experts: torch.nn.Module | None = None, +) -> mk.FusedMoEKernel: + # Currently, we only support MarlinExperts and BatchedMarlinExperts + assert experts_cls in (MarlinExperts, BatchedMarlinExperts) + + from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, + ) + + prepare_finalize = maybe_make_prepare_finalize( + moe=moe_config, + quant_config=moe_quant_config, + routing_tables=routing_tables, + allow_new_interface=True, + ) + assert prepare_finalize is not None + assert isinstance(prepare_finalize, mk.FusedMoEPrepareAndFinalizeModular) + + if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + assert experts_cls == BatchedMarlinExperts + max_num_tokens = prepare_finalize.max_num_tokens_per_rank() + assert max_num_tokens is not None + experts: mk.FusedMoEExperts = BatchedMarlinExperts( + max_num_tokens=max_num_tokens, + num_dispatchers=prepare_finalize.num_dispatchers(), + moe_config=moe_config, + quant_config=moe_quant_config, + w13_g_idx=w13_g_idx, + w2_g_idx=w2_g_idx, + w13_g_idx_sort_indices=w13_g_idx_sort_indices, + w2_g_idx_sort_indices=w2_g_idx_sort_indices, + is_k_full=is_k_full, + ) + else: + assert experts_cls == MarlinExperts + experts = MarlinExperts( + moe_config=moe_config, + quant_config=moe_quant_config, + w13_g_idx=w13_g_idx, + w2_g_idx=w2_g_idx, + w13_g_idx_sort_indices=w13_g_idx_sort_indices, + w2_g_idx_sort_indices=w2_g_idx_sort_indices, + is_k_full=is_k_full, + ) + + return mk.FusedMoEKernel( + prepare_finalize, + experts, + shared_experts=shared_experts, + inplace=not moe_config.disable_inplace, + ) + + +# --------------------------------------------------------------------------- +# Per-backend weight post-processing +# --------------------------------------------------------------------------- + + +def _process_weights_marlin( + layer: torch.nn.Module, + quant_config: "GPTQMarlinConfig", + input_dtype: torch.dtype | None, + w13_qweight: torch.Tensor, + w2_qweight: torch.Tensor, + w13_scales: torch.Tensor, + w2_scales: torch.Tensor, + w13_g_idx: torch.Tensor, + w2_g_idx: torch.Tensor, + w13_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, +) -> tuple[ + torch.Tensor, # w13_qweight + torch.Tensor, # w2_qweight + torch.Tensor, # w13_scales + torch.Tensor, # w2_scales + torch.Tensor, # w13_g_idx + torch.Tensor, # w2_g_idx + torch.Tensor, # w13_g_idx_sort_indices + torch.Tensor, # w2_g_idx_sort_indices + torch.Tensor | None, # w13_input_global_scale + torch.Tensor | None, # w2_input_global_scale + torch.Tensor | None, # w13_bias + torch.Tensor | None, # w2_bias +]: + """Standard Marlin weight post-processing shared by MARLIN and + BATCHED_MARLIN backends. + + Steps + ----- + 1. Optional FP8 preprocessing of packed weights / scales. + 2. Sort / reset g_idx tensors for act-order handling. + 3. Repack weights via ``gptq_marlin_moe_repack``. + 4. Permute scales (and optionally extract INT8 global scales). + 5. Permute bias tensors. + """ + is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1 + + marlin_w13_qweight: torch.Tensor + marlin_w2_qweight: torch.Tensor + marlin_w13_scales: torch.Tensor + marlin_w2_scales: torch.Tensor + w13_g_idx_sort_indices: torch.Tensor | None = None + w2_g_idx_sort_indices: torch.Tensor | None = None + w13_input_global_scale: torch.Tensor | None = None + w2_input_global_scale: torch.Tensor | None = None + w13_bias_out: torch.Tensor | None = None + w2_bias_out: torch.Tensor | None = None + + # --- FP8 weight / scale adjustment --- + if input_dtype == torch.float8_e4m3fn: + marlin_w13_qweight = ops.marlin_int4_fp8_preprocess(w13_qweight, inplace=False) + marlin_w2_qweight = ops.marlin_int4_fp8_preprocess(w2_qweight, inplace=False) + marlin_w13_scales = w13_scales.data * 512 + marlin_w2_scales = w2_scales.data * 512 + else: + marlin_w13_qweight = w13_qweight + marlin_w2_qweight = w2_qweight + marlin_w13_scales = w13_scales + marlin_w2_scales = w2_scales + + # --- Process act_order (g_idx) --- + if quant_config.desc_act: + num_experts = w13_g_idx.shape[0] + w13_g_idx_sort_indices = torch.empty_like(w13_g_idx) + w2_g_idx_sort_indices = torch.empty_like(w2_g_idx) + w13_sorted_g_idx = torch.empty_like(w13_g_idx) + w2_sorted_g_idx = torch.empty_like(w2_g_idx) + for e in range(num_experts): + w13_g_idx_sort_indices[e] = torch.argsort(w13_g_idx[e]).to(torch.int32) + w2_g_idx_sort_indices[e] = torch.argsort(w2_g_idx[e]).to(torch.int32) + w13_sorted_g_idx[e] = w13_g_idx[e][w13_g_idx_sort_indices[e]] + w2_sorted_g_idx[e] = w2_g_idx[e][w2_g_idx_sort_indices[e]] + else: + num_experts = w13_g_idx.shape[0] + device = w13_g_idx.device + w13_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + w2_g_idx = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + w13_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + w2_g_idx_sort_indices = torch.nn.Parameter( + torch.empty((num_experts, 0), dtype=torch.int32, device=device), + requires_grad=False, + ) + + # --- Repack weights --- + marlin_w13_qweight = ops.gptq_marlin_moe_repack( + marlin_w13_qweight, + w13_g_idx_sort_indices, + marlin_w13_qweight.shape[1] * quant_config.pack_factor, + marlin_w13_qweight.shape[2], + quant_config.quant_type.size_bits, + is_a_8bit=is_a_8bit, + ) + marlin_w2_qweight = ops.gptq_marlin_moe_repack( + marlin_w2_qweight, + w2_g_idx_sort_indices, + marlin_w2_qweight.shape[1] * quant_config.pack_factor, + marlin_w2_qweight.shape[2], + quant_config.quant_type.size_bits, + is_a_8bit=is_a_8bit, + ) + + # --- Permute scales --- + marlin_w13_scales = marlin_moe_permute_scales( + s=marlin_w13_scales, + size_k=layer.intermediate_size_per_partition, + size_n=marlin_w13_scales.shape[2], + group_size=quant_config.group_size, + is_a_8bit=is_a_8bit, + ) + marlin_w2_scales = marlin_moe_permute_scales( + s=marlin_w2_scales, + size_k=marlin_w2_scales.shape[1] + * ( + quant_config.group_size + if quant_config.group_size != -1 + else quant_config.pack_factor + ), + size_n=marlin_w2_scales.shape[2], + group_size=quant_config.group_size, + is_a_8bit=is_a_8bit, + ) + + if input_dtype == torch.int8: + if layer.num_groups_w13 > 1: + marlin_w13_scales, w13_input_global_scale = marlin_act_int8_process_scales( + marlin_w13_scales + ) + if layer.num_groups_w2 > 1: + marlin_w2_scales, w2_input_global_scale = marlin_act_int8_process_scales( + marlin_w2_scales + ) + + # --- Permute bias --- + if w13_bias is not None: + w13_bias_out = marlin_permute_bias(w13_bias) + if w2_bias is not None: + w2_bias_out = marlin_permute_bias(w2_bias) + + return ( + marlin_w13_qweight, + marlin_w2_qweight, + marlin_w13_scales, + marlin_w2_scales, + w13_g_idx, + w2_g_idx, + w13_g_idx_sort_indices, + w2_g_idx_sort_indices, + w13_input_global_scale, + w2_input_global_scale, + w13_bias_out, + w2_bias_out, + ) + + +def convert_to_wna16_moe_kernel_format( + backend: WNA16MoEBackend, + layer: torch.nn.Module, + quant_config: QuantizationConfig, + input_dtype: torch.dtype | None, + w13: torch.Tensor, + w2: torch.Tensor, + w13_scale: torch.Tensor, + w2_scale: torch.Tensor, + w13_g_idx: torch.Tensor, + w2_g_idx: torch.Tensor, + w13_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, +) -> tuple[ + torch.Tensor, # w13_qweight + torch.Tensor, # w2_qweight + torch.Tensor, # w13_scales + torch.Tensor, # w2_scales + torch.Tensor | None, # w13_g_idx + torch.Tensor | None, # w2_g_idx + torch.Tensor | None, # w13_g_idx_sort_indices + torch.Tensor | None, # w2_g_idx_sort_indices + torch.Tensor | None, # w13_input_global_scale + torch.Tensor | None, # w2_input_global_scale + torch.Tensor | None, # w13_bias + torch.Tensor | None, # w2_bias +]: + """Dispatch weight post-processing to the appropriate per-backend handler. + + To add a new backend, implement a ``_process_weights_`` helper and + add a branch here. + + Args: + backend: the selected ``WNA16MoEBackend``. + layer: the ``FusedMoE`` layer whose parameters are being prepared. + quant_config: the ``QuantizationConfig`` for this layer. + input_dtype: optional activation dtype, usually should be 16 bit. + """ + if backend in ( + WNA16MoEBackend.MARLIN, + WNA16MoEBackend.BATCHED_MARLIN, + ): + from vllm.model_executor.layers.quantization.gptq_marlin import ( + GPTQMarlinConfig, + ) + + if not isinstance(quant_config, GPTQMarlinConfig): + raise TypeError( + "Marlin WNA16 MoE backend requires GPTQMarlinConfig, got " + f"{type(quant_config).__name__}." + ) + return _process_weights_marlin( + layer, + quant_config, + input_dtype, + w13, + w2, + w13_scale, + w2_scale, + w13_g_idx, + w2_g_idx, + w13_bias, + w2_bias, + ) + else: + raise ValueError(f"Unsupported wna16 MoE backend: {backend.value}") diff --git a/vllm/model_executor/layers/quantization/gptq_marlin.py b/vllm/model_executor/layers/quantization/gptq_marlin.py index 1ca551d6351..7b6f1f9cf6c 100644 --- a/vllm/model_executor/layers/quantization/gptq_marlin.py +++ b/vllm/model_executor/layers/quantization/gptq_marlin.py @@ -9,7 +9,6 @@ from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE from transformers import PretrainedConfig import vllm.model_executor.layers.fused_moe # noqa -from vllm import _custom_ops as ops from vllm.logger import init_logger from vllm.model_executor.kernels.linear import ( MPLinearLayerConfig, @@ -19,13 +18,17 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe from vllm.model_executor.layers.fused_moe.layer import ( FusedMoE, FusedMoEMethodBase, FusedMoeWeightScaleSupported, UnquantizedFusedMoEMethod, ) +from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( + convert_to_wna16_moe_kernel_format, + make_wna16_moe_kernel, + select_wna16_moe_backend, +) from vllm.model_executor.layers.linear import LinearMethodBase, set_weight_attrs from vllm.model_executor.layers.quantization import QuantizationMethods from vllm.model_executor.layers.quantization.base_config import ( @@ -42,13 +45,15 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import ( check_marlin_supported, check_moe_marlin_supports_layer, get_marlin_input_dtype, - marlin_act_int8_process_scales, marlin_make_workspace_new, - marlin_moe_permute_scales, - marlin_permute_bias, marlin_repeat_scales_on_all_ranks, verify_marlin_supported, ) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kInt4StaticGroupScale, + kInt8StaticGroupScale, +) from vllm.model_executor.parameter import ( ChannelQuantScaleParameter, GroupQuantScaleParameter, @@ -500,13 +505,20 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): super().__init__(moe) self.quant_config = quant_config if self.quant_config.quant_type.size_bits == 4: - self.quant_type = scalar_types.uint4b8 + quant_type = scalar_types.uint4b8 + scale = kInt4StaticGroupScale elif self.quant_config.quant_type.size_bits == 8: - self.quant_type = scalar_types.uint8b128 + quant_type = scalar_types.uint8b128 + scale = kInt8StaticGroupScale else: raise ValueError("GPTQMarlinMoEMethod only supports int4 and int8 now.") self.input_dtype = None self.use_marlin = True + weight_key = QuantKey(quant_type, scale) + + self.wna16_moe_backend, self.experts_cls = select_wna16_moe_backend( + moe, weight_key, quant_config.weight_bits + ) def create_weights( self, @@ -521,7 +533,7 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1 if is_a_8bit: - assert self.quant_type == scalar_types.uint4b8, ( + assert self.quant_config.quant_type.size_bits == 8, ( "W8A8-INT8 is not supported by marlin kernel." ) @@ -668,134 +680,100 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): is_a_8bit = self.input_dtype is not None and self.input_dtype.itemsize == 1 if is_a_8bit: - assert self.quant_type == scalar_types.uint4b8, ( + assert self.quant_config.quant_type.size_bits == 8, ( "W8A8-INT8 is not supported by marlin kernel." ) - if self.input_dtype == torch.float8_e4m3fn: - ops.marlin_int4_fp8_preprocess(layer.w13_qweight, inplace=True) - ops.marlin_int4_fp8_preprocess(layer.w2_qweight, inplace=True) - layer.w13_scales.data = layer.w13_scales.data * 512 - layer.w2_scales.data = layer.w2_scales.data * 512 + ( + w13, + w2, + w13_scale, + w2_scale, + w13_g_idx, + w2_g_idx, + w13_g_idx_sort_indices, + w2_g_idx_sort_indices, + w13_input_global_scale, + w2_input_global_scale, + w13_bias, + w2_bias, + ) = convert_to_wna16_moe_kernel_format( + backend=self.wna16_moe_backend, + layer=layer, + quant_config=self.quant_config, + input_dtype=self.input_dtype, + w13=layer.w13_qweight, + w2=layer.w2_qweight, + w13_scale=layer.w13_scales, + w2_scale=layer.w2_scales, + w13_g_idx=layer.w13_g_idx, + w2_g_idx=layer.w2_g_idx, + w13_bias=getattr(layer, "w13_bias", None), + w2_bias=getattr(layer, "w2_bias", None), + ) - # Process act_order - if self.quant_config.desc_act: - # Get sorting based on g_idx - num_experts = layer.w13_g_idx.shape[0] - w13_g_idx_sort_indices = torch.empty_like(layer.w13_g_idx) - w2_g_idx_sort_indices = torch.empty_like(layer.w2_g_idx) - w13_sorted_g_idx = torch.empty_like(layer.w13_g_idx) - w2_sorted_g_idx = torch.empty_like(layer.w2_g_idx) - for e in range(num_experts): - w13_g_idx_sort_indices[e] = torch.argsort(layer.w13_g_idx[e]).to( - torch.int32 + replace_parameter(layer, "w13_qweight", w13) + replace_parameter(layer, "w2_qweight", w2) + replace_parameter(layer, "w13_scales", w13_scale) + replace_parameter(layer, "w2_scales", w2_scale) + replace_parameter(layer, "w13_g_idx", w13_g_idx) + replace_parameter(layer, "w2_g_idx", w2_g_idx) + replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices) + replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices) + if w13_input_global_scale is not None: + if hasattr(layer, "w13_input_global_scale"): + replace_parameter( + layer, "w13_input_global_scale", w13_input_global_scale ) - w2_g_idx_sort_indices[e] = torch.argsort(layer.w2_g_idx[e]).to( - torch.int32 + else: + layer.register_parameter( + "w13_input_global_scale", + torch.nn.Parameter(w13_input_global_scale, requires_grad=False), ) - w13_sorted_g_idx[e] = layer.w13_g_idx[e][w13_g_idx_sort_indices[e]] - w2_sorted_g_idx[e] = layer.w2_g_idx[e][w2_g_idx_sort_indices[e]] - replace_parameter(layer, "w13_g_idx", w13_sorted_g_idx) - replace_parameter(layer, "w2_g_idx", w2_sorted_g_idx) - replace_parameter(layer, "w13_g_idx_sort_indices", w13_g_idx_sort_indices) - replace_parameter(layer, "w2_g_idx_sort_indices", w2_g_idx_sort_indices) - else: - # Reset g_idx related tensors - num_experts = layer.w13_g_idx.shape[0] - device = layer.w13_g_idx.device - layer.w13_g_idx = torch.nn.Parameter( - torch.empty((num_experts, 0), dtype=torch.int32, device=device), - requires_grad=False, - ) - layer.w2_g_idx = torch.nn.Parameter( - torch.empty((num_experts, 0), dtype=torch.int32, device=device), - requires_grad=False, - ) - layer.w13_g_idx_sort_indices = torch.nn.Parameter( - torch.empty((num_experts, 0), dtype=torch.int32, device=device), - requires_grad=False, - ) - layer.w2_g_idx_sort_indices = torch.nn.Parameter( - torch.empty((num_experts, 0), dtype=torch.int32, device=device), - requires_grad=False, - ) - # Repack weights - marlin_w13_qweight = ops.gptq_marlin_moe_repack( - layer.w13_qweight, - layer.w13_g_idx_sort_indices, - layer.w13_qweight.shape[1] * self.quant_config.pack_factor, - layer.w13_qweight.shape[2], - self.quant_config.quant_type.size_bits, - is_a_8bit=is_a_8bit, + if w2_input_global_scale is not None: + if hasattr(layer, "w2_input_global_scale"): + replace_parameter(layer, "w2_input_global_scale", w2_input_global_scale) + else: + layer.register_parameter( + "w2_input_global_scale", + torch.nn.Parameter(w2_input_global_scale, requires_grad=False), + ) + if w13_bias is not None: + if hasattr(layer, "w13_bias"): + replace_parameter(layer, "w13_bias", w13_bias) + else: + layer.register_parameter( + "w13_bias", torch.nn.Parameter(w13_bias, requires_grad=False) + ) + if w2_bias is not None: + if hasattr(layer, "w2_bias"): + replace_parameter(layer, "w2_bias", w2_bias) + else: + layer.register_parameter( + "w2_bias", torch.nn.Parameter(w2_bias, requires_grad=False) + ) + + self._setup_kernel(layer) + + def _setup_kernel(self, layer: FusedMoE) -> None: + """Build the FusedMoEKernel for this layer.""" + + self.moe_quant_config = self.get_fused_moe_quant_config(layer) + self.moe_kernel = make_wna16_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + experts_cls=self.experts_cls, + layer=layer, + is_k_full=self.is_k_full, + w13_g_idx=layer.w13_g_idx, + w2_g_idx=layer.w2_g_idx, + w13_g_idx_sort_indices=layer.w13_g_idx_sort_indices, + w2_g_idx_sort_indices=layer.w2_g_idx_sort_indices, + routing_tables=layer._maybe_init_expert_routing_tables(), + shared_experts=layer.shared_experts, ) - replace_parameter(layer, "w13_qweight", marlin_w13_qweight) - marlin_w2_qweight = ops.gptq_marlin_moe_repack( - layer.w2_qweight, - layer.w2_g_idx_sort_indices, - layer.w2_qweight.shape[1] * self.quant_config.pack_factor, - layer.w2_qweight.shape[2], - self.quant_config.quant_type.size_bits, - is_a_8bit=is_a_8bit, - ) - replace_parameter(layer, "w2_qweight", marlin_w2_qweight) - # The modular kernel expects w13_weight and w2_weight, - # but GPTQ uses w13_qweight and w2_qweight - # Alias for modular kernel - layer.w13_weight = layer.w13_qweight - # Alias for modular kernel - layer.w2_weight = layer.w2_qweight - - # Repack scales - marlin_w13_scales = marlin_moe_permute_scales( - s=layer.w13_scales, - size_k=layer.intermediate_size_per_partition, - size_n=layer.w13_scales.shape[2], - group_size=self.quant_config.group_size, - is_a_8bit=is_a_8bit, - ) - if self.input_dtype == torch.int8 and layer.num_groups_w13 > 1: - marlin_w13_scales, w13_input_global_scale = marlin_act_int8_process_scales( - marlin_w13_scales - ) - layer.register_parameter( - "w13_input_global_scale", - torch.nn.Parameter(w13_input_global_scale, requires_grad=False), - ) - - replace_parameter(layer, "w13_scales", marlin_w13_scales) - marlin_w2_scales = marlin_moe_permute_scales( - s=layer.w2_scales, - size_k=layer.w2_scales.shape[1] - * ( - self.quant_config.group_size - if self.quant_config.group_size != -1 - else self.quant_config.pack_factor - ), - size_n=layer.w2_scales.shape[2], - group_size=self.quant_config.group_size, - is_a_8bit=is_a_8bit, - ) - if self.input_dtype == torch.int8 and layer.num_groups_w2 > 1: - marlin_w2_scales, w2_input_global_scale = marlin_act_int8_process_scales( - marlin_w2_scales - ) - layer.register_parameter( - "w2_input_global_scale", - torch.nn.Parameter(w2_input_global_scale, requires_grad=False), - ) - - replace_parameter(layer, "w2_scales", marlin_w2_scales) - - if hasattr(layer, "w13_bias") and layer.w13_bias is not None: - layer.w13_bias.data = marlin_permute_bias(layer.w13_bias) - - if hasattr(layer, "w2_bias") and layer.w2_bias is not None: - layer.w2_bias.data = marlin_permute_bias(layer.w2_bias) - - def get_fused_moe_quant_config( - self, layer: torch.nn.Module - ) -> FusedMoEQuantConfig | None: + def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: from vllm.model_executor.layers.fused_moe.config import ( gptq_marlin_moe_quant_config, ) @@ -820,86 +798,11 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): prepare_finalize, layer: torch.nn.Module, ): - """ - Select the GEMM implementation for GPTQ-Marlin MoE. - - Returns MarlinExperts configured for GPTQ quantization. - This is ONLY used when LoRA is enabled. - Without LoRA, GPTQ uses its own apply() method. - """ - # Only use modular kernels when LoRA is enabled - # Without LoRA, GPTQ's own apply() method works fine and is more efficient - if not self.moe.is_lora_enabled: - raise NotImplementedError( - "GPTQ-Marlin uses its own apply() method when LoRA is not enabled. " - "Modular kernels are only used for LoRA support." - ) - - # The modular marlin kernels do not support 8-bit weights. - if self.quant_config.weight_bits == 8: - raise NotImplementedError( - "GPTQ-Marlin kernel does not support 8-bit weights." - ) - - from vllm.model_executor.layers.fused_moe import modular_kernel as mk - from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( - BatchedMarlinExperts, - MarlinExperts, + raise ValueError( + f"{self.__class__.__name__} uses the new modular kernel " + "initialization logic. This function should not be called." ) - # Ensure quant config is initialized - assert self.moe_quant_config is not None, ( - "moe_quant_config must be initialized before select_gemm_impl" - ) - - w13_g_idx = ( - getattr(layer, "w13_g_idx", None) if self.quant_config.desc_act else None - ) - w2_g_idx = ( - getattr(layer, "w2_g_idx", None) if self.quant_config.desc_act else None - ) - w13_g_idx_sort_indices = ( - getattr(layer, "w13_g_idx_sort_indices", None) - if self.quant_config.desc_act - else None - ) - w2_g_idx_sort_indices = ( - getattr(layer, "w2_g_idx_sort_indices", None) - if self.quant_config.desc_act - else None - ) - - # Check if using batched expert format (for Expert Parallelism) - if ( - prepare_finalize.activation_format - == mk.FusedMoEActivationFormat.BatchedExperts - ): - # For batched format, use BatchedMarlinExperts - max_num_tokens_per_rank = prepare_finalize.max_num_tokens_per_rank() - assert max_num_tokens_per_rank is not None - return BatchedMarlinExperts( - max_num_tokens=max_num_tokens_per_rank, - num_dispatchers=prepare_finalize.num_dispatchers(), - moe_config=self.moe, - quant_config=self.moe_quant_config, - w13_g_idx=w13_g_idx, - w2_g_idx=w2_g_idx, - w13_g_idx_sort_indices=w13_g_idx_sort_indices, - w2_g_idx_sort_indices=w2_g_idx_sort_indices, - is_k_full=self.is_k_full, - ) - else: - # Standard Marlin experts for GPTQ - return MarlinExperts( - moe_config=self.moe, - quant_config=self.moe_quant_config, - w13_g_idx=w13_g_idx, - w2_g_idx=w2_g_idx, - w13_g_idx_sort_indices=w13_g_idx_sort_indices, - w2_g_idx_sort_indices=w2_g_idx_sort_indices, - is_k_full=self.is_k_full, - ) - def apply( self, layer: FusedMoE, @@ -908,28 +811,17 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts_input: torch.Tensor | None, ) -> torch.Tensor: - return fused_marlin_moe( - x, - layer.w13_qweight, - layer.w2_qweight, - getattr(layer, "w13_bias", None), - getattr(layer, "w2_bias", None), - layer.w13_scales, - layer.w2_scales, - topk_weights, - topk_ids, - input_global_scale1=getattr(layer, "w13_input_global_scale", None), - input_global_scale2=getattr(layer, "w2_input_global_scale", None), - quant_type_id=self.quant_type.id, - apply_router_weight_on_input=layer.apply_router_weight_on_input, + assert not self.is_monolithic + assert self.moe_kernel is not None + return self.moe_kernel.apply( + hidden_states=x, + w1=layer.w13_qweight, + w2=layer.w2_qweight, + topk_weights=topk_weights, + topk_ids=topk_ids, + activation=layer.activation, global_num_experts=layer.global_num_experts, + apply_router_weight_on_input=layer.apply_router_weight_on_input, expert_map=layer.expert_map, - g_idx1=layer.w13_g_idx, - g_idx2=layer.w2_g_idx, - sort_indices1=layer.w13_g_idx_sort_indices, - sort_indices2=layer.w2_g_idx_sort_indices, - workspace=layer.workspace, - is_k_full=self.is_k_full, - input_dtype=self.input_dtype, - inplace=not self.moe.disable_inplace, + shared_experts_input=shared_experts_input, ) diff --git a/vllm/model_executor/layers/quantization/utils/quant_utils.py b/vllm/model_executor/layers/quantization/utils/quant_utils.py index fedb9067207..0b180252241 100644 --- a/vllm/model_executor/layers/quantization/utils/quant_utils.py +++ b/vllm/model_executor/layers/quantization/utils/quant_utils.py @@ -20,6 +20,8 @@ if TYPE_CHECKING: FP8_DTYPE = current_platform.fp8_dtype() FP4_DTYPE = torch.uint8 MXFP_SCALE_DTYPE = torch.uint8 +INT4_DTYPE = scalar_types.uint4b8 +INT8_DTYPE = scalar_types.uint8b128 def get_fp8_min_max() -> tuple[float, float]: @@ -170,6 +172,12 @@ kMxfp8Dynamic = QuantKey(FP8_DTYPE, scale=kMxfp8DynamicGroupScale, symmetric=Tru kMxfp4StaticGroupScale = ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)) kMxfp4Static = QuantKey(FP4_DTYPE, scale=kMxfp4StaticGroupScale, symmetric=True) +# TODO: convert this to use SCALAR_TYPE. This is not right. +kInt4StaticGroupScale = ScaleDesc(torch.float16, True, GroupShape(1, -1)) +kInt4Static = QuantKey(INT4_DTYPE, scale=kInt4StaticGroupScale, symmetric=True) +kInt8StaticGroupScale = ScaleDesc(torch.float16, True, GroupShape(1, -1)) +kInt8Static = QuantKey(INT8_DTYPE, scale=kInt8StaticGroupScale, symmetric=True) + kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True) kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True) From 98a242ff61f7264f6bb7e2d8dd6f78c8496fb39e Mon Sep 17 00:00:00 2001 From: Zhengxu Chen Date: Thu, 23 Apr 2026 01:43:18 -0400 Subject: [PATCH 056/153] [compile] Skip FX graph deserialiaztion on loading, further reducing warm compile time. (#40151) Signed-off-by: zhxchen17 --- vllm/compilation/backends.py | 9 ++- vllm/compilation/caching.py | 35 +++++++----- vllm/compilation/codegen.py | 103 +++++++++++++++++++++++++---------- 3 files changed, 97 insertions(+), 50 deletions(-) diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index 8045e296e13..569b0ac0801 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -23,6 +23,10 @@ from torch._logging._internal import trace_structured from torch.fx._lazy_graph_module import _use_lazy_graph_module import vllm.envs as envs +from vllm.compilation.codegen import ( + compile_execution_fn, + generate_execution_code, +) from vllm.config import CompilationConfig, CUDAGraphMode, VllmConfig from vllm.config.compilation import DynamicShapesType from vllm.config.utils import Range, hash_factors @@ -1244,11 +1248,6 @@ class VllmBackend: original_split_gm if envs.VLLM_USE_MEGA_AOT_ARTIFACT else self.graph ) - from vllm.compilation.codegen import ( - compile_execution_fn, - generate_execution_code, - ) - execution_code, submod_names = generate_execution_code(self.split_gm) # Use getattr to get correct callables: __dict__ has PiecewiseBackend # instances (from PiecewiseCompileInterpreter), _modules has originals. diff --git a/vllm/compilation/caching.py b/vllm/compilation/caching.py index 6b61c0c770b..81c3d7b2865 100644 --- a/vllm/compilation/caching.py +++ b/vllm/compilation/caching.py @@ -16,6 +16,7 @@ from torch.fx._graph_pickler import GraphPickler, Options from torch.utils import _pytree as pytree import vllm.envs as envs +from vllm.compilation.codegen import compile_execution_fn from vllm.compilation.compiler_interface import get_inductor_factors from vllm.compilation.counter import compilation_counter from vllm.config import VllmConfig, get_current_vllm_config @@ -176,7 +177,7 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc] def __init__( self, - graph_module: torch.fx.GraphModule, + graph_module: torch.fx.GraphModule | bytes, example_inputs: Sequence[Any], prefix: str, optimized_call: Callable[..., Any], @@ -187,7 +188,6 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc] execution_code: str | None = None, submod_names: list[str] | None = None, ) -> None: - assert isinstance(graph_module, torch.fx.GraphModule) self.graph_module = graph_module self.example_inputs = example_inputs self.prefix = prefix @@ -302,10 +302,6 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc] state = pickle.loads(data) fake_mode = FakeTensorMode(shape_env=ShapeEnv()) - state["graph_module"] = cls.deserialize_graph_module( - state["graph_module"], fake_mode - ) - state["graph_module"].recompile() state["example_inputs"] = GraphPickler.loads(state["example_inputs"], fake_mode) standalone_compile_artifacts = state.pop("standalone_compile_artifacts", None) @@ -331,6 +327,7 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc] vllm_config=get_current_vllm_config(), sym_shape_indices_map=sym_shape_indices_map, returns_tuple_map=returns_tuple_map, + fake_mode=fake_mode, ) logger.info( @@ -342,6 +339,11 @@ class VllmSerializableFunction(SerializableCallable): # type: ignore[misc] return fn + state["graph_module"] = cls.deserialize_graph_module( + state["graph_module"], fake_mode + ) + state["graph_module"].recompile() + # Fall back to standard VllmBackend. # Use a lazy closure: the backend needs traced_files for cache # dir computation, but those are only populated after @@ -410,6 +412,7 @@ def reconstruct_serializable_fn_from_mega_artifact( vllm_config: VllmConfig, sym_shape_indices_map: dict[str, list[int]], returns_tuple_map: dict[str, bool], + fake_mode: FakeTensorMode, ) -> "VllmSerializableFunction": """Construct a VllmSerializableFunction from cached inductor artifacts. @@ -452,7 +455,6 @@ def reconstruct_serializable_fn_from_mega_artifact( prefix = state["prefix"] is_encoder = state.get("is_encoder", False) - split_gm = state["graph_module"] compilation_config = vllm_config.compilation_config standalone_compile_artifacts.load_all() @@ -476,13 +478,16 @@ def reconstruct_serializable_fn_from_mega_artifact( ) # spot check that cached submodules exist in the graph structure - graph_children = {name for name, _ in split_gm.named_children()} + # if an old cache is used, this will fail but that's fine because + # we will just try this error and re-generate the new cache. + graph_children = set(state["submod_names"]) missing = set(piecewise_submod_names) - graph_children assert not missing, ( f"artifacts reference submodules not in graph: {missing}. " f"graph has: {sorted(graph_children)}" ) + submod_callables = {} for i, submod_name in enumerate(piecewise_submod_names): assert submod_name in sym_shape_indices_map and submod_name in returns_tuple_map @@ -511,7 +516,7 @@ def reconstruct_serializable_fn_from_mega_artifact( is_last, ) - split_gm.__dict__[submod_name] = wrapped_backend + submod_callables[submod_name] = wrapped_backend logger.debug( "Replaced submodule %s with piecewise backend from cache", submod_name, @@ -521,16 +526,16 @@ def reconstruct_serializable_fn_from_mega_artifact( execution_code = state.get("execution_code") submod_names = state.get("submod_names") if execution_code is not None and submod_names is not None: - from vllm.compilation.codegen import compile_execution_fn - - submod_callables = { - name: getattr(split_gm, name) for name, _ in split_gm.named_children() - } runtime_callable = compile_execution_fn( execution_code, submod_callables, submod_names ) else: - runtime_callable = split_gm + logger.warning( + "No execution code found, falling back to graph module execution." + ) + runtime_callable = GraphPickler.loads( + state["graph_module"], fake_mode=fake_mode + ) if compilation_config.cudagraph_copy_inputs: sym_tensor_indices = state["sym_tensor_indices"] diff --git a/vllm/compilation/codegen.py b/vllm/compilation/codegen.py index 661b56cfb75..1baad435764 100644 --- a/vllm/compilation/codegen.py +++ b/vllm/compilation/codegen.py @@ -15,26 +15,14 @@ from typing import Any import torch.fx from torch._dynamo.utils import dynamo_timed from torch._logging import trace_structured +from torch.fx.node import _get_qualified_name -@dynamo_timed("vllm.generate_execution_code") -def generate_execution_code( +def generate_execution_code_with_name( split_gm: torch.fx.GraphModule, + fn_name: str, + with_submod: bool, ) -> tuple[str, list[str]]: - """Generate Python source code from a split_gm's stitching graph. - - Walks split_gm.graph.nodes and produces a function that calls - submodules via a __vllm_submods__ list, avoiding FX GraphModule overhead - and dict lookup cost. - - Args: - split_gm: The split graph module produced by split_graph(). - - Returns: - A tuple of (code, submod_names) where code is the Python source - and submod_names is the ordered list of submodule target names - corresponding to list indices used in the generated code. - """ lines: list[str] = [] param_names: list[str] = [] submod_names: list[str] = [] @@ -43,6 +31,7 @@ def generate_execution_code( # Build node ordering for liveness analysis. nodes = list(split_gm.graph.nodes) node_order = {node: i for i, node in enumerate(nodes)} + inlined_submods: list[str] = [] # For each value-producing node, find the position of its last consumer. # If the last consumer is the output node, skip (return handles cleanup). @@ -65,6 +54,10 @@ def generate_execution_code( elif node.op == "call_module": target = node.target + if not with_submod: + raise RuntimeError( + f"call_module is not allowed for codegen target {target}." + ) if target not in submod_index: submod_index[target] = len(submod_names) submod_names.append(target) @@ -74,13 +67,32 @@ def generate_execution_code( f"{k}={_node_ref(v)}" for k, v in node.kwargs.items() ) all_args = ", ".join(filter(None, [args_str, kwargs_str])) - lines.append(f" {node.name} = __vllm_submods__[{idx}]({all_args})") + submod = getattr(split_gm, target) + if isinstance(submod, torch.fx.GraphModule): + callable_name = f"__vllm_inlined_submods__{idx}" + inlined_code, _ = generate_execution_code_with_name( + submod, callable_name, with_submod=False + ) + inlined_submods.append(inlined_code) + else: + callable_name = f"__vllm_submods__[{idx}]" + lines.append(f" {node.name} = {callable_name}({all_args})") - elif node.op == "call_function" and node.target is operator.getitem: - source = _node_ref(node.args[0]) - index = node.args[1] - assert isinstance(index, int) - lines.append(f" {node.name} = {source}[{index}]") + elif node.op == "call_function": + if node.target is operator.getitem: + source = _node_ref(node.args[0]) + index = node.args[1] + assert isinstance(index, int) + lines.append(f" {node.name} = {source}[{index}]") + else: + args_str = ", ".join(_node_ref(a) for a in node.args) + kwargs_str = ", ".join( + f"{k}={_node_ref(v)}" for k, v in node.kwargs.items() + ) + all_args = ", ".join(filter(None, [args_str, kwargs_str])) + lines.append( + f" {node.name} = {_get_qualified_name(node.target)}({all_args})" + ) elif node.op == "output": assert len(node.args) == 1 @@ -91,14 +103,44 @@ def generate_execution_code( raise RuntimeError(f"Unsupported node from codegen: {node.format_node()}") # Emit del for variables whose last use was this node. - if i in del_after: + if i in del_after and i < len(nodes) - 2: names = sorted(del_after[i]) lines.append(f" del {', '.join(names)}") assert len(param_names) > 0 params = ", ".join(param_names) - header = f"def execution_fn({params}, *, __vllm_submods__):" - return "import torch\n" + "\n".join([header] + lines) + "\n", submod_names + header = ( + f"\ndef {fn_name}({params}{', *, __vllm_submods__' if with_submod else ''}):" + ) + return "".join(inlined_submods) + "\n".join([header] + lines) + "\n", submod_names + + +@dynamo_timed("vllm.generate_execution_code") +def generate_execution_code( + split_gm: torch.fx.GraphModule, +) -> tuple[str, list[str]]: + """Generate Python source code from a split_gm's stitching graph. + + Walks split_gm.graph.nodes and produces a function that calls + submodules via a __vllm_submods__ list, avoiding FX GraphModule overhead + and dict lookup cost. + + If a submodule is a plain torch.fx.GraphModule, it is inlined directly + in the generated code and we do not need to serialize it in the artifact. + + Args: + split_gm: The split graph module produced by split_graph(). + + Returns: + A tuple of (code, submod_names) where code is the Python source + and submod_names is the ordered list of submodule target names + corresponding to list indices used in the generated code. + """ + + code, submod_names = generate_execution_code_with_name( + split_gm, "execution_fn", with_submod=True + ) + return "import torch\nimport operator\n" + code, submod_names @dynamo_timed("vllm.compile_execution_fn") @@ -129,11 +171,12 @@ def compile_execution_fn( namespace: dict[str, Any] = {} exec(code, namespace) # noqa: S102 fn = namespace["execution_fn"] - # Use .forward() directly to avoid nn.Module.__call__ overhead. - submods_list = [ - c.forward if isinstance(c, torch.fx.GraphModule) else c - for c in (submod_callables[name] for name in submod_names) - ] + # Using .get() is intentional here because only piecewise backend will + # be stored in submod_callables. The other submodules are inlined and + # we don't need to bind them to the execution function. Instead, we + # should use None as placeholder to ensure the list indices are preserved + # for better debuggability. + submods_list = [submod_callables.get(name) for name in submod_names] return partial(fn, __vllm_submods__=submods_list) From 8317cedc7718301866963400646b1d624a6471ea Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Thu, 23 Apr 2026 01:46:10 -0400 Subject: [PATCH 057/153] [Responses] Add tool_choice/tools validation to match OpenAI behavior (#40399) Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../test_responses_request_validations.py | 184 ++++++++++++++++++ vllm/entrypoints/openai/responses/protocol.py | 40 ++++ vllm/entrypoints/openai/responses/serving.py | 5 +- 3 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 tests/tool_use/test_responses_request_validations.py diff --git a/tests/tool_use/test_responses_request_validations.py b/tests/tool_use/test_responses_request_validations.py new file mode 100644 index 00000000000..63a1828c500 --- /dev/null +++ b/tests/tool_use/test_responses_request_validations.py @@ -0,0 +1,184 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +from pydantic import ValidationError + +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + +SAMPLE_TOOL = { + "type": "function", + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string", "description": "City name"}}, + "required": ["location"], + }, +} + +NAMED_TOOL_CHOICE = { + "type": "function", + "name": "get_weather", +} + + +def test_responses_request_with_no_tools(): + # tools key is not present — defaults tool_choice to "none" + request = ResponsesRequest.model_validate({"input": "Hello", "model": "test-model"}) + assert request.tool_choice == "none" + + # tools key present but empty + request = ResponsesRequest.model_validate( + {"input": "Hello", "model": "test-model", "tools": []} + ) + assert request.tool_choice == "none" + + +def test_responses_request_no_tools_tool_choice_none(): + request = ResponsesRequest.model_validate( + {"input": "Hello", "model": "test-model", "tool_choice": "none"} + ) + assert request.tool_choice == "none" + + +def test_responses_request_no_tools_tool_choice_auto(): + request = ResponsesRequest.model_validate( + {"input": "Hello", "model": "test-model", "tool_choice": "auto"} + ) + assert request.tool_choice == "none" + + +@pytest.mark.parametrize("tools", [None, []]) +def test_responses_request_required_without_tools(tools): + kwargs = {"input": "Hello", "model": "test-model", "tool_choice": "required"} + if tools is not None: + kwargs["tools"] = tools + with pytest.raises( + ValidationError, match="Tool choice 'required' must be specified" + ): + ResponsesRequest.model_validate(kwargs) + + +def test_responses_request_named_tool_choice_without_tools(): + with pytest.raises(ValidationError, match="not found in 'tools' parameter"): + ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tool_choice": NAMED_TOOL_CHOICE, + } + ) + + +def test_responses_request_with_tools_default_tool_choice(): + request = ResponsesRequest.model_validate( + {"input": "Hello", "model": "test-model", "tools": [SAMPLE_TOOL]} + ) + assert request.tool_choice == "auto" + + +def test_responses_request_with_tools_tool_choice_none(): + request = ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [SAMPLE_TOOL], + "tool_choice": "none", + } + ) + assert request.tool_choice == "none" + + +def test_responses_request_named_tool_choice_matching(): + request = ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [SAMPLE_TOOL], + "tool_choice": NAMED_TOOL_CHOICE, + } + ) + assert request.tool_choice.type == "function" + assert request.tool_choice.name == "get_weather" + + +def test_responses_request_named_tool_choice_not_matching(): + with pytest.raises(ValidationError, match="not found in 'tools' parameter"): + ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [SAMPLE_TOOL], + "tool_choice": {"type": "function", "name": "nonexistent"}, + } + ) + + +def test_responses_request_with_tools_tool_choice_auto(): + request = ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [SAMPLE_TOOL], + "tool_choice": "auto", + } + ) + assert request.tool_choice == "auto" + + +def test_responses_request_with_tools_tool_choice_required(): + request = ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [SAMPLE_TOOL], + "tool_choice": "required", + } + ) + assert request.tool_choice == "required" + + +def test_responses_request_empty_tools_tool_choice_none(): + request = ResponsesRequest.model_validate( + {"input": "Hello", "model": "test-model", "tools": [], "tool_choice": "none"} + ) + assert request.tool_choice == "none" + + +def test_responses_request_empty_tools_tool_choice_auto(): + request = ResponsesRequest.model_validate( + {"input": "Hello", "model": "test-model", "tools": [], "tool_choice": "auto"} + ) + assert request.tool_choice == "none" + + +@pytest.mark.parametrize( + "tool_choice", + [ + {"type": "function"}, + {"type": "function", "name": ""}, + ], +) +def test_responses_request_named_tool_choice_missing_name(tool_choice): + with pytest.raises(ValidationError, match="not found in 'tools' parameter"): + ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [SAMPLE_TOOL], + "tool_choice": tool_choice, + } + ) + + +def test_responses_request_empty_tools_named_tool_choice(): + with pytest.raises(ValidationError, match="not found in 'tools' parameter"): + ResponsesRequest.model_validate( + { + "input": "Hello", + "model": "test-model", + "tools": [], + "tool_choice": NAMED_TOOL_CHOICE, + } + ) diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 79f5894fb91..96876e3f00f 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -492,6 +492,46 @@ class ResponsesRequest(OpenAIBaseModel): data["input"] = processed_input return data + @model_validator(mode="before") + @classmethod + def check_tool_usage(cls, data): + if not isinstance(data, dict): + return data + + tools = data.get("tools") + tool_choice = data.get("tool_choice", "auto") + has_tools = tools is not None and len(tools) > 0 + is_named_tool_choice = ( + isinstance(tool_choice, dict) and tool_choice.get("type") == "function" + ) + + if not has_tools: + if tool_choice in ("auto", "none"): + data["tool_choice"] = "none" + elif tool_choice == "required": + raise VLLMValidationError( + "Tool choice 'required' must be specified with 'tools' parameter.", + parameter="tool_choice", + ) + elif is_named_tool_choice: + raise VLLMValidationError( + "Tool choice 'function' not found in 'tools' parameter.", + parameter="tool_choice", + ) + elif is_named_tool_choice and tools is not None: + tool_name = tool_choice.get("name") + tool_names = { + t.get("name") if isinstance(t, dict) else getattr(t, "name", None) + for t in tools + } + if not tool_name or tool_name not in tool_names: + raise VLLMValidationError( + "Tool choice 'function' not found in 'tools' parameter.", + parameter="tool_choice", + ) + + return data + class ResponsesResponse(OpenAIBaseModel): id: str = Field(default_factory=lambda: f"resp_{random_uuid()}") diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 6af25c9bcce..6d018f9c5b5 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -718,9 +718,10 @@ class OpenAIServingResponses(OpenAIServing): request: ResponsesRequest, prev_response: ResponsesResponse | None, ): - if request.tool_choice != "auto": + if request.tool_choice not in ("auto", "none"): raise NotImplementedError( - "Only 'auto' tool_choice is supported in response API with Harmony" + "Only 'auto' or 'none' tool_choice is supported " + "in response API with Harmony" ) arrival_time = time.time() From fe57be7809672e5c4d100b55ce8649dd34d3bbc0 Mon Sep 17 00:00:00 2001 From: Shanshan Shen <467638484@qq.com> Date: Thu, 23 Apr 2026 13:46:14 +0800 Subject: [PATCH 058/153] [MM][CG] Support `--enable-vit-cuda-graph` option for VLM examples (#40580) Signed-off-by: shen-shanshan <467638484@qq.com> --- examples/offline_inference/vision_language.py | 40 ++++++++++++++++--- vllm/model_executor/models/qwen3_vl.py | 6 ++- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/examples/offline_inference/vision_language.py b/examples/offline_inference/vision_language.py index a5d2d7f41d8..cfeda8804a0 100755 --- a/examples/offline_inference/vision_language.py +++ b/examples/offline_inference/vision_language.py @@ -2463,6 +2463,12 @@ MODELS_NEED_VIDEO_METADATA = [ ] +MODELS_SUPPORT_VIT_CUDA_GRAPH = [ + "qwen3_vl", + "qwen3_vl_moe", +] + + def get_multi_modal_input(args): """ return { @@ -2575,6 +2581,29 @@ def apply_image_repeat( return inputs, inputs_with_empty_media +def maybe_add_vit_cuda_graph_compilation_config(args, engine_args): + model = args.model_type + modality = args.modality + enable_vit_cuda_graph = args.enable_vit_cuda_graph + + if enable_vit_cuda_graph and model in MODELS_SUPPORT_VIT_CUDA_GRAPH: + if modality == "image" or modality == "video": + vision_items_per_batch = 1 + elif modality == "image+video": + vision_items_per_batch = 2 + else: + raise ValueError( + f"modality={modality} is not supported for vit cuda graph." + ) + + engine_args.compilation_config = { + "cudagraph_mm_encoder": True, + "encoder_cudagraph_max_vision_items_per_batch": vision_items_per_batch, + } + + return engine_args + + @contextmanager def time_counter(enable: bool): if enable: @@ -2625,33 +2654,28 @@ def parse_args(): default=0, help="Set the seed when initializing `vllm.LLM`.", ) - parser.add_argument( "--image-repeat-prob", type=float, default=None, help="Simulates the hit-ratio for multi-modal preprocessor cache (if enabled)", ) - parser.add_argument( "--disable-mm-processor-cache", action="store_true", help="If True, disables caching of multi-modal processor.", ) - parser.add_argument( "--time-generate", action="store_true", help="If True, then print the total generate() call time", ) - parser.add_argument( "--use-different-prompt-per-request", action="store_true", help="If True, then use different prompt (with the same multi-modal " "data) for each request.", ) - parser.add_argument( "--verify-mm-cache-hit-with-uuids", action="store_true", @@ -2665,6 +2689,11 @@ def parse_args(): default=None, help="Tensor parallel size to override the model's default setting. ", ) + parser.add_argument( + "--enable-vit-cuda-graph", + action="store_true", + help="If True, will enable vit cuda graph capture and replay for the model.", + ) return parser.parse_args() @@ -2698,6 +2727,7 @@ def main(args): engine_args.mm_processor_cache_gb = mm_processor_cache_gb if args.tensor_parallel_size is not None: engine_args.tensor_parallel_size = args.tensor_parallel_size + engine_args = maybe_add_vit_cuda_graph_compilation_config(args, engine_args) llm = LLM.from_engine_args(engine_args) # Don't want to check the flag multiple times, so just hijack `prompts`. diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 7f110a4c2d0..f060a700f86 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -1802,7 +1802,11 @@ class Qwen3VLForConditionalGeneration( # spatial_merge_size=2 → 8x8 = 64 tokens min_budget = 64 # Max: capped by max_num_batched_tokens - max_budget = vllm_config.scheduler_config.max_num_batched_tokens + # TODO(shen-shanshan): the max_budget auto-infer needs to be optimized later. + max_budget = min( + vllm_config.scheduler_config.max_num_batched_tokens, + self.model_config.max_model_len, + ) return (min_budget, max_budget) def _get_pixel_values_by_modality( From 9c2492e501d91d5c69a163084123a06fd9ce25d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 23 Apr 2026 09:42:23 +0200 Subject: [PATCH 059/153] [Misc] Support Human-readable (k/K/m/M..) json cli arg (#40473) Signed-off-by: NickLucche --- tests/engine/test_arg_utils.py | 30 +++++++++++ vllm/engine/arg_utils.py | 94 ++++++++++------------------------ vllm/utils/argparse_utils.py | 73 +++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 67 deletions(-) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 9f03078a459..bf3b400d9d7 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -12,6 +12,7 @@ from pydantic import Field from vllm.config import AttentionConfig, CompilationConfig, config from vllm.engine.arg_utils import ( EngineArgs, + _expand_json_human_readable_numbers, contains_type, get_kwargs, get_type, @@ -563,3 +564,32 @@ def test_ir_op_priority(): ir_op_priority=ir_op_priority, kernel_config=KernelConfig(ir_op_priority=ir_op_priority), ).create_engine_config() + + +@pytest.mark.parametrize( + ("input_json", "expected_json"), + [ + # Decimal suffixes (lowercase) + ('{"x": 80g}', '{"x": 80000000000}'), + ('{"x": 1k}', '{"x": 1000}'), + ('{"x": 5m}', '{"x": 5000000}'), + ('{"x": 2t}', '{"x": 2000000000000}'), + # Binary suffixes (uppercase) + ('{"x": 1K}', f'{{"x": {2**10}}}'), + ('{"x": 1G}', f'{{"x": {2**30}}}'), + # Decimal values + ('{"x": 1.5g}', '{"x": 1500000000}'), + # Quoted strings must NOT be modified + ('{"my_key": 80g}', '{"my_key": 80000000000}'), + ('{"name": "80g"}', '{"name": "80g"}'), + ('{"model_name": "foo_bar"}', '{"model_name": "foo_bar"}'), + # Multiple values + ('{"a": 1k, "b": 2m}', '{"a": 1000, "b": 2000000}'), + # Plain numbers are untouched + ('{"x": 42}', '{"x": 42}'), + # Nested JSON + ('{"outer": {"inner": 10g}}', '{"outer": {"inner": 10000000000}}'), + ], +) +def test_expand_json_human_readable_numbers(input_json, expected_json): + assert _expand_json_human_readable_numbers(input_json) == expected_json diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index e6528849b21..ef3a9a982a5 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -105,7 +105,11 @@ from vllm.transformers_utils.config import ( from vllm.transformers_utils.gguf_utils import is_gguf from vllm.transformers_utils.repo_utils import get_model_path from vllm.transformers_utils.utils import is_cloud_storage -from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.argparse_utils import ( + FlexibleArgumentParser, + human_readable_int, + human_readable_int_or_auto, +) from vllm.utils.mem_constants import GiB_bytes from vllm.utils.network_utils import get_ip from vllm.utils.torch_utils import resolve_kv_cache_dtype_string @@ -256,6 +260,28 @@ def _maybe_add_docs_url(cls: Any) -> str: return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{cls.__name__}" +def _expand_json_human_readable_numbers(val: str) -> str: + """Expand human-readable number suffixes in a JSON string. + + Based on :func:`human_readable_int` so that the ``k/m/g/t`` (decimal) and + ``K/M/G/T`` (binary) conventions work out the box. + Also works inside JSON config arguments such + as ``--kv-transfer-config '{"cpu_bytes_to_use": 80m}'``. + + Only bare (unquoted) tokens are replaced so that JSON string values + like ``"model_name"`` are never modified. + """ + # Split on quoted strings so we only touch non-string regions. + parts = re.split(r'("(?:[^"\\]|\\.)*")', val) + for i in range(0, len(parts), 2): # even indices = outside strings + parts[i] = re.sub( + r"\b\d+(?:\.\d+)?[kKmMgGtT]\b", + lambda m: str(human_readable_int(m.group())), + parts[i], + ) + return "".join(parts) + + @functools.lru_cache(maxsize=30) def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: # Save time only getting attr docs if we're generating help text @@ -301,6 +327,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, dict[str, Any]]: def parse_dataclass(val: str, cls=dataclass_cls) -> Any: try: + val = _expand_json_human_readable_numbers(val) return TypeAdapter(cls).validate_json(val) except ValidationError as e: raise argparse.ArgumentTypeError(repr(e)) from e @@ -2419,68 +2446,3 @@ def _raise_unsupported_error(feature_name: str): f"remove {feature_name} from your config." ) raise NotImplementedError(msg) - - -def human_readable_int(value: str) -> int: - """Parse human-readable integers like '1k', '2M', etc. - Including decimal values with decimal multipliers. - - Examples: - - '1k' -> 1,000 - - '1K' -> 1,024 - - '25.6k' -> 25,600 - """ - value = value.strip() - - match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value) - if match: - decimal_multiplier = { - "k": 10**3, - "m": 10**6, - "g": 10**9, - "t": 10**12, - } - binary_multiplier = { - "K": 2**10, - "M": 2**20, - "G": 2**30, - "T": 2**40, - } - - number, suffix = match.groups() - if suffix in decimal_multiplier: - mult = decimal_multiplier[suffix] - return int(float(number) * mult) - elif suffix in binary_multiplier: - mult = binary_multiplier[suffix] - # Do not allow decimals with binary multipliers - try: - return int(number) * mult - except ValueError as e: - raise argparse.ArgumentTypeError( - "Decimals are not allowed " - f"with binary suffixes like {suffix}. Did you mean to use " - f"{number}{suffix.lower()} instead?" - ) from e - - # Regular plain number. - return int(value) - - -def human_readable_int_or_auto(value: str) -> int: - """Parse human-readable integers like '1k', '2M', etc. - Including decimal values with decimal multipliers. - Also accepts -1 or 'auto' as a special value for auto-detection. - - Examples: - - '1k' -> 1,000 - - '1K' -> 1,024 - - '25.6k' -> 25,600 - - '-1' or 'auto' -> -1 (special value for auto-detection) - """ - value = value.strip() - - if value == "-1" or value.lower() == "auto": - return -1 - - return human_readable_int(value) diff --git a/vllm/utils/argparse_utils.py b/vllm/utils/argparse_utils.py index 04c70bf79e6..84c85375719 100644 --- a/vllm/utils/argparse_utils.py +++ b/vllm/utils/argparse_utils.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Argument parsing utilities for vLLM.""" +import argparse import json import sys import textwrap @@ -25,6 +26,71 @@ from vllm.logger import init_logger logger = init_logger(__name__) +def human_readable_int(value: str) -> int: + """Parse human-readable integers like '1k', '2M', etc. + Including decimal values with decimal multipliers. + + Examples: + - '1k' -> 1,000 + - '1K' -> 1,024 + - '25.6k' -> 25,600 + """ + value = value.strip() + + match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value) + if match: + decimal_multiplier = { + "k": 10**3, + "m": 10**6, + "g": 10**9, + "t": 10**12, + } + binary_multiplier = { + "K": 2**10, + "M": 2**20, + "G": 2**30, + "T": 2**40, + } + + number, suffix = match.groups() + if suffix in decimal_multiplier: + mult = decimal_multiplier[suffix] + return int(float(number) * mult) + elif suffix in binary_multiplier: + mult = binary_multiplier[suffix] + # Do not allow decimals with binary multipliers + try: + return int(number) * mult + except ValueError as e: + raise argparse.ArgumentTypeError( + "Decimals are not allowed " + f"with binary suffixes like {suffix}. Did you mean to use " + f"{number}{suffix.lower()} instead?" + ) from e + + # Regular plain number. + return int(value) + + +def human_readable_int_or_auto(value: str) -> int: + """Parse human-readable integers like '1k', '2M', etc. + Including decimal values with decimal multipliers. + Also accepts -1 or 'auto' as a special value for auto-detection. + + Examples: + - '1k' -> 1,000 + - '1K' -> 1,024 + - '25.6k' -> 25,600 + - '-1' or 'auto' -> -1 (special value for auto-detection) + """ + value = value.strip() + + if value == "-1" or value.lower() == "auto": + return -1 + + return human_readable_int(value) + + class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter): """SortedHelpFormatter that sorts arguments by their option strings.""" @@ -338,7 +404,12 @@ class FlexibleArgumentParser(ArgumentParser): try: value = json.loads(value_str) except json.decoder.JSONDecodeError: - value = value_str + # Support human-readable suffixes (e.g. 1k, 80g) for + # dotted config args like --config.field 80g + try: + value = human_readable_int(value_str) # type: ignore[assignment] + except (ValueError, ArgumentTypeError): + value = value_str # Merge all values with the same key into a single dict arg_dict = create_nested_dict(keys, value) From 3ed5231c6a7a9042a6ac4000e569ad2d85a21b9a Mon Sep 17 00:00:00 2001 From: Shengqi Chen Date: Thu, 23 Apr 2026 15:51:28 +0800 Subject: [PATCH 060/153] [Build] Switch default CUDA to 13.0, update CUDA architecture lists, clean up stale build-args (#39878) Signed-off-by: Shengqi Chen Co-authored-by: Claude Opus 4.6 (1M context) --- .buildkite/release-pipeline.yaml | 119 +++++++++--------- .buildkite/scripts/annotate-release.sh | 44 +++---- .buildkite/scripts/check-ray-compatibility.sh | 2 +- .../generate-and-upload-nightly-index.sh | 2 +- .github/workflows/scripts/build.sh | 2 +- CMakeLists.txt | 9 +- docker/Dockerfile | 4 +- docker/docker-bake.hcl | 2 - docker/versions.json | 2 +- vllm/envs.py | 4 +- 10 files changed, 98 insertions(+), 92 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index b3a6bb8ed4c..ee41ae2868e 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -1,3 +1,13 @@ +# CUDA architecture lists — following PyTorch RELEASE.md +# (https://github.com/pytorch/pytorch/blob/main/RELEASE.md) +# SM86 included for broader Ampere coverage; SM89 for marlin fp8 support +env: + CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX" + # aarch64 only architectures: 8.7 for Orin, 11.0 for Thor (since CUDA 13) + CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX" + CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0" + CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0" + steps: - input: "Provide Release version here" id: input-release-version @@ -14,12 +24,10 @@ steps: agents: queue: arm64_cpu_queue_release commands: - # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here: - # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31" env: DOCKER_BUILDKIT: "1" @@ -29,9 +37,7 @@ steps: agents: queue: arm64_cpu_queue_release commands: - # #NOTE: torch_cuda_arch_list is derived from upstream PyTorch build files here: - # https://github.com/pytorch/pytorch/blob/main/.ci/aarch64_linux/aarch64_ci_build.sh#L7 - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" @@ -57,7 +63,7 @@ steps: agents: queue: cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_31" @@ -70,7 +76,7 @@ steps: agents: queue: cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" @@ -108,96 +114,95 @@ steps: depends_on: block-build-release-images allow_dependency_failure: true steps: - - label: "Build release image - x86_64 - CUDA 12.9" + - label: "Build release image - x86_64 - CUDA 13.0" depends_on: ~ id: build-release-image-x86 agents: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" # re-tag to default image tag and push, just in case arm64 build fails - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - label: "Build release image - aarch64 - CUDA 12.9" + - label: "Build release image - aarch64 - CUDA 13.0" depends_on: ~ id: build-release-image-arm64 agents: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - - label: "Build release image - x86_64 - CUDA 13.0" + - label: "Build release image - x86_64 - CUDA 12.9" depends_on: ~ - id: build-release-image-x86-cuda-13-0 + id: build-release-image-x86-cuda-12-9 agents: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130" + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" # re-tag to default image tag and push, just in case arm64 build fails - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130" - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130" + - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - - label: "Build release image - aarch64 - CUDA 13.0" + - label: "Build release image - aarch64 - CUDA 12.9" depends_on: ~ - id: build-release-image-arm64-cuda-13-0 + id: build-release-image-arm64-cuda-12-9 agents: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - # compute capability 12.0 for RTX-50 series / RTX PRO 6000 Blackwell, 12.1 for DGX Spark - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130 --target vllm-openai --progress plain -f docker/Dockerfile ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130" + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" - - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04" + - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ id: build-release-image-x86-ubuntu2404 agents: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04" + - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ id: build-release-image-arm64-ubuntu2404 agents: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0' --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" + - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04" depends_on: ~ - id: build-release-image-x86-cuda-13-0-ubuntu2404 + id: build-release-image-x86-cuda-12-9-ubuntu2404 agents: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404" - - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404" - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404" + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" + - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04" + - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04" depends_on: ~ - id: build-release-image-arm64-cuda-13-0-ubuntu2404 + id: build-release-image-arm64-cuda-12-9-ubuntu2404 agents: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg FLASHINFER_AOT_COMPILE=true --build-arg torch_cuda_arch_list='8.7 8.9 9.0 10.0+PTX 12.0 12.1' --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu130-ubuntu2404" + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg INSTALL_KV_CONNECTORS=true --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" - block: "Build release image for x86_64 CPU" key: block-cpu-release-image-build @@ -238,7 +243,7 @@ steps: - group: "Publish release images" key: "publish-release-images" steps: - - label: "Create multi-arch manifest - CUDA 12.9" + - label: "Create multi-arch manifest - CUDA 13.0" depends_on: - build-release-image-x86 - build-release-image-arm64 @@ -250,7 +255,7 @@ steps: - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - label: "Annotate release workflow - CUDA 12.9" + - label: "Annotate release workflow - CUDA 13.0" depends_on: - create-multi-arch-manifest id: annotate-release-workflow @@ -259,19 +264,19 @@ steps: commands: - "bash .buildkite/scripts/annotate-release.sh" - - label: "Create multi-arch manifest - CUDA 13.0" + - label: "Create multi-arch manifest - CUDA 12.9" depends_on: - - build-release-image-x86-cuda-13-0 - - build-release-image-arm64-cuda-13-0 - id: create-multi-arch-manifest-cuda-13-0 + - build-release-image-x86-cuda-12-9 + - build-release-image-arm64-cuda-12-9 + id: create-multi-arch-manifest-cuda-12-9 agents: queue: small_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04" + - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" depends_on: - build-release-image-x86-ubuntu2404 - build-release-image-arm64-ubuntu2404 @@ -283,17 +288,17 @@ steps: - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" + - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04" depends_on: - - build-release-image-x86-cuda-13-0-ubuntu2404 - - build-release-image-arm64-cuda-13-0-ubuntu2404 - id: create-multi-arch-manifest-cuda-13-0-ubuntu2404 + - build-release-image-x86-cuda-12-9-ubuntu2404 + - build-release-image-arm64-cuda-12-9-ubuntu2404 + id: create-multi-arch-manifest-cuda-12-9-ubuntu2404 agents: queue: small_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu130-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu130-ubuntu2404 --amend" - - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu130-ubuntu2404" + - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend" + - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - label: "Publish nightly multi-arch image to DockerHub" depends_on: @@ -313,16 +318,16 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" - - label: "Publish nightly multi-arch image to DockerHub - CUDA 13.0" + - label: "Publish nightly multi-arch image to DockerHub - CUDA 12.9" depends_on: - - create-multi-arch-manifest-cuda-13-0 + - create-multi-arch-manifest-cuda-12-9 if: build.env("NIGHTLY") == "1" agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/push-nightly-builds.sh cu130" + - "bash .buildkite/scripts/push-nightly-builds.sh cu129" # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu130-nightly-" + - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu129-nightly-" plugins: - docker-login#v3.0.0: username: vllmbot diff --git a/.buildkite/scripts/annotate-release.sh b/.buildkite/scripts/annotate-release.sh index 2da9db2f2e5..6f41d1cdda4 100755 --- a/.buildkite/scripts/annotate-release.sh +++ b/.buildkite/scripts/annotate-release.sh @@ -13,12 +13,12 @@ ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key) buildkite-agent annotate --style 'info' --context 'release-workflow' << EOF To download the wheel (by commit): \`\`\` -aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_x86_64.whl . -aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_31_aarch64.whl . +aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_35_x86_64.whl . +aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}-cp38-abi3-manylinux_2_35_aarch64.whl . -(Optional) For CUDA 13.0: -aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_x86_64.whl . -aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu130-cp38-abi3-manylinux_2_35_aarch64.whl . +(Optional) For CUDA 12.9: +aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux_2_31_x86_64.whl . +aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cu129-cp38-abi3-manylinux_2_31_aarch64.whl . (Optional) For CPU: aws s3 cp s3://vllm-wheels/${BUILDKITE_COMMIT}/vllm-${RELEASE_VERSION}+cpu-cp38-abi3-manylinux_2_35_x86_64.whl . @@ -33,8 +33,8 @@ To download and upload the image: docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130 -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130 +docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu129 +docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu129 docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm docker pull public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION} @@ -50,11 +50,11 @@ docker tag vllm/vllm-openai:x86_64 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 docker push vllm/vllm-openai:latest-x86_64 docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64 -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu130 vllm/vllm-openai:x86_64-cu130 -docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:latest-x86_64-cu130 -docker tag vllm/vllm-openai:x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130 -docker push vllm/vllm-openai:latest-x86_64-cu130 -docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130 +docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-x86_64-cu129 vllm/vllm-openai:x86_64-cu129 +docker tag vllm/vllm-openai:x86_64-cu129 vllm/vllm-openai:latest-x86_64-cu129 +docker tag vllm/vllm-openai:x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 +docker push vllm/vllm-openai:latest-x86_64-cu129 +docker push vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64 vllm/vllm-openai:aarch64 docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:latest-aarch64 @@ -62,11 +62,11 @@ docker tag vllm/vllm-openai:aarch64 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64 docker push vllm/vllm-openai:latest-aarch64 docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64 -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu130 vllm/vllm-openai:aarch64-cu130 -docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:latest-aarch64-cu130 -docker tag vllm/vllm-openai:aarch64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130 -docker push vllm/vllm-openai:latest-aarch64-cu130 -docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130 +docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-aarch64-cu129 vllm/vllm-openai:aarch64-cu129 +docker tag vllm/vllm-openai:aarch64-cu129 vllm/vllm-openai:latest-aarch64-cu129 +docker tag vllm/vllm-openai:aarch64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129 +docker push vllm/vllm-openai:latest-aarch64-cu129 +docker push vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129 ## ROCm @@ -104,11 +104,11 @@ docker manifest create vllm/vllm-openai:v${RELEASE_VERSION} vllm/vllm-openai:v${ docker manifest push vllm/vllm-openai:latest docker manifest push vllm/vllm-openai:v${RELEASE_VERSION} -docker manifest rm vllm/vllm-openai:latest-cu130 -docker manifest create vllm/vllm-openai:latest-cu130 vllm/vllm-openai:latest-x86_64-cu130 vllm/vllm-openai:latest-aarch64-cu130 -docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu130 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu130 -docker manifest push vllm/vllm-openai:latest-cu130 -docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu130 +docker manifest rm vllm/vllm-openai:latest-cu129 +docker manifest create vllm/vllm-openai:latest-cu129 vllm/vllm-openai:latest-x86_64-cu129 vllm/vllm-openai:latest-aarch64-cu129 +docker manifest create vllm/vllm-openai:v${RELEASE_VERSION}-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-x86_64-cu129 vllm/vllm-openai:v${RELEASE_VERSION}-aarch64-cu129 +docker manifest push vllm/vllm-openai:latest-cu129 +docker manifest push vllm/vllm-openai:v${RELEASE_VERSION}-cu129 docker manifest rm vllm/vllm-openai-cpu:latest || true docker manifest create vllm/vllm-openai-cpu:latest vllm/vllm-openai-cpu:latest-x86_64 vllm/vllm-openai-cpu:latest-arm64 diff --git a/.buildkite/scripts/check-ray-compatibility.sh b/.buildkite/scripts/check-ray-compatibility.sh index 1572fe94168..b056d4403db 100644 --- a/.buildkite/scripts/check-ray-compatibility.sh +++ b/.buildkite/scripts/check-ray-compatibility.sh @@ -29,7 +29,7 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then TORCH_INDEX_URL="" fi else - TORCH_INDEX_URL="https://download.pytorch.org/whl/cu129" + TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130" fi echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}" diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/scripts/generate-and-upload-nightly-index.sh index 7cef252c607..88c4f517313 100755 --- a/.buildkite/scripts/generate-and-upload-nightly-index.sh +++ b/.buildkite/scripts/generate-and-upload-nightly-index.sh @@ -9,7 +9,7 @@ set -ex BUCKET="vllm-wheels" INDICES_OUTPUT_DIR="indices" -DEFAULT_VARIANT_ALIAS="cu129" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py +DEFAULT_VARIANT_ALIAS="cu130" # align with vLLM_MAIN_CUDA_VERSION in vllm/envs.py PYTHON="${PYTHON_PROG:-python3}" # try to read from env var, otherwise use python3 SUBPATH=$BUILDKITE_COMMIT S3_COMMIT_PREFIX="s3://$BUCKET/$SUBPATH/" diff --git a/.github/workflows/scripts/build.sh b/.github/workflows/scripts/build.sh index 30c63ee0da9..5bb9f719680 100644 --- a/.github/workflows/scripts/build.sh +++ b/.github/workflows/scripts/build.sh @@ -14,7 +14,7 @@ $python_executable -m pip install -r requirements/build/cuda.txt -r requirements # Limit the number of parallel jobs to avoid OOM export MAX_JOBS=1 # Make sure release wheels are built for the following architectures -export TORCH_CUDA_ARCH_LIST="7.0 7.5 8.0 8.6 8.9 9.0+PTX" +export TORCH_CUDA_ARCH_LIST="7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX" bash tools/check_repo.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index fbc335b7f8e..8f859c9cc40 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,12 +94,15 @@ find_package(Torch REQUIRED) # This check must happen after find_package(Torch) because that's when CMAKE_CUDA_COMPILER_VERSION gets defined if(DEFINED CMAKE_CUDA_COMPILER_VERSION AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0) - set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0;12.1") + # starting from CUDA 12.9 and Blackwell (10.0), we use family-specific targets (10.0f, 12.0f, etc) + # to support the whole generation without specifying all sub-architectures + # see: https://developer.nvidia.com/blog/nvidia-blackwell-and-nvidia-cuda-12-9-introduce-family-specific-architecture-features/ + set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;11.0;12.0") elseif(DEFINED CMAKE_CUDA_COMPILER_VERSION AND CMAKE_CUDA_COMPILER_VERSION VERSION_GREATER_EQUAL 12.8) - set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;12.0;12.1") + set(CUDA_SUPPORTED_ARCHS "7.5;8.0;8.6;8.7;8.9;9.0;10.0;10.1;10.3;12.0;12.1") else() - set(CUDA_SUPPORTED_ARCHS "7.0;7.2;7.5;8.0;8.6;8.7;8.9;9.0") + set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.7;8.9;9.0") endif() # diff --git a/docker/Dockerfile b/docker/Dockerfile index 50cceb892ac..d76a2e986b7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -188,7 +188,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Explicitly set the list to avoid issues with torch 2.2 # See https://github.com/pytorch/pytorch/pull/123243 # From versions.json: .torch.cuda_arch_list -ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0' +ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### BUILD BASE IMAGE #################### @@ -765,7 +765,7 @@ ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL ENV UV_HTTP_TIMEOUT=500 # install kv_connectors if requested -ARG torch_cuda_arch_list='7.0 7.5 8.0 8.9 9.0 10.0 12.0' +ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=requirements/kv_connectors.txt,target=/tmp/kv_connectors.txt,ro \ diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index e1c2fbba63a..055287ca3be 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -88,7 +88,6 @@ target "test-ubuntu2404" { args = { UBUNTU_VERSION = "24.04" GDRCOPY_OS_VERSION = "Ubuntu24_04" - FLASHINFER_AOT_COMPILE = "true" } output = ["type=docker"] } @@ -100,7 +99,6 @@ target "openai-ubuntu2404" { args = { UBUNTU_VERSION = "24.04" GDRCOPY_OS_VERSION = "Ubuntu24_04" - FLASHINFER_AOT_COMPILE = "true" } output = ["type=docker"] } diff --git a/docker/versions.json b/docker/versions.json index 52a2149b2f0..f4e05914afa 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -32,7 +32,7 @@ "default": "false" }, "TORCH_CUDA_ARCH_LIST": { - "default": "7.0 7.5 8.0 8.9 9.0 10.0 12.0" + "default": "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX" }, "MAX_JOBS": { "default": "2" diff --git a/vllm/envs.py b/vllm/envs.py index faafe93ac5f..66116e4ee44 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -79,7 +79,7 @@ if TYPE_CHECKING: VLLM_MEDIA_CONNECTOR: str = "http" VLLM_MM_HASHER_ALGORITHM: str = "blake3" VLLM_TARGET_DEVICE: str = "cuda" - VLLM_MAIN_CUDA_VERSION: str = "12.9" + VLLM_MAIN_CUDA_VERSION: str = "13.0" VLLM_FLOAT32_MATMUL_PRECISION: Literal["highest", "high", "medium"] = "highest" VLLM_BATCH_INVARIANT: bool = False MAX_JOBS: str | None = None @@ -493,7 +493,7 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_TARGET_DEVICE": lambda: os.getenv("VLLM_TARGET_DEVICE", "cuda").lower(), # Main CUDA version of vLLM. This follows PyTorch but can be overridden. "VLLM_MAIN_CUDA_VERSION": lambda: ( - os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "12.9" + os.getenv("VLLM_MAIN_CUDA_VERSION", "").lower() or "13.0" ), # Controls PyTorch float32 matmul precision mode within vLLM workers. # Valid options mirror torch.set_float32_matmul_precision From 4a79262e0f560c50a9831ee672a853e598047ee5 Mon Sep 17 00:00:00 2001 From: liuzhenwei Date: Thu, 23 Apr 2026 16:22:28 +0800 Subject: [PATCH 061/153] [UT][Hardware] let torchrun example tests use the default backend (#39879) Signed-off-by: zhenwei-intel --- tests/distributed/test_torchrun_example.py | 3 ++- tests/distributed/test_torchrun_example_moe.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/distributed/test_torchrun_example.py b/tests/distributed/test_torchrun_example.py index af9c76d9c7e..f56d037fa54 100644 --- a/tests/distributed/test_torchrun_example.py +++ b/tests/distributed/test_torchrun_example.py @@ -10,7 +10,8 @@ import torch.distributed as dist from vllm import LLM, SamplingParams from vllm.distributed.parallel_state import get_world_group -dist.init_process_group(backend="gloo") +# Let PyTorch choose the WORLD backend for the current device type. +dist.init_process_group() # Create prompts prompts = [ diff --git a/tests/distributed/test_torchrun_example_moe.py b/tests/distributed/test_torchrun_example_moe.py index c0437d9b930..8c1d00561b1 100644 --- a/tests/distributed/test_torchrun_example_moe.py +++ b/tests/distributed/test_torchrun_example_moe.py @@ -10,7 +10,8 @@ import torch.distributed as dist from vllm import LLM, SamplingParams from vllm.distributed.parallel_state import get_tp_group, get_world_group -dist.init_process_group(backend="gloo") +# Let PyTorch choose the WORLD backend for the current device type. +dist.init_process_group() # Create prompts prompts = [ From 4b7869d6bc64f5b124e2403891b4c2e29713bbf5 Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Thu, 23 Apr 2026 10:32:04 +0200 Subject: [PATCH 062/153] [ROCm] Add gfx1102/gfx1103 support (#40037) Signed-off-by: Matthias Gehre --- CMakeLists.txt | 2 +- csrc/rocm/attention.cu | 13 ++----------- csrc/rocm/skinny_gemms.cu | 41 +++++++++++++++++---------------------- 3 files changed, 21 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f859c9cc40..e79c5b9f912 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,7 +37,7 @@ install(CODE "set(CMAKE_INSTALL_LOCAL_ONLY TRUE)" ALL_COMPONENTS) set(PYTHON_SUPPORTED_VERSIONS "3.10" "3.11" "3.12" "3.13") # Supported AMD GPU architectures. -set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201") +set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx942;gfx950;gfx1030;gfx1100;gfx1101;gfx1102;gfx1103;gfx1150;gfx1151;gfx1152;gfx1153;gfx1200;gfx1201") # ROCm installation prefix. Default to /opt/rocm but allow override via # -DROCM_PATH=/your/rocm/path when invoking cmake. diff --git a/csrc/rocm/attention.cu b/csrc/rocm/attention.cu index a339c5641bb..9e6c0726d19 100644 --- a/csrc/rocm/attention.cu +++ b/csrc/rocm/attention.cu @@ -40,15 +40,6 @@ using __hip_fp8_e5m2 = __hip_fp8_e5m2_fnuz; #define __HIP__FP8MFMA__ #endif -#if defined(__HIPCC__) && (defined(__gfx1100__) || defined(__gfx1101__) || \ - defined(__gfx1150__) || defined(__gfx1151__)) - #define __HIP__GFX11__ -#endif - -#if defined(__HIPCC__) && (defined(__gfx1200__) || defined(__gfx1201__)) - #define __HIP__GFX12__ -#endif - #if defined(NDEBUG) #undef NDEBUG #include @@ -1629,7 +1620,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel( } } -#elif defined(__HIP__GFX11__) +#elif defined(__GFX11__) using floatx8 = __attribute__((__vector_size__(8 * sizeof(float)))) float; @@ -2388,7 +2379,7 @@ __launch_bounds__(NUM_THREADS) void paged_attention_ll4mi_reduce_kernel( out_ptr[threadIdx.x] = from_float(acc); } -#elif defined(__HIP__GFX12__) +#elif defined(__GFX12__) using floatx8 = __attribute__((__vector_size__(8 * sizeof(float)))) float; diff --git a/csrc/rocm/skinny_gemms.cu b/csrc/rocm/skinny_gemms.cu index 60e10e53391..3342db37be9 100644 --- a/csrc/rocm/skinny_gemms.cu +++ b/csrc/rocm/skinny_gemms.cu @@ -26,16 +26,11 @@ #define __HIP__GFX9__ #endif -#if defined(__HIPCC__) && \ - (defined(__gfx1100__) || defined(__gfx1101__) || defined(__gfx1150__) || \ - defined(__gfx1151__) || defined(__gfx1200__) || defined(__gfx1201__)) +// Combined RDNA macro (gfx11 + gfx12) - both use 32-wide wavefronts +#if defined(__GFX11__) || defined(__GFX12__) #define __HIP__GFX1X__ #endif -#if defined(__HIPCC__) && (defined(__gfx1200__) || defined(__gfx1201__)) - #define __HIP__GFX12__ -#endif - #if defined(__HIPCC__) && (defined(__gfx942__) || defined(__gfx950__)) #define __HIP__MI3XX__ #endif @@ -1845,7 +1840,7 @@ torch::Tensor wvSplitKrc(const at::Tensor& in_a, const at::Tensor& in_b, return out_c; } -#if defined(__HIP__MI3XX__) || defined(__HIP__GFX12__) +#if defined(__HIP__MI3XX__) || defined(__GFX12__) template __global__ void __launch_bounds__(WvPrGrp* THRDS) @@ -1893,7 +1888,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) float sB = *s_B; while (m < M) { - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ // gfx12: per-lane scalar accumulation via v_dot4_f32_fp8_fp8 float sum[N][YTILE] = {}; #else @@ -1931,7 +1926,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) #pragma unroll for (uint32_t k2 = 0; k2 < UNRL; k2++) { for (uint32_t n = 0; n < N; n++) { - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ // gfx12: 4 x dot4 per A_CHUNK=16 bytes (4 FP8 per dot4) for (int y = 0; y < YTILE; ++y) { #pragma unroll @@ -1955,7 +1950,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) } // Final reduction - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ // gfx12 wave32: DPP row_shr within 16-lane rows + cross-row shuffle for (int n = 0; n < N; n++) { for (int y = 0; y < YTILE; y++) { @@ -1993,7 +1988,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) #endif const bool writeback_lane = - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ threadIdx.x == (THRDS - 1); #else threadIdx.x == 0; @@ -2009,7 +2004,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) for (int n = 0; n < N; n++) { for (int y = 0; y < YTILE; y++) { if (y + m >= M) break; // To avoid mem access fault. - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ float result = sum[n][y] * sA * sB; #else float result = sum[n][y][0] * sA * sB; @@ -2027,7 +2022,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) m += CuCount * _WvPrGrp * YTILE; } } -#else // !defined(__HIP__MI3XX__) && !defined(__HIP__GFX12__) +#else // !defined(__HIP__MI3XX__) && !defined(__GFX12__) template __global__ void wvSplitKQ_hf_sml_(const int K, const int Kap, const int Kbp, @@ -2039,9 +2034,9 @@ __global__ void wvSplitKQ_hf_sml_(const int K, const int Kap, const int Kbp, const int _WvPrGrp, const int CuCount) { UNREACHABLE_CODE } -#endif // defined(__HIP__MI3XX__) || defined(__HIP__GFX12__) +#endif // defined(__HIP__MI3XX__) || defined(__GFX12__) -#if defined(__HIP__MI3XX__) || defined(__HIP__GFX12__) +#if defined(__HIP__MI3XX__) || defined(__GFX12__) template __global__ void __launch_bounds__(WvPrGrp* THRDS) @@ -2088,7 +2083,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) float sB = *s_B; while (m < M) { - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ // gfx12: per-lane scalar accumulation via v_dot4_f32_fp8_fp8 float sum[N][YTILE] = {}; #else @@ -2128,7 +2123,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) #pragma unroll for (uint32_t k2 = 0; k2 < UNRL; k2++) { for (uint32_t n = 0; n < N; n++) { - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ // gfx12: 4 x dot4 per A_CHUNK=16 bytes (4 FP8 per dot4) for (int y = 0; y < YTILE; ++y) { #pragma unroll @@ -2152,7 +2147,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) } // Final reduction - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ // gfx12 wave32: DPP row_shr within 16-lane rows + cross-row shuffle for (int n = 0; n < N; n++) { for (int y = 0; y < YTILE; y++) { @@ -2190,7 +2185,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) #endif const bool writeback_lane = - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ threadIdx.x == (THRDS - 1); #else threadIdx.x == 0; @@ -2206,7 +2201,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) for (int n = 0; n < N; n++) { for (int y = 0; y < YTILE; y++) { if (y + m >= M) break; // To avoid mem access fault. - #ifdef __HIP__GFX12__ + #ifdef __GFX12__ float result = sum[n][y] * sA * sB; #else float result = sum[n][y][0] * sA * sB; @@ -2224,7 +2219,7 @@ __global__ void __launch_bounds__(WvPrGrp* THRDS) m += CuCount * _WvPrGrp * YTILE; } } -#else // !defined(__HIP__MI3XX__) && !defined(__HIP__GFX12__) +#else // !defined(__HIP__MI3XX__) && !defined(__GFX12__) template __global__ void wvSplitKQ_hf_(const int K, const int Kap, const int Kbp, @@ -2236,7 +2231,7 @@ __global__ void wvSplitKQ_hf_(const int K, const int Kap, const int Kbp, const int CuCount) { UNREACHABLE_CODE } -#endif // defined(__HIP__MI3XX__) || defined(__HIP__GFX12__) +#endif // defined(__HIP__MI3XX__) || defined(__GFX12__) void wvSplitKQ(const at::Tensor& in_b, const at::Tensor& in_a, const std::optional& in_bias, at::Tensor& out_c, From 2196bac1359a62a29f76936f37380235cc47f096 Mon Sep 17 00:00:00 2001 From: BadrBasowid <61441185+BadrBasowid@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:10:36 +0800 Subject: [PATCH 063/153] [Compilation] Refactor SiluMul activation+quant Fusion Pass (#39684) Signed-off-by: BadrBasowid --- tests/compile/fusions_e2e/conftest.py | 8 +- .../passes/test_silu_mul_quant_fusion.py | 2 +- .../passes/fusion/act_quant_fusion.py | 129 ++++++++---------- .../passes/fusion/rocm_aiter_fusion.py | 43 ++---- 4 files changed, 80 insertions(+), 102 deletions(-) diff --git a/tests/compile/fusions_e2e/conftest.py b/tests/compile/fusions_e2e/conftest.py index f7896728f9d..b4b1202d930 100644 --- a/tests/compile/fusions_e2e/conftest.py +++ b/tests/compile/fusions_e2e/conftest.py @@ -189,7 +189,7 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): # TODO: Remove log counting in unit tests # once all matchers implement VllmFusionPatternMatcherPass n_expected = tp_size * num_ranges_activated - if match_name != "attn_quant_fusion": + if match_name not in ("attn_quant_fusion", "act_quant_fusion"): assert len(log_matches) == n_expected, ( f"Could not find {n_expected} {match_name} " f"(found {len(log_matches)}) in:\n {log_holder.text}" @@ -250,6 +250,12 @@ def run_e2e_fusion_test(monkeypatch, caplog_mp_spawn): f"entries (SP took precedence), found: {log_matches}" ) + elif match_name == "act_quant_fusion": + actual_match = match_table.get("activation_quant_fusion_pass", 0) + assert actual_match == expected_matches * n_expected, ( + f"Could not find {expected_matches * n_expected} " + f"{match_name} (found {actual_match})." + ) elif match_name == "attn_quant_fusion": actual_match = match_table.get( "attn_quant_fusion", 0 diff --git a/tests/compile/passes/test_silu_mul_quant_fusion.py b/tests/compile/passes/test_silu_mul_quant_fusion.py index f3d800b2815..5f3f789fad9 100644 --- a/tests/compile/passes/test_silu_mul_quant_fusion.py +++ b/tests/compile/passes/test_silu_mul_quant_fusion.py @@ -168,7 +168,7 @@ class TestSiluMulGroupFp8QuantModel(torch.nn.Module): def forward(self, x): y = self.silu_and_mul(x) - x2 = self.w8a8_block_fp8_linear(y, self.w, self.wscale) + x2 = self.w8a8_block_fp8_linear(y) return x2 def ops_in_model_before(self): diff --git a/vllm/compilation/passes/fusion/act_quant_fusion.py b/vllm/compilation/passes/fusion/act_quant_fusion.py index 3a961cf5348..73234ec7920 100644 --- a/vllm/compilation/passes/fusion/act_quant_fusion.py +++ b/vllm/compilation/passes/fusion/act_quant_fusion.py @@ -1,16 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from abc import ABC, abstractmethod +import itertools from typing import Any import torch from torch._higher_order_ops.auto_functionalize import auto_functionalized -from torch._inductor.pattern_matcher import ( - PatternMatcherPass, - fwd_only, - register_replacement, -) from torch._ops import OpOverload from vllm.config import VllmConfig @@ -24,8 +19,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.platforms import current_platform -from ..inductor_pass import enable_fake_mode -from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass +from ..vllm_inductor_pass import VllmFusionPatternMatcherPass, VllmPatternReplacement from .matcher_utils import MatcherQuantFP8, MatcherSiluAndMul from .rms_quant_fusion import QUANT_OPS, empty_bf16, empty_fp32, empty_i32 @@ -50,9 +44,9 @@ if current_platform.is_cuda_alike(): FUSED_OPS[kFp8Dynamic64Sym] = torch.ops._C.silu_and_mul_per_block_quant.default -class ActivationQuantPattern(ABC): +class ActivationQuantPattern(VllmPatternReplacement): """ - The base class for Activation+Quant fusions. + Base class for Activation+Quant fusions. Should not be used directly. """ @@ -79,10 +73,6 @@ class ActivationQuantPattern(ABC): kwargs = {"dtype": self.quant_dtype, "device": "cuda", **kwargs} return torch.empty(*args, **kwargs) - @abstractmethod - def register(self, pm_pass: PatternMatcherPass) -> None: - raise NotImplementedError - class SiluMulFp8StaticQuantPattern(ActivationQuantPattern): """ @@ -100,8 +90,9 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern): scale, ] - def register(self, pm_pass: PatternMatcherPass) -> None: - def pattern( + @property + def pattern(self): + def _pattern( input: torch.Tensor, scale: torch.Tensor, ) -> torch.Tensor: @@ -109,7 +100,11 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern): result_quant = self.quant_matcher(result_silu_mul, scale) return result_quant[0] - def replacement( + return _pattern + + @property + def replacement(self): + def _replacement( input: torch.Tensor, scale: torch.Tensor, ) -> torch.Tensor: @@ -123,10 +118,7 @@ class SiluMulFp8StaticQuantPattern(ActivationQuantPattern): ) return at[1] - inps = self.get_inputs() - pattern(*inps) - - register_replacement(pattern, replacement, inps, fwd_only, pm_pass) + return _replacement class SiluMulNvfp4QuantPattern(ActivationQuantPattern): @@ -144,8 +136,9 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern): scale = empty_fp32(1, 1) return [result, output_scale, input_, scale] - def register(self, pm_pass: PatternMatcherPass) -> None: - def pattern( + @property + def pattern(self): + def _pattern( result: torch.Tensor, output_scale: torch.Tensor, input: torch.Tensor, @@ -162,7 +155,11 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern): ) return at[1], at[2] - def replacement( + return _pattern + + @property + def replacement(self): + def _replacement( result: torch.Tensor, output_scale: torch.Tensor, input: torch.Tensor, @@ -177,7 +174,7 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern): ) return at[1], at[2] - register_replacement(pattern, replacement, self.get_inputs(), fwd_only, pm_pass) + return _replacement class SiluMulBlockQuantPattern(ActivationQuantPattern): @@ -210,10 +207,9 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern): scale = self.quant_matcher.empty_f32(1, 1) return self.silu_and_mul_matcher.inputs() + [scale] - def register(self, pm_pass: PatternMatcherPass) -> None: - is_scale_transposed = self.is_scale_transposed - - def pattern( + @property + def pattern(self): + def _pattern( input: torch.Tensor, scale: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: @@ -235,12 +231,16 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern): fp8_min=finfo.min, fp8_max=finfo.max, scale_ue8m0=self.is_e8m0, - dummy_is_scale_transposed=is_scale_transposed, + dummy_is_scale_transposed=self.is_scale_transposed, dummy_is_tma_aligned=self.is_tma_aligned, ) return result, scale - def replacement( + return _pattern + + @property + def replacement(self): + def _replacement( input: torch.Tensor, scale: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: @@ -249,7 +249,7 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern): result = torch.empty( output_shape, device=input.device, dtype=self.quant_dtype ) - if is_scale_transposed: + if self.is_scale_transposed: scale = torch.empty( (d // self.group_size, input.shape[0]), device=input.device, @@ -268,15 +268,14 @@ class SiluMulBlockQuantPattern(ActivationQuantPattern): scales=scale, group_size=self.group_size, scale_ub=None, - is_scale_transposed=is_scale_transposed, + is_scale_transposed=self.is_scale_transposed, ) return at[1], at[2] - inps = self.get_inputs() - register_replacement(pattern, replacement, inps, fwd_only, pm_pass) + return _replacement -class ActivationQuantFusionPass(VllmPatternMatcherPass): +class ActivationQuantFusionPass(VllmFusionPatternMatcherPass): """ This pass fuses a pre-defined set of custom ops into fused ops. It uses the torch pattern matcher to find the patterns and replace them. @@ -286,45 +285,33 @@ class ActivationQuantFusionPass(VllmPatternMatcherPass): https://github.com/pytorch/pytorch/pull/139321#issuecomment-2452354980 """ - @enable_fake_mode def __init__(self, config: VllmConfig) -> None: - super().__init__(config) + super().__init__(config, "activation_quant_fusion_pass") - self.patterns: PatternMatcherPass = PatternMatcherPass( - pass_name="activation_quant_fusion_pass" - ) - - pattern_silu_mul_fp8 = SiluMulFp8StaticQuantPattern() - pattern_silu_mul_fp8.register(self.patterns) + self.register(SiluMulFp8StaticQuantPattern()) if silu_and_mul_nvfp4_quant_supported: - pattern_silu_mul_nvfp4 = SiluMulNvfp4QuantPattern() - pattern_silu_mul_nvfp4.register(self.patterns) + self.register(SiluMulNvfp4QuantPattern()) if current_platform.is_cuda(): - for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]: - for is_scale_transposed in [False, True]: - for is_e8m0 in [True, False]: - for is_tma_aligned in [False, True]: - SiluMulBlockQuantPattern( - quant_key, - is_scale_transposed=is_scale_transposed, - is_e8m0=is_e8m0, - is_tma_aligned=is_tma_aligned, - ).register(self.patterns) + for ( + quant_key, + is_scale_transposed, + is_e8m0, + is_tma_aligned, + ) in itertools.product( + [kFp8Dynamic128Sym, kFp8Dynamic64Sym], + [False, True], + [True, False], + [False, True], + ): + self.register( + SiluMulBlockQuantPattern( + quant_key, + is_scale_transposed=is_scale_transposed, + is_e8m0=is_e8m0, + is_tma_aligned=is_tma_aligned, + ) + ) - self.dump_patterns(config, self.patterns) - - @VllmInductorPass.time_and_log - def __call__(self, graph: torch.fx.Graph) -> None: - self.matched_count = self.patterns.apply(graph) - logger.debug("Replaced %s patterns", self.matched_count) - - def uuid(self) -> str: - return VllmInductorPass.hash_source( - self, - ActivationQuantPattern, - SiluMulFp8StaticQuantPattern, - SiluMulNvfp4QuantPattern, - SiluMulBlockQuantPattern, - ) + self.dump_patterns(config, self.pm_pass) diff --git a/vllm/compilation/passes/fusion/rocm_aiter_fusion.py b/vllm/compilation/passes/fusion/rocm_aiter_fusion.py index 51b1a802f2e..cdd0e23773d 100644 --- a/vllm/compilation/passes/fusion/rocm_aiter_fusion.py +++ b/vllm/compilation/passes/fusion/rocm_aiter_fusion.py @@ -28,7 +28,6 @@ from ..vllm_inductor_pass import ( VllmPatternMatcherPass, VllmPatternReplacement, ) -from .act_quant_fusion import ActivationQuantPattern from .matcher_utils import ( MatcherFusedAddRMSNorm, MatcherQuantFP8, @@ -345,7 +344,7 @@ class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass): return self.hash_source(self, *fusion_patterns) -class AiterSiluMulFp8GroupQuantPattern(ActivationQuantPattern): +class AiterSiluMulFp8GroupQuantPattern(VllmPatternReplacement): """ This pattern fuses aiter silu_and_mul & group fp8 quant custom ops into an aiter silu_and_mul_group_fp8_quant op. @@ -364,26 +363,29 @@ class AiterSiluMulFp8GroupQuantPattern(ActivationQuantPattern): self.silu_and_mul_matcher.inputs()[0], ] - def register(self, pm_pass: PatternMatcherPass) -> None: - def pattern( + @property + def pattern(self): + def _pattern( input: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: at1 = self.silu_and_mul_matcher(input) at2 = self.quant_matcher(at1) return at2[0], at2[1] - def replacement( + return _pattern + + @property + def replacement(self): + def _replacement( input: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: at = self.FUSED_SILU_MUL_QUANT_OP(x=input, group_size=128) return at[0], at[1] - pm.register_replacement( - pattern, replacement, self.get_inputs(), pm.fwd_only, pm_pass - ) + return _replacement -class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmPatternMatcherPass): +class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmFusionPatternMatcherPass): """ This pass fuses a pre-defined set of custom ops into fused ops. It uses the torch pattern matcher to find the patterns and replace them. @@ -393,29 +395,12 @@ class RocmAiterSiluMulFp8GroupQuantFusionPass(VllmPatternMatcherPass): https://github.com/pytorch/pytorch/pull/139321#issuecomment-2452354980 """ - @enable_fake_mode def __init__(self, config: VllmConfig) -> None: - super().__init__(config) + super().__init__(config, "rocm_aiter_silu_mul_fp8_group_quant_fusion_pass") - self.patterns: PatternMatcherPass = PatternMatcherPass( - pass_name="rocm_aiter_silu_mul_fp8_group_quant_fusion_pass" - ) + self.register(AiterSiluMulFp8GroupQuantPattern()) - AiterSiluMulFp8GroupQuantPattern().register(self.patterns) - - self.dump_patterns(config, self.patterns) - - @VllmInductorPass.time_and_log - def __call__(self, graph: torch.fx.Graph) -> None: - self.matched_count = self.patterns.apply(graph) - logger.debug("Replaced %s patterns", self.matched_count) - - def uuid(self) -> str: - fusion_patterns = [ - ActivationQuantPattern, - AiterSiluMulFp8GroupQuantPattern, - ] - return VllmInductorPass.hash_source(self, *fusion_patterns) + self.dump_patterns(config, self.pm_pass) class AddAiterRMSNormPadPattern: From 2f314bc5e6706bffe77933ae5b13756d04641ed8 Mon Sep 17 00:00:00 2001 From: almayne Date: Thu, 23 Apr 2026 14:14:44 +0100 Subject: [PATCH 064/153] [CPU] Added faster exp routine for lower precision data types. (#38112) Signed-off-by: Anna Mayne Co-authored-by: Fadi Arafeh Co-authored-by: Li, Jiang --- csrc/cpu/cpu_arch_macros.h | 50 ++++++++++++++++++++++++++++++++++++-- csrc/cpu/cpu_attn_impl.hpp | 32 +++++++++++++++++++++--- 2 files changed, 76 insertions(+), 6 deletions(-) diff --git a/csrc/cpu/cpu_arch_macros.h b/csrc/cpu/cpu_arch_macros.h index c73b62ecdec..9be45a3efce 100644 --- a/csrc/cpu/cpu_arch_macros.h +++ b/csrc/cpu/cpu_arch_macros.h @@ -61,8 +61,23 @@ #endif #ifdef __aarch64__ - // Implementation copied from Arm Optimized Routines (expf AdvSIMD) + // Implementation of neon_expf copied from Arm Optimized Routines (expf + // AdvSIMD) // https://github.com/ARM-software/optimized-routines/blob/master/math/aarch64/advsimd/expf.c + // + // Additional fast exponential intended for cases where outputs will be + // downcasted to FP16 / BF16 (e.g. attention softmax). Accurate within 1 ULP + // for FP16 Accurate within 1 ULP for BF16 for inputs in [-87.683, 88.376] & + // clamps inputs outside this range to 0 / inf. Implementation is similar to + // exp_u20, but: + // - uses a third degree polynomial approximation for exp(r) instead of a + // fifth degree one, with coefficients re-tuned. + // - does not split natural log (ln) into high / low parts + // - clamps exp(x) to 0 for x < -87.683113f and inf for x > 88.3762589f + // exp(x) = 2^n (exp(r)) + // r = x - n*ln2, with n = round(x/ln2) + // exp(r) ~ poly(r) = 1 + r + r^2 * (c3 + c2 * r) + // n = round(x / ln2), r = x - n*ln2 #include #define DEFINE_FAST_EXP \ const float32x4_t inv_ln2 = vdupq_n_f32(0x1.715476p+0f); \ @@ -106,7 +121,38 @@ result.val[2] = neon_expf(vec.reg.val[2]); \ result.val[3] = neon_expf(vec.reg.val[3]); \ return vec_op::FP32Vec16(result); \ - }; + }; \ + const float32x4_t lower_bound = vdupq_n_f32(-0x1.5ebb82p+6f); \ + const float32x4_t upper_bound = vdupq_n_f32(0x1.61814ap+6f); \ + constexpr float ln2 = 0x1.62e43p-1f; \ + constexpr float f_c2 = 0x1.5592ecp-3f; \ + const float32x4_t f_c3 = vdupq_n_f32(0x1.017d34p-1f); \ + auto neon_expf_f16 = [&](float32x4_t values) __attribute__(( \ + always_inline)) { \ + const uint32x4_t lt_lower = vcltq_f32(values, lower_bound); \ + const uint32x4_t gt_upper = vcgtq_f32(values, upper_bound); \ + float32x4_t n = vrndaq_f32(vmulq_f32(values, inv_ln2)); \ + float32x4_t r = vfmsq_n_f32(values, n, ln2); \ + uint32x4_t e = vshlq_n_u32(vreinterpretq_u32_s32(vcvtq_s32_f32(n)), 23); \ + float32x4_t r2 = vmulq_f32(r, r); \ + float32x4_t q = vfmaq_n_f32(f_c3, r, f_c2); \ + float32x4_t s = vaddq_f32(vdupq_n_f32(1.0f), r); \ + float32x4_t p = vfmaq_f32(s, q, r2); \ + float32x4_t y = \ + vreinterpretq_f32_u32(vaddq_u32(vreinterpretq_u32_f32(p), e)); \ + y = vbslq_f32(lt_lower, vdupq_n_f32(0.0f), y); \ + y = vbslq_f32(gt_upper, vdupq_n_f32(INFINITY), y); \ + return y; \ + }; \ + auto fast_exp_f16 = [&](const vec_op::FP32Vec16& vec) \ + __attribute__((always_inline)) { \ + float32x4x4_t result; \ + result.val[0] = neon_expf_f16(vec.reg.val[0]); \ + result.val[1] = neon_expf_f16(vec.reg.val[1]); \ + result.val[2] = neon_expf_f16(vec.reg.val[2]); \ + result.val[3] = neon_expf_f16(vec.reg.val[3]); \ + return vec_op::FP32Vec16(result); \ + }; #endif // __aarch64__ diff --git a/csrc/cpu/cpu_attn_impl.hpp b/csrc/cpu/cpu_attn_impl.hpp index 08f42459e14..c1974bfd0a5 100644 --- a/csrc/cpu/cpu_attn_impl.hpp +++ b/csrc/cpu/cpu_attn_impl.hpp @@ -1152,7 +1152,11 @@ class AttentionMainLoop { bool use_sink) { #ifdef DEFINE_FAST_EXP DEFINE_FAST_EXP + bool constexpr IsReducedPrecision = + std::is_same_v || + std::is_same_v; #endif + using prob_buffer_vec_t = typename VecTypeTrait::vec_t; static_assert(sizeof(prob_buffer_t) <= sizeof(logits_buffer_t)); @@ -1201,8 +1205,17 @@ class AttentionMainLoop { vec = vec - max_vec; // compute exp -#ifdef DEFINE_FAST_EXP - vec = fast_exp(vec); + +#if defined(DEFINE_FAST_EXP) + #ifdef __aarch64__ + if constexpr (IsReducedPrecision) { + vec = fast_exp_f16(vec); + } else + #endif + { + vec = fast_exp(vec); + } + prob_buffer_vec_t output_vec(vec); output_vec.save(curr_prob_buffer_iter); #else @@ -1258,7 +1271,11 @@ class AttentionMainLoop { int32_t kv_tile_token_num, float softcap_scale) { #ifdef DEFINE_FAST_EXP DEFINE_FAST_EXP + bool constexpr IsReducedPrecision = + std::is_same_v || + std::is_same_v; #endif + float inv_softcap_scale = 1.0 / softcap_scale; vec_op::FP32Vec16 softcap_scale_vec(softcap_scale); vec_op::FP32Vec16 inv_softcap_scale_vec(inv_softcap_scale); @@ -1272,8 +1289,15 @@ class AttentionMainLoop { vec_op::FP32Vec16 vec(curr_logits_buffer_iter); vec = vec * inv_softcap_scale_vec; -#ifdef DEFINE_FAST_EXP - vec = fast_exp(vec); +#if defined(DEFINE_FAST_EXP) + #ifdef __aarch64__ + if constexpr (IsReducedPrecision) { + vec = fast_exp_f16(vec); + } else + #endif + { + vec = fast_exp(vec); + } vec_op::FP32Vec16 inv_vec = ones_vec / vec; vec = (vec - inv_vec) / (vec + inv_vec); #else From 01cb41dcf5a3dc81a0e3f2fc484554ce2d6466b4 Mon Sep 17 00:00:00 2001 From: xiangdong <40376367+zxd1997066@users.noreply.github.com> Date: Thu, 23 Apr 2026 21:42:22 +0800 Subject: [PATCH 065/153] [XPU][CI]Temporary disable 3 cases on Intel GPU in CI (#40683) Signed-off-by: zengxian --- .buildkite/intel_jobs/lora_intel.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml index 91fee21f121..729edd159cd 100644 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ b/.buildkite/intel_jobs/lora_intel.yaml @@ -20,7 +20,7 @@ steps: 'cd tests && pytest -v -s lora/test_layers.py && pytest -v -s lora/test_lora_checkpoints.py && - pytest -v -s lora/test_lora_functions.py && + (pytest -v -s lora/test_lora_functions.py --deselect="tests/lora/test_lora_functions.py::test_lora_functions_sync" --deselect="tests/lora/test_lora_functions.py::test_lora_functions_async" || true) && pytest -v -s lora/test_lora_huggingface.py && pytest -v -s lora/test_lora_manager.py && pytest -v -s lora/test_lora_utils.py && @@ -125,6 +125,6 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && pytest -v -s lora/test_default_mm_loras.py && - pytest -v -s lora/test_qwen3_unembed.py && - pytest -v -s lora/test_qwenvl.py && + (pytest -v -s lora/test_qwen3_unembed.py || true) && + (pytest -v -s lora/test_qwenvl.py || true) && pytest -v -s lora/test_whisper.py' From da1e7311cad640e008284c2fb1754f5f692259c8 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Thu, 23 Apr 2026 21:42:52 +0800 Subject: [PATCH 066/153] [Misc] use model arch converter for bidi models identification (#40701) Signed-off-by: Isotr0py --- vllm/config/model.py | 17 ++------------ vllm/config/model_arch.py | 3 +++ .../model_arch_config_convertor.py | 23 +++++++++++++++++++ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/vllm/config/model.py b/vllm/config/model.py index 2b767b21a7c..599742d6014 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1197,22 +1197,9 @@ class ModelConfig: def is_deepseek_mla(self) -> bool: return self.model_arch_config.is_deepseek_mla - @cached_property + @property def is_mm_prefix_lm(self) -> bool: - """Whether to use bidirectional attention for mm positions.""" - if hasattr(self.hf_config, "is_mm_prefix_lm"): - return bool(self.hf_config.is_mm_prefix_lm) - # fallback to list of known models - MM_PREFIX_LM_MODELS = ( - "bagel", - "gemma3", - "molmo2", - "paligemma", - "umm", - ) - if not hasattr(self.hf_config, "model_type"): - return False - return self.hf_config.model_type in MM_PREFIX_LM_MODELS + return self.model_arch_config.is_mm_prefix_lm def get_head_size(self) -> int: return self.model_arch_config.head_size diff --git a/vllm/config/model_arch.py b/vllm/config/model_arch.py index 24d1baea0a9..0b99df22b88 100644 --- a/vllm/config/model_arch.py +++ b/vllm/config/model_arch.py @@ -53,5 +53,8 @@ class ModelArchitectureConfig: is_deepseek_mla: bool """Whether the model is a DeepSeek MLA model.""" + is_mm_prefix_lm: bool + """Whether the model uses image bidirectional attention.""" + derived_max_model_len_and_key: tuple[float, str | None] """Derived maximum model length and key from the hf config.""" diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index eb7d3eeda30..cef65580347 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -250,6 +250,22 @@ class ModelArchConfigConvertorBase: ) return False + def is_mm_prefix_lm(self) -> bool: + """Whether to use bidirectional attention for mm positions.""" + if hasattr(self.hf_config, "is_mm_prefix_lm"): + return bool(self.hf_config.is_mm_prefix_lm) + # fallback to list of known models + MM_PREFIX_LM_MODELS = ( + "bagel", + "gemma3", + "molmo2", + "paligemma", + "umm", + ) + if not hasattr(self.hf_config, "model_type"): + return False + return self.hf_config.model_type in MM_PREFIX_LM_MODELS + def derive_max_model_len_and_key(self) -> tuple[float, str | None]: derived_max_model_len = float("inf") possible_keys = [ @@ -299,6 +315,7 @@ class ModelArchConfigConvertorBase: num_experts=self.get_num_experts(), quantization_config=self.get_quantization_config(), is_deepseek_mla=self.is_deepseek_mla(), + is_mm_prefix_lm=self.is_mm_prefix_lm(), derived_max_model_len_and_key=self.derive_max_model_len_and_key(), ) @@ -451,6 +468,12 @@ class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase): class Gemma4ModelArchConfigConvertor(ModelArchConfigConvertorBase): + def is_mm_prefix_lm(self) -> bool: + return ( + getattr(self.hf_text_config, "use_bidirectional_attention", None) + == "vision" + ) + def get_head_size(self) -> int: # Gemma4 uses dual head dimensions: head_dim (sliding attention) # and global_head_dim (full attention). Return the largest so From 424033f4fceeb5a1469fa77dfa7fc0c5d60f002d Mon Sep 17 00:00:00 2001 From: Richard Zou Date: Thu, 23 Apr 2026 09:52:59 -0400 Subject: [PATCH 067/153] [Bugfix] Include inductor and functorch configs in compilation cache key (#40627) Signed-off-by: Richard Zou --- tests/compile/h100/test_startup.py | 2 ++ tests/compile/test_config.py | 18 ++++++++++++++++++ vllm/compilation/compiler_interface.py | 22 ++++++++++++++++++++-- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tests/compile/h100/test_startup.py b/tests/compile/h100/test_startup.py index 1e1c93217f9..ff4496c2ba6 100644 --- a/tests/compile/h100/test_startup.py +++ b/tests/compile/h100/test_startup.py @@ -56,6 +56,7 @@ def _cold_start(vllm_runner): def test_moe_startup(monkeypatch, vllm_runner, fresh_vllm_cache, mega_aot_artifact): monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") monkeypatch.setenv("VLLM_USE_MEGA_AOT_ARTIFACT", mega_aot_artifact) + monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip") # Cold start in a forked child (must fork before CUDA init). # This model has 32 identical transformer layers which produce @@ -235,6 +236,7 @@ def _cold_start_model(vllm_runner, spec: ModelStartupSpec): @fork_new_process_for_each_test def test_model_startup(monkeypatch, vllm_runner, fresh_vllm_cache, spec): monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0") + monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip") # Cold start in a forked child (must fork before CUDA init). ctx = mp.get_context("fork") diff --git a/tests/compile/test_config.py b/tests/compile/test_config.py index 12518b4cfa2..913d7eb46a1 100644 --- a/tests/compile/test_config.py +++ b/tests/compile/test_config.py @@ -617,6 +617,24 @@ def test_inductor_asserts_enabled_in_debug(monkeypatch): assert config.inductor_compile_config.get("scalar_asserts") is True +def test_get_inductor_factors_includes_configs(): + """Changing inductor or functorch config must change the cache key factors.""" + from torch._functorch import config as functorch_config + from torch._inductor import config as inductor_config + + from vllm.compilation.compiler_interface import get_inductor_factors + + baseline = get_inductor_factors() + + with inductor_config.patch("max_autotune", not inductor_config.max_autotune): + patched = get_inductor_factors() + assert baseline != patched, "inductor config change was not reflected" + + with functorch_config.patch("donated_buffer", not functorch_config.donated_buffer): + patched = get_inductor_factors() + assert baseline != patched, "functorch config change was not reflected" + + def test_inductor_asserts_user_override(monkeypatch): """Test that explicit inductor_compile_config overrides the debug-logging default.""" diff --git a/vllm/compilation/compiler_interface.py b/vllm/compilation/compiler_interface.py index ae280fbcb97..933554faa28 100644 --- a/vllm/compilation/compiler_interface.py +++ b/vllm/compilation/compiler_interface.py @@ -152,6 +152,17 @@ class AlwaysHitShapeEnv: return "" +def _get_vllm_functorch_config() -> dict[str, Any]: + """Return the functorch config overrides that vLLM applies at compile time. + + Used by both set_functorch_config() and get_inductor_factors() to ensure + the compile-time config and cache key are always consistent.""" + cfg: dict[str, Any] = {} + if not envs.VLLM_USE_MEGA_AOT_ARTIFACT: + cfg["bundled_autograd_cache"] = False + return cfg + + def get_inductor_factors() -> list[Any]: factors: list[Any] = [] # summarize system state @@ -165,6 +176,13 @@ def get_inductor_factors() -> list[Any]: torch_factors = torch_key() factors.append(torch_factors) + + from torch._functorch import config as functorch_config + from torch._inductor import config as inductor_config + + factors.append(inductor_config.save_config_portable()) + with functorch_config.patch(_get_vllm_functorch_config()): + factors.append(functorch_config.save_config_portable()) return factors @@ -739,8 +757,8 @@ def set_inductor_config(config: dict[str, Any], compile_range: Range) -> None: def set_functorch_config() -> None: - if not envs.VLLM_USE_MEGA_AOT_ARTIFACT: - torch._functorch.config.bundled_autograd_cache = False + for k, v in _get_vllm_functorch_config().items(): + setattr(torch._functorch.config, k, v) class EagerAdaptor(CompilerInterface): From d0009ddb0b96e95bcfae6038e9b8673bd2263058 Mon Sep 17 00:00:00 2001 From: stevenkuang Date: Thu, 23 Apr 2026 22:08:26 +0800 Subject: [PATCH 068/153] [Model] Support Hy3 preview (#40681) Signed-off-by: stevenkuang Co-authored-by: Jee Jee Li --- docs/models/supported_models.md | 1 + tests/models/registry.py | 5 + .../reasoning/test_hy_v3_reasoning_parser.py | 243 ++++++ tests/tool_parsers/test_hy_v3_tool_parser.py | 274 +++++++ vllm/config/speculative.py | 8 + .../model_loader/weight_utils.py | 8 + vllm/model_executor/models/hy_v3.py | 707 ++++++++++++++++++ vllm/model_executor/models/hy_v3_mtp.py | 470 ++++++++++++ vllm/model_executor/models/registry.py | 2 + vllm/reasoning/__init__.py | 4 + vllm/reasoning/hy_v3_reasoning_parser.py | 137 ++++ vllm/tool_parsers/__init__.py | 4 + vllm/tool_parsers/hy_v3_tool_parser.py | 645 ++++++++++++++++ vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 2 + vllm/transformers_utils/configs/hy_v3.py | 185 +++++ 16 files changed, 2696 insertions(+) create mode 100644 tests/reasoning/test_hy_v3_reasoning_parser.py create mode 100644 tests/tool_parsers/test_hy_v3_tool_parser.py create mode 100644 vllm/model_executor/models/hy_v3.py create mode 100644 vllm/model_executor/models/hy_v3_mtp.py create mode 100644 vllm/reasoning/hy_v3_reasoning_parser.py create mode 100644 vllm/tool_parsers/hy_v3_tool_parser.py create mode 100644 vllm/transformers_utils/configs/hy_v3.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 70956f5092a..d1e6aff7e9a 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -419,6 +419,7 @@ th { | `Grok1ForCausalLM` | Grok2 | `xai-org/grok-2` | ✅︎ | ✅︎ | | `HunYuanDenseV1ForCausalLM` | Hunyuan Dense | `tencent/Hunyuan-7B-Instruct` | ✅︎ | ✅︎ | | `HunYuanMoEV1ForCausalLM` | Hunyuan-A13B | `tencent/Hunyuan-A13B-Instruct`, `tencent/Hunyuan-A13B-Pretrain`, `tencent/Hunyuan-A13B-Instruct-FP8`, etc. | ✅︎ | ✅︎ | +| `HYV3ForCausalLM` | HY3 | `tencent/Hy3-preview-Base`, `tencent/Hy3-preview` | ✅︎ | ✅︎ | | `HyperCLOVAXForCausalLM` | HyperCLOVAX-SEED-Think-14B | `naver-hyperclovax/HyperCLOVAX-SEED-Think-14B` | ✅︎ | ✅︎ | | `InternLMForCausalLM` | InternLM | `internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc. | ✅︎ | ✅︎ | | `InternLM2ForCausalLM` | InternLM2 | `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc. | ✅︎ | ✅︎ | diff --git a/tests/models/registry.py b/tests/models/registry.py index 4c418ae4ee7..09e1ee42f2e 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -324,6 +324,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = { "HunYuanMoEV1ForCausalLM": _HfExamplesInfo( "tencent/Hunyuan-A13B-Instruct", trust_remote_code=True ), + "HYV3ForCausalLM": _HfExamplesInfo("tencent/Hy3-preview", trust_remote_code=True), "HyperCLOVAXForCausalLM": _HfExamplesInfo( "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", trust_remote_code=True, @@ -1516,6 +1517,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = { is_available_online=False, min_transformers_version="5.1.0", ), + "HYV3MTPModel": _HfExamplesInfo( + "tencent/Hy3-preview", + speculative_model="tencent/Hy3-preview", + ), "LongCatFlashMTPModel": _HfExamplesInfo( "meituan-longcat/LongCat-Flash-Chat", trust_remote_code=True, diff --git a/tests/reasoning/test_hy_v3_reasoning_parser.py b/tests/reasoning/test_hy_v3_reasoning_parser.py new file mode 100644 index 00000000000..4c1858cc99b --- /dev/null +++ b/tests/reasoning/test_hy_v3_reasoning_parser.py @@ -0,0 +1,243 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest + +from tests.reasoning.utils import run_reasoning_extraction +from vllm.reasoning import ReasoningParser, ReasoningParserManager +from vllm.tokenizers import get_tokenizer + +parser_name = "hy_v3" +MODEL = "tencent/Hy3-preview" + + +@pytest.fixture(scope="module") +def hy_v3_tokenizer(): + return get_tokenizer(tokenizer_name=MODEL) + + +WITH_THINK = { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", + "is_reasoning_end": True, + "reasoning_effort": "high", +} + +WITH_THINK_STREAM = { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", + "is_reasoning_end": True, + "reasoning_effort": "high", +} + +WITHOUT_THINK = { + "output": "This is the rest", + "reasoning": None, + "content": "This is the rest", + "is_reasoning_end": True, + "reasoning_effort": "no_think", +} + +WITHOUT_THINK_STREAM = { + "output": "This is the rest", + "reasoning": None, + "content": "This is the rest", + "is_reasoning_end": True, + "reasoning_effort": "no_think", +} + +WITH_REASONING_EFFORT_NONE = { + "output": "This is the rest", + "reasoning": None, + "content": "This is the rest", + "is_reasoning_end": True, +} + +WITH_REASONING_EFFORT_NONE_STREAM = { + "output": "This is the rest", + "reasoning": None, + "content": "This is the rest", + "is_reasoning_end": True, +} + +COMPLETE_REASONING = { + "output": "This is a reasoning section", + "reasoning": "This is a reasoning section", + "content": None, + "is_reasoning_end": True, + "reasoning_effort": "high", +} +MULTILINE_REASONING = { + "output": "This is a reasoning\nsectionThis is the rest\nThat", + "reasoning": "This is a reasoning\nsection", + "content": "This is the rest\nThat", + "is_reasoning_end": True, + "reasoning_effort": "high", +} +ONLY_OPEN_TAG = { + "output": "This is a reasoning section", + "reasoning": "This is a reasoning section", + "content": None, + "is_reasoning_end": False, + "reasoning_effort": "high", +} + +ONLY_OPEN_TAG_STREAM = { + "output": "This is a reasoning section", + "reasoning": "This is a reasoning section", + "content": None, + "is_reasoning_end": False, + "reasoning_effort": "high", +} + +TEST_CASES = [ + pytest.param( + False, + WITH_THINK, + id="with_think", + ), + pytest.param( + True, + WITH_THINK_STREAM, + id="with_think_stream", + ), + pytest.param( + False, + WITHOUT_THINK, + id="without_think", + ), + pytest.param( + True, + WITHOUT_THINK_STREAM, + id="without_think_stream", + ), + pytest.param( + False, + WITH_REASONING_EFFORT_NONE, + id="with_reasoning_effort_none", + ), + pytest.param( + True, + WITH_REASONING_EFFORT_NONE_STREAM, + id="with_reasoning_effort_none_stream", + ), + pytest.param( + False, + COMPLETE_REASONING, + id="complete_reasoning", + ), + pytest.param( + True, + COMPLETE_REASONING, + id="complete_reasoning_stream", + ), + pytest.param( + False, + MULTILINE_REASONING, + id="multiline_reasoning", + ), + pytest.param( + True, + MULTILINE_REASONING, + id="multiline_reasoning_stream", + ), + pytest.param( + False, + ONLY_OPEN_TAG, + id="only_open_tag", + ), + pytest.param( + True, + ONLY_OPEN_TAG_STREAM, + id="only_open_tag_stream", + ), +] + +STILL_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜> +You are a helpful assistant. +<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜> +What is the capital of France?<｜hy_Assistant｜> +The user is asking for the capital of""" + +DONE_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜> +You are a helpful assistant. +<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜> +What is the capital of France?<｜hy_Assistant｜> +The user is asking for the capital of France. +The capital of France is Paris.""" + +MULTI_TURN_STILL_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜> +You are a helpful assistant. +<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜> +What is the capital of France?<｜hy_Assistant｜ +>The capital of France is Paris. +<｜hy_User｜>What about Chile?<｜hy_Assistant｜> +The user is asking for the capital of""" + +MULTI_TURN_DONE_REASONING_PROMPT = """<｜hy_begin▁of▁sentence｜> +You are a helpful assistant. +<｜reasoning_mode｜>reasoning_effort:high<｜hy_User｜> +What is the capital of France?<｜hy_Assistant｜ +>The capital of France is Paris. +<｜hy_User｜>What about Chile?<｜hy_Assistant｜> +The user is asking for the capital of Chile. +The capital of Chile is Santiago.""" + +REASONING_END_TEST_CASES = [ + pytest.param(STILL_REASONING_PROMPT, False, id="still_reasoning"), + pytest.param(DONE_REASONING_PROMPT, True, id="done_reasoning"), + pytest.param( + MULTI_TURN_STILL_REASONING_PROMPT, False, id="multi_turn_still_reasoning" + ), + pytest.param( + MULTI_TURN_DONE_REASONING_PROMPT, True, id="multi_turn_done_reasoning" + ), +] + + +@pytest.mark.parametrize("streaming, param_dict", TEST_CASES) +def test_reasoning( + streaming: bool, + param_dict: dict, + hy_v3_tokenizer, +): + output = hy_v3_tokenizer.tokenize(param_dict["output"]) + output_tokens: list[str] = [ + hy_v3_tokenizer.convert_tokens_to_string([token]) for token in output + ] + + parser_kwargs = {} + if "reasoning_effort" in param_dict: + parser_kwargs["chat_template_kwargs"] = { + "reasoning_effort": param_dict["reasoning_effort"] + } + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( + hy_v3_tokenizer, + **parser_kwargs, + ) + + reasoning, content = run_reasoning_extraction( + parser, output_tokens, streaming=streaming + ) + + assert reasoning == param_dict["reasoning"] + assert content == param_dict["content"] + + output_ids = hy_v3_tokenizer.convert_tokens_to_ids(output) + is_reasoning_end = parser.is_reasoning_end(output_ids) + assert is_reasoning_end == param_dict["is_reasoning_end"] + + +@pytest.mark.parametrize("prompt, is_reasoning_end", REASONING_END_TEST_CASES) +def test_is_reasoning_end_full_prompt( + prompt: str, is_reasoning_end: bool, hy_v3_tokenizer +): + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( + hy_v3_tokenizer, + chat_template_kwargs={"reasoning_effort": "high"}, + ) + tokens = hy_v3_tokenizer.tokenize(prompt) + token_ids = hy_v3_tokenizer.convert_tokens_to_ids(tokens) + check_is_reasoning_end = parser.is_reasoning_end(token_ids) + assert check_is_reasoning_end == is_reasoning_end diff --git a/tests/tool_parsers/test_hy_v3_tool_parser.py b/tests/tool_parsers/test_hy_v3_tool_parser.py new file mode 100644 index 00000000000..b5aaaf52988 --- /dev/null +++ b/tests/tool_parsers/test_hy_v3_tool_parser.py @@ -0,0 +1,274 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# ruff: noqa: E501 +"""Tests for the HYV3 tool call parser.""" + +import json +from unittest.mock import Mock + +import pytest + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, + FunctionDefinition, +) +from vllm.entrypoints.openai.engine.protocol import DeltaMessage +from vllm.tokenizers import get_tokenizer +from vllm.tool_parsers.hy_v3_tool_parser import HYV3ToolParser + +parser_name = "hy_v3" +MODEL = "tencent/Hy3-preview" + + +@pytest.fixture(scope="module") +def hy_v3_tokenizer(): + return get_tokenizer(tokenizer_name=MODEL) + + +@pytest.fixture +def hy_v3_tool_parser(hy_v3_tokenizer): + return HYV3ToolParser(hy_v3_tokenizer) + + +@pytest.fixture +def mock_request() -> ChatCompletionRequest: + request = Mock(spec=ChatCompletionRequest) + request.tools = [ + ChatCompletionToolsParam( + function=FunctionDefinition(name="get_current_date", parameters={}), + ), + ChatCompletionToolsParam( + function=FunctionDefinition( + name="get_weather", + parameters={ + "type": "object", + "properties": { + "city": {"type": "string"}, + "date": {"type": "string"}, + }, + }, + ), + ), + ] + request.tool_choice = "auto" + return request + + +class TestHYV3ExtractToolCalls: + def test_no_tool_call(self, hy_v3_tool_parser, mock_request): + out = "This is a plain response." + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert not r.tools_called + assert r.content == out + + def test_zero_arg_inline(self, hy_v3_tool_parser, mock_request): + out = ( + "get_current_date" + ) + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert r.tools_called + assert r.tool_calls[0].function.name == "get_current_date" + assert json.loads(r.tool_calls[0].function.arguments) == {} + assert r.content is None + + def test_zero_arg_newline(self, hy_v3_tool_parser, mock_request): + out = "\nget_current_date\n\n" + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert r.tools_called + assert r.tool_calls[0].function.name == "get_current_date" + + def test_args_same_line(self, hy_v3_tool_parser, mock_request): + out = ( + "get_weathercityBeijing" + "date2026-03-30" + ) + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert r.tools_called + assert json.loads(r.tool_calls[0].function.arguments) == { + "city": "Beijing", + "date": "2026-03-30", + } + + def test_args_with_newlines(self, hy_v3_tool_parser, mock_request): + out = ( + "\nget_weather\ncity\nBeijing" + "\ndate\n2026-03-30\n\n" + ) + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert r.tools_called + assert json.loads(r.tool_calls[0].function.arguments) == { + "city": "Beijing", + "date": "2026-03-30", + } + + def test_content_before(self, hy_v3_tool_parser, mock_request): + out = "Checking.\nget_current_date\n\n" + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert r.tools_called + assert r.content == "Checking." + + def test_multiple(self, hy_v3_tool_parser, mock_request): + out = ( + "\nget_weather\ncity\nBeijing" + "\ndate\n2026-03-30\n\n" + "get_weather\ncity\nHangzhou\n" + "date\n2026-03-30\n\n" + ) + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert len(r.tool_calls) == 2 + + def test_empty_content_none(self, hy_v3_tool_parser, mock_request): + out = "\nget_current_date\n\n" + r = hy_v3_tool_parser.extract_tool_calls(out, request=mock_request) + assert r.content is None + + +def _simulate_streaming( + parser: HYV3ToolParser, + deltas: list[str], + request: ChatCompletionRequest, +) -> list[DeltaMessage | None]: + results: list[DeltaMessage | None] = [] + previous_text = "" + previous_token_ids: list[int] = [] + vocab = parser.vocab + for delta_text in deltas: + current_text = previous_text + delta_text + delta_token_ids = [tid for tok, tid in vocab.items() if tok in delta_text] + current_token_ids = previous_token_ids + delta_token_ids + result = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=previous_token_ids, + current_token_ids=current_token_ids, + delta_token_ids=delta_token_ids, + request=request, + ) + results.append(result) + previous_text = current_text + previous_token_ids = current_token_ids + return results + + +def _collect_streaming_tool_calls(results: list[DeltaMessage | None]) -> list[dict]: + tool_calls: dict[int, dict] = {} + for result in results: + if result is None or not result.tool_calls: + continue + for tc in result.tool_calls: + idx = tc.index + if idx not in tool_calls: + tool_calls[idx] = { + "name": tc.function.name or "", + "arguments": tc.function.arguments or "", + } + else: + if tc.function.name: + tool_calls[idx]["name"] += tc.function.name + if tc.function.arguments: + tool_calls[idx]["arguments"] += tc.function.arguments + return [tool_calls[i] for i in sorted(tool_calls.keys())] + + +def _collect_streaming_content(results: list[DeltaMessage | None]) -> str: + parts = [] + for result in results: + if result is not None and result.content: + parts.append(result.content) + return "".join(parts) + + +class TestHYV3ExtractToolCallsStreaming: + def test_no_tool_call_streaming(self, hy_v3_tool_parser, mock_request): + deltas = ["This is ", "a plain ", "response."] + results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request) + content = _collect_streaming_content(results) + assert content == "This is a plain response." + assert len(_collect_streaming_tool_calls(results)) == 0 + + def test_zero_arg_streaming(self, hy_v3_tool_parser, mock_request): + deltas = [ + "", + "\n", + "get_current_date", + "", + "\n", + "\n", + ] + results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request) + tc = _collect_streaming_tool_calls(results) + assert len(tc) == 1 + assert tc[0]["name"] == "get_current_date" + assert json.loads(tc[0]["arguments"]) == {} + + def test_args_streaming(self, hy_v3_tool_parser, mock_request): + deltas = [ + "", + "\n", + "get_weather", + "", + "\ncity", + "\nBeijing", + "\ndate", + "\n2026-03-30", + "\n", + "\n", + ] + results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request) + tc = _collect_streaming_tool_calls(results) + assert len(tc) == 1 and tc[0]["name"] == "get_weather" + assert json.loads(tc[0]["arguments"]) == { + "city": "Beijing", + "date": "2026-03-30", + } + + def test_content_before_streaming(self, hy_v3_tool_parser, mock_request): + deltas = [ + "Checking.", + "", + "\n", + "get_current_date", + "", + "\n", + "\n", + ] + results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request) + assert "Checking." in _collect_streaming_content(results) + tc = _collect_streaming_tool_calls(results) + assert len(tc) == 1 and tc[0]["name"] == "get_current_date" + + def test_multiple_streaming(self, hy_v3_tool_parser, mock_request): + deltas = [ + "", + "\n", + "get_weather", + "", + "\ncity", + "\nBeijing", + "\ndate", + "\n2026-03-30", + "\n", + "\n", + "get_weather", + "", + "\ncity", + "\nHangzhou", + "\ndate", + "\n2026-03-30", + "\n", + "\n", + ] + results = _simulate_streaming(hy_v3_tool_parser, deltas, mock_request) + tc = _collect_streaming_tool_calls(results) + assert len(tc) == 2 + assert json.loads(tc[0]["arguments"])["city"] == "Beijing" + assert json.loads(tc[1]["arguments"])["city"] == "Hangzhou" + + def test_all_in_one_delta_streaming(self, hy_v3_tool_parser, mock_request): + out = "\nget_current_date\n\n" + results = _simulate_streaming(hy_v3_tool_parser, [out], mock_request) + tc = _collect_streaming_tool_calls(results) + assert len(tc) == 1 and tc[0]["name"] == "get_current_date" + assert json.loads(tc[0]["arguments"]) == {} diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index bbe923f68f1..4e6a47ee46c 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -47,6 +47,7 @@ MTPModelTypes = Literal[ "mtp", "pangu_ultra_moe_mtp", "step3p5_mtp", + "hy_v3_mtp", ] NgramGPUTypes = Literal["ngram_gpu"] DFlashModelTypes = Literal["dflash"] @@ -364,6 +365,13 @@ class SpeculativeConfig: if initial_architecture == "MistralLarge3ForCausalLM": hf_config.update({"architectures": ["EagleMistralLarge3ForCausalLM"]}) + if hf_config.model_type == "hy_v3": + hf_config.model_type = "hy_v3_mtp" + n_predict = getattr(hf_config, "num_nextn_predict_layers", None) + hf_config.update( + {"n_predict": n_predict, "architectures": ["HYV3MTPModel"]} + ) + return hf_config def __post_init__(self): diff --git a/vllm/model_executor/model_loader/weight_utils.py b/vllm/model_executor/model_loader/weight_utils.py index 8282e6b099d..31b00df4e4c 100644 --- a/vllm/model_executor/model_loader/weight_utils.py +++ b/vllm/model_executor/model_loader/weight_utils.py @@ -1562,6 +1562,11 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None: # NemotronH format: .mixer.{k,v}_proj.{k,v}_scale -> # .mixer.attn.{k,v}_scale (r"\.mixer\.[kv]_proj\.([kv])_scale$", r".mixer.attn.\1_scale"), + # HYV3 format: .self_attn.q.scale -> .self_attn.attn.q_scale + (r"\.self_attn\.q\.scale$", r".self_attn.attn.q_scale"), + # HYV3 format: .self_attn.{k,v}_cache.scale -> + # .self_attn.attn.{k,v}_scale + (r"\.self_attn\.([kv])_cache\.scale$", r".self_attn.attn.\1_scale"), # Default format: .{k,v}_scale -> .attn.{k,v}_scale (r"\.([qkv])_scale$", r".attn.\1_scale"), (r"\.([qkv])_zero_point$", r".attn.\1_zero_point"), @@ -1576,6 +1581,9 @@ def maybe_remap_kv_scale_name(name: str, params_dict: dict) -> str | None: ".k_zero_point", ".v_zero_point", ".q_zero_point", + ".q.scale", + ".k_cache.scale", + ".v_cache.scale", ) ): import regex as re diff --git a/vllm/model_executor/models/hy_v3.py b/vllm/model_executor/models/hy_v3.py new file mode 100644 index 00000000000..bfff84b8049 --- /dev/null +++ b/vllm/model_executor/models/hy_v3.py @@ -0,0 +1,707 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# coding=utf-8 +# Copyright 2026 The HY team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only HY model compatible with HuggingFace weights.""" + +import typing +from collections.abc import Callable, Iterable +from itertools import islice +from typing import Any + +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import CacheConfig, VllmConfig, get_current_vllm_config +from vllm.distributed import ( + get_ep_group, + get_pp_group, + get_tensor_model_parallel_world_size, +) +from vllm.logger import init_logger +from vllm.model_executor.layers.activation import SiluAndMul +from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.fused_moe import FusedMoE, GateLinear +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ( + MergedColumnParallelLinear, + QKVParallelLinear, + RowParallelLinear, +) +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.rotary_embedding import get_rope +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.hy_v3 import HYV3Config + +from .interfaces import SupportsLoRA, SupportsPP +from .utils import ( + AutoWeightsLoader, + PPMissingLayer, + is_pp_missing_parameter, + make_empty_intermediate_tensors_factory, + make_layers, + maybe_prefix, +) + +logger = init_logger(__name__) + + +class HYV3FeedForward(nn.Module): + def __init__( + self, + hidden_size: int, + intermediate_size: int, + hidden_act: str, + quant_config: QuantizationConfig | None = None, + reduce_results: bool = True, + expert_gate: torch.nn.Linear | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.gate_up_proj = MergedColumnParallelLinear( + hidden_size, + [intermediate_size] * 2, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.gate_up_proj", + ) + self.down_proj = RowParallelLinear( + intermediate_size, + hidden_size, + bias=False, + quant_config=quant_config, + reduce_results=reduce_results, + prefix=f"{prefix}.down_proj", + ) + if hidden_act != "silu": + raise ValueError( + f"Unsupported activation: {hidden_act}. Only silu is supported for now." + ) + self.act_fn = SiluAndMul() + + def forward(self, x): + gate_up, _ = self.gate_up_proj(x) + out = self.act_fn(gate_up) + out, _ = self.down_proj(out) + return out + + +class HYV3MoEFused(nn.Module): + def __init__( + self, + config: HYV3Config, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + enable_eplb: bool = False, + ): + super().__init__() + self.tp_size = get_tensor_model_parallel_world_size() + self.ep_group = get_ep_group().device_group + self.ep_rank = get_ep_group().rank_in_group + self.ep_size = self.ep_group.size() + self.n_routed_experts = config.num_experts + if self.tp_size > config.num_experts: + raise ValueError( + f"Tensor parallel size {self.tp_size} is greater than " + f"the number of experts {config.num_experts}." + ) + top_k = config.num_experts_per_tok + intermediate_size = config.expert_hidden_dim + router_scaling_factor = getattr(config, "router_scaling_factor", 1.0) + vllm_config = get_current_vllm_config() + eplb_config = vllm_config.parallel_config.eplb_config + self.enable_eplb = enable_eplb + + self.n_logical_experts = self.n_routed_experts + self.n_redundant_experts = eplb_config.num_redundant_experts + self.n_physical_experts = self.n_logical_experts + self.n_redundant_experts + self.n_local_physical_experts = self.n_physical_experts // self.ep_size + self.physical_expert_start = self.ep_rank * self.n_local_physical_experts + self.physical_expert_end = ( + self.physical_expert_start + self.n_local_physical_experts + ) + self.gate = GateLinear( + config.hidden_size, + config.num_experts, + bias=False, + out_dtype=torch.float32, + params_dtype=torch.float32, + prefix=f"{prefix}.gate", + ) + + if config.num_shared_experts > 0: + self.shared_mlp = HYV3FeedForward( + hidden_size=config.hidden_size, + intermediate_size=config.expert_hidden_dim * config.num_shared_experts, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}", + reduce_results=False, + ) + else: + self.shared_mlp = None + + self.expert_bias = nn.Parameter(torch.empty(config.num_experts)) + scoring_func = "sigmoid" + e_score_correction_bias = self.expert_bias + + self.experts = FusedMoE( + num_experts=self.n_routed_experts, + top_k=top_k, + hidden_size=config.hidden_size, + intermediate_size=intermediate_size, + renormalize=config.route_norm, + quant_config=quant_config, + prefix=f"{prefix}.experts", + enable_eplb=self.enable_eplb, + num_redundant_experts=self.n_redundant_experts, + scoring_func=scoring_func, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + routed_scaling_factor=router_scaling_factor, + e_score_correction_bias=e_score_correction_bias, + n_shared_experts=config.num_shared_experts, + shared_experts=self.shared_mlp, + ) + + def forward( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + orig_shape = hidden_states.shape + hidden_dim = hidden_states.shape[-1] + hidden_states = hidden_states.view(-1, hidden_dim) + + # router_logits: (num_tokens, n_experts) + router_logits, _ = self.gate(hidden_states) + + final_hidden_states = self.experts( + hidden_states=hidden_states, router_logits=router_logits + ) + return final_hidden_states.view(orig_shape) + + +class HYV3Attention(nn.Module): + def __init__( + self, + config: PretrainedConfig, + hidden_size: int, + num_heads: int, + num_kv_heads: int, + rope_parameters: dict[str, Any], + max_position_embeddings: int = 8192, + head_dim: int | None = None, + rms_norm_eps: float = 1e-5, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + dual_chunk_attention_config: dict[str, Any] | None = None, + ) -> None: + super().__init__() + self.hidden_size = hidden_size + tp_size = get_tensor_model_parallel_world_size() + self.total_num_heads = num_heads + assert self.total_num_heads % tp_size == 0 + self.num_heads = self.total_num_heads // tp_size + self.total_num_kv_heads = num_kv_heads + if self.total_num_kv_heads >= tp_size: + # Number of KV heads is greater than TP size, so we partition + # the KV heads across multiple tensor parallel GPUs. + assert self.total_num_kv_heads % tp_size == 0 + else: + # Number of KV heads is less than TP size, so we replicate + # the KV heads across multiple tensor parallel GPUs. + assert tp_size % self.total_num_kv_heads == 0 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size) + + if hasattr(config, "head_dim") and config.head_dim: + self.head_dim = config.head_dim + else: + self.head_dim = head_dim or (hidden_size // self.total_num_heads) + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self.use_qk_norm = getattr(config, "qk_norm", False) + self.max_position_embeddings = max_position_embeddings + + self.qkv_proj = QKVParallelLinear( + hidden_size, + self.head_dim, + self.total_num_heads, + self.total_num_kv_heads, + quant_config=quant_config, + bias=None, + prefix=f"{prefix}.qkv_proj", + ) + self.o_proj = RowParallelLinear( + self.total_num_heads * self.head_dim, + hidden_size, + bias=False, + quant_config=quant_config, + prefix=f"{prefix}.o_proj", + ) + self.rotary_emb = get_rope( + self.head_dim, + max_position=max_position_embeddings, + rope_parameters=rope_parameters, + is_neox_style=True, + ) + self.attn = Attention( + self.num_heads, + self.head_dim, + self.scaling, + num_kv_heads=self.num_kv_heads, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.attn", + ) + if self.use_qk_norm: + self.q_norm = RMSNorm(self.head_dim, rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, rms_norm_eps) + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + ) -> torch.Tensor: + qkv, _ = self.qkv_proj(hidden_states) + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + output_shape = None + if self.use_qk_norm: + q_by_head = q.view( + *q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim + ) + q_by_head = self.q_norm(q_by_head) + q = q_by_head.view(q.shape) + + k_by_head = k.view( + *k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim + ) + k_by_head = self.k_norm(k_by_head) + k = k_by_head.view(k.shape) + q, k = self.rotary_emb(positions, q, k) + attn_output = self.attn(q, k, v, output_shape) + attn_output = attn_output.view(q.shape[0], -1) + output, _ = self.o_proj(attn_output) + return output + + +class HYV3DecoderLayer(nn.Module): + def __init__( + self, + config: PretrainedConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + prefix: str = "", + ) -> None: + super().__init__() + self.hidden_size = config.hidden_size + layer_idx = int(prefix.split(".")[-1]) + max_position_embeddings = getattr(config, "max_position_embeddings", 8192) + self.self_attn = HYV3Attention( + config=config, + hidden_size=self.hidden_size, + num_heads=config.num_attention_heads, + num_kv_heads=config.num_key_value_heads, + rope_parameters=config.rope_parameters, + max_position_embeddings=max_position_embeddings, + head_dim=config.head_dim, + rms_norm_eps=config.rms_norm_eps, + cache_config=cache_config, + quant_config=quant_config, + prefix=f"{prefix}.self_attn", + ) + self.input_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(config.hidden_size, config.rms_norm_eps) + if not hasattr(config, "first_k_dense_replace"): + raise ValueError("first_k_dense_replace not exist,please check config") + if layer_idx < config.first_k_dense_replace: + self.mlp = HYV3FeedForward( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + hidden_act=config.hidden_act, + quant_config=quant_config, + prefix=f"{prefix}.mlp", + ) + self.block_type = "feedforward" + else: + self.mlp = HYV3MoEFused( + config=config, quant_config=quant_config, prefix=f"{prefix}.mlp" + ) + self.block_type = "moe" + + def forward( + self, + positions: torch.Tensor, + hidden_states: torch.Tensor, + residual: torch.Tensor | None, + idx: int = -1, + ) -> torch.Tensor: + if residual is None: + residual = hidden_states + hidden_states = self.input_layernorm(hidden_states) + else: + hidden_states, residual = self.input_layernorm(hidden_states, residual) + + hidden_states = self.self_attn( + positions=positions, + hidden_states=hidden_states, + ) + + hidden_states, residual = self.post_attention_layernorm(hidden_states, residual) + + hidden_states = self.mlp(hidden_states) + + return hidden_states, residual + + +@support_torch_compile +class HYV3Model(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + + config = vllm_config.model_config.hf_config + cache_config = vllm_config.cache_config + quant_config = vllm_config.quant_config + + parallel_config = vllm_config.parallel_config + eplb_config = parallel_config.eplb_config + self.num_redundant_experts = eplb_config.num_redundant_experts + + self.vocab_size = config.vocab_size + self.config = config + self.quant_config = quant_config + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + + self.start_layer, self.end_layer, self.layers = make_layers( + config.num_hidden_layers, + lambda prefix: HYV3DecoderLayer( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ), + prefix=f"{prefix}.layers", + ) + self.norm = RMSNorm(config.hidden_size, config.rms_norm_eps) + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory( + ["hidden_states", "residual"], config.hidden_size + ) + + # Set MoE hyperparameters + self.expert_weights = [] + self.num_expert_groups = 1 + self.moe_layers = [] + example_layer = None + for layer in self.layers: + if isinstance(layer, PPMissingLayer): + continue + + assert isinstance(layer, HYV3DecoderLayer) + if layer.block_type == "moe": + example_layer = layer.mlp + self.moe_layers.append(layer.mlp.experts) + + if example_layer is None: + self.num_moe_layers = 0 + raise RuntimeError("No MoE layer found in model.layers.") + + self.num_moe_layers = len(self.moe_layers) + self.num_logical_experts = getattr(example_layer, "n_logical_experts", None) + self.num_physical_experts = getattr(example_layer, "n_physical_experts", None) + self.num_local_physical_experts = getattr( + example_layer, "n_local_physical_experts", None + ) + self.num_routed_experts = getattr(example_layer, "n_routed_experts", None) + self.num_redundant_experts = getattr(example_layer, "n_redundant_experts", None) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.embed_tokens(input_ids) + + def update_physical_experts_metadata( + self, + num_physical_experts: int, + num_local_physical_experts: int, + ) -> None: + assert self.num_local_physical_experts == num_local_physical_experts + self.num_physical_experts = num_physical_experts + self.num_local_physical_experts = num_local_physical_experts + self.num_redundant_experts = num_physical_experts - self.num_logical_experts + for layer in self.layers: + if isinstance(layer.mlp, HYV3MoEFused): + moe = layer.mlp + moe.n_local_physical_experts = num_local_physical_experts + moe.n_physical_experts = num_physical_experts + moe.n_redundant_experts = self.num_redundant_experts + moe.experts.update_expert_map() + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + # Params for weights, fp8 weight scales, fp8 activation scales + # (param_name, weight_name, expert_id, shard_id) + return FusedMoE.make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts, + ) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + if get_pp_group().is_first_rank: + if inputs_embeds is not None: + hidden_states = inputs_embeds + else: + hidden_states = self.embed_input_ids(input_ids) + residual = None + else: + assert intermediate_tensors is not None + hidden_states = intermediate_tensors["hidden_states"] + residual = intermediate_tensors["residual"] + + for idx, layer in enumerate( + islice(self.layers, self.start_layer, self.end_layer) + ): + hidden_states, residual = layer(positions, hidden_states, residual, idx=idx) + if not get_pp_group().is_last_rank: + return IntermediateTensors( + {"hidden_states": hidden_states, "residual": residual} + ) + + hidden_states = hidden_states + residual + residual = hidden_states + + hidden_states = self.norm(hidden_states) + + return hidden_states + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + params_dict = dict(self.named_parameters()) + expert_params_mapping = self.get_expert_mapping() + loaded_params: set[str] = set() + for name, loaded_weight in weights: + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + if self.quant_config is not None and ( + scale_name := self.quant_config.get_cache_scale(name) + ): + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + loaded_weight = ( + loaded_weight if loaded_weight.dim() == 0 else loaded_weight[0] + ) + weight_loader(param, loaded_weight) + loaded_params.add(scale_name) + continue + if "scale" in name: + # Remapping the name of FP8 kv-scale. + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + is_found = False + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + if "mlp.experts" in name: + continue + name = name.replace(weight_name, param_name) + # Skip loading extra bias for GPTQ models. + if name.endswith(".bias") and name not in params_dict: + continue + + # Skip layers on other devices. + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + loaded_params.add(name) + is_found = True + break + if is_found: + continue + + if name.endswith(".bias") and name not in params_dict: + continue + is_expert_weight = False + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + is_expert_weight = True + name_mapped = name.replace(weight_name, param_name) + # Skip layers on other devices. + if is_pp_missing_parameter(name_mapped, self): + continue + + param = params_dict[name_mapped] + weight_loader = typing.cast(Callable[..., bool], param.weight_loader) + success = weight_loader( + param, + loaded_weight, + name_mapped, + shard_id=shard_id, + expert_id=expert_id, + return_success=True, + ) + if success: + name = name_mapped + break + else: + if is_expert_weight: + # We've checked that this is an expert weight + # However it's not mapped locally to this rank + # So we simply skip it + continue + if name is None: + continue + if is_pp_missing_parameter(name, self): + continue + if "router.gate." in name: + name = name.replace("router.", "") + + param = params_dict[name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + weight_loader(param, loaded_weight) + loaded_params.add(name) + + return loaded_params + + +def get_spec_layer_idx_from_weight_name( + config: PretrainedConfig, weight_name: str +) -> int | None: + # HYV3MTP is enabled only when num_nextn_predict_layers is greater than 1 + if ( + hasattr(config, "num_nextn_predict_layers") + and config.num_nextn_predict_layers > 0 + ): + layer_idx = config.num_hidden_layers + for i in range(config.num_nextn_predict_layers): + if weight_name.startswith(f"model.layers.{layer_idx + i}."): + return layer_idx + i + return None + + +class HYV3ForCausalLM(nn.Module, SupportsPP, SupportsLoRA): + packed_modules_mapping = { + "qkv_proj": ["q_proj", "k_proj", "v_proj"], + "gate_up_proj": ["gate_proj", "up_proj"], + } + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + quant_config = vllm_config.quant_config + self.config = config + self.quant_config = quant_config + + parallel_config = vllm_config.parallel_config + eplb_config = parallel_config.eplb_config + self.num_redundant_experts = eplb_config.num_redundant_experts + + self.model = HYV3Model( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + self.lm_head = ParallelLMHead( + config.vocab_size, + config.hidden_size, + quant_config=quant_config, + prefix=maybe_prefix(prefix, "lm_head"), + ) + if self.config.tie_word_embeddings: + self.lm_head.weight = self.model.embed_tokens.weight + self.logits_processor = LogitsProcessor(config.vocab_size) + self.make_empty_intermediate_tensors = ( + self.model.make_empty_intermediate_tensors + ) + + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor: + return self.model.embed_input_ids(input_ids) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor | IntermediateTensors: + hidden_states = self.model( + input_ids, positions, intermediate_tensors, inputs_embeds + ) + + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + ) -> torch.Tensor | None: + logits = self.logits_processor(self.lm_head, hidden_states) + return logits + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: + def _filter_weights(weights): + for name, weight in weights: + spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) + if spec_layer is not None: + continue + yield name, weight + + loader = AutoWeightsLoader( + self, + skip_prefixes=(["lm_head."] if self.config.tie_word_embeddings else None), + ) + return loader.load_weights(_filter_weights(weights)) + + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: + return self.model.get_expert_mapping() diff --git a/vllm/model_executor/models/hy_v3_mtp.py b/vllm/model_executor/models/hy_v3_mtp.py new file mode 100644 index 00000000000..8594a38c3ab --- /dev/null +++ b/vllm/model_executor/models/hy_v3_mtp.py @@ -0,0 +1,470 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# coding=utf-8 +# Copyright 2026 The HY team. +# Copyright 2023 The vLLM team. +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved. +# +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX +# and OPT implementations in this library. It has been modified from its +# original forms to accommodate minor architectural differences compared +# to GPT-NeoX and OPT used by the Meta AI team that trained the model. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only HY V3 MTP model compatible with HuggingFace weights.""" + +from collections.abc import Iterable + +import regex as re +import torch +from torch import nn +from transformers import PretrainedConfig + +from vllm.config import CacheConfig, ModelConfig, VllmConfig +from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.logits_processor import LogitsProcessor +from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.vocab_parallel_embedding import ( + ParallelLMHead, + VocabParallelEmbedding, +) +from vllm.model_executor.model_loader.weight_utils import ( + default_weight_loader, + maybe_remap_kv_scale_name, +) +from vllm.sequence import IntermediateTensors +from vllm.v1.outputs import SamplerOutput +from vllm.v1.sample.metadata import SamplingMetadata +from vllm.v1.sample.sampler import Sampler + +from .hy_v3 import HYV3DecoderLayer, get_spec_layer_idx_from_weight_name +from .utils import is_pp_missing_parameter, maybe_prefix + + +def _is_moe(config: PretrainedConfig) -> bool: + return bool( + getattr(config, "num_experts", None) + and ( + (isinstance(config.num_experts, int) and config.num_experts > 1) + or (isinstance(config.num_experts, list) and max(config.num_experts) > 1) + ) + ) + + +def _get_cla_factor(config: PretrainedConfig) -> int: + if not getattr(config, "use_cla", False): + return 1 + return getattr(config, "cla_share_factor", 1) + + +class HYV3SharedHead(nn.Module): + def __init__( + self, + config: PretrainedConfig, + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__() + self.head = ParallelLMHead( + config.vocab_size, config.hidden_size, quant_config=quant_config + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + return hidden_states + + +class HYV3MultiTokenPredictorLayer(nn.Module): + def __init__( + self, + config: PretrainedConfig, + prefix: str, + model_config: ModelConfig, + cache_config: CacheConfig | None = None, + quant_config: QuantizationConfig | None = None, + ) -> None: + super().__init__() + + self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + self.shared_head = HYV3SharedHead(config=config, quant_config=quant_config) + self.mtp_block = HYV3DecoderLayer( + config=config, + cache_config=cache_config, + quant_config=quant_config, + prefix=prefix, + ) + # Final layernorm applied after transformer block, before logits + # projection (matches HF HYV3MTPDecoderLayer.final_layernorm) + self.final_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + spec_step_index: int = 0, + ) -> torch.Tensor: + assert inputs_embeds is not None + # masking inputs at position 0, as not needed by MTP + inputs_embeds[positions == 0] = 0 + inputs_embeds = self.enorm(inputs_embeds) + previous_hidden_states = self.hnorm(previous_hidden_states) + + hidden_states = self.eh_proj( + torch.cat([inputs_embeds, previous_hidden_states], dim=-1) + ) + + # HYV3DecoderLayer returns (hidden_states, residual) + hidden_states, residual = self.mtp_block( + positions=positions, hidden_states=hidden_states, residual=None + ) + hidden_states = residual + hidden_states + hidden_states = self.final_layernorm(hidden_states) + return hidden_states + + +class HYV3MultiTokenPredictor(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config = vllm_config.model_config.hf_config + self.mtp_start_layer_idx = config.num_hidden_layers + self.num_mtp_layers = config.num_nextn_predict_layers + + # to map the exact layer index from weights + self.layers = torch.nn.ModuleDict( + { + str(idx): HYV3MultiTokenPredictorLayer( + config, + f"{prefix}.layers.{idx}", + model_config=vllm_config.model_config, + cache_config=vllm_config.cache_config, + quant_config=vllm_config.quant_config, + ) + for idx in range( + self.mtp_start_layer_idx, + self.mtp_start_layer_idx + self.num_mtp_layers, + ) + } + ) + + self.embed_tokens = VocabParallelEmbedding( + config.vocab_size, + config.hidden_size, + ) + + self.logits_processor = LogitsProcessor(config.vocab_size) + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + previous_hidden_states: torch.Tensor, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + current_step_idx = spec_step_idx % self.num_mtp_layers + return self.layers[str(self.mtp_start_layer_idx + current_step_idx)]( + input_ids, + positions, + previous_hidden_states, + inputs_embeds, + current_step_idx, + ) + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor: + current_step_idx = spec_step_idx % self.num_mtp_layers + mtp_layer = self.layers[str(self.mtp_start_layer_idx + current_step_idx)] + logits = self.logits_processor( + mtp_layer.shared_head.head, mtp_layer.shared_head(hidden_states) + ) + return logits + + +class HYV3MTP(nn.Module): + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + self.config = vllm_config.model_config.hf_config + self.quant_config = vllm_config.quant_config + self.model = HYV3MultiTokenPredictor( + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") + ) + + self.sampler = Sampler() + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + hidden_states: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + spec_step_idx: int = 0, + ) -> torch.Tensor: + hidden_states = self.model( + input_ids, positions, hidden_states, inputs_embeds, spec_step_idx + ) + return hidden_states + + def compute_logits( + self, + hidden_states: torch.Tensor, + spec_step_idx: int = 0, + ) -> torch.Tensor | None: + return self.model.compute_logits(hidden_states, spec_step_idx) + + def sample( + self, + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, + ) -> SamplerOutput | None: + next_tokens = self.sampler(logits, sampling_metadata) + return next_tokens + + def _split_qkv_weight(self, qkv: torch.Tensor): + num_attention_heads = self.config.num_attention_heads + num_kv_heads = getattr( + self.config, "num_key_value_heads", self.config.num_attention_heads + ) + num_key_value_groups = num_attention_heads // num_kv_heads + hidden_size = self.config.hidden_size + + if hasattr(self.config, "head_dim"): + attention_head_dim = self.config.head_dim + elif hasattr(self.config, "attention_head_dim"): + attention_head_dim = self.config.attention_head_dim + else: + attention_head_dim = self.config.hidden_size // num_attention_heads + + qkv = qkv.reshape( + num_kv_heads, num_key_value_groups + 2, attention_head_dim, hidden_size + ) + q, k, v = torch.split(qkv, (num_key_value_groups, 1, 1), dim=1) + q = q.reshape(-1, hidden_size) + k = k.reshape(-1, hidden_size) + v = v.reshape(-1, hidden_size) + return torch.concat((q, k, v)) + + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]): + cla_factor = _get_cla_factor(self.config) + stacked_params_mapping = [ + # (param_name, shard_name, shard_id) + (".qkv_proj", ".q_proj", "q"), + (".qkv_proj", ".k_proj", "k"), + (".qkv_proj", ".v_proj", "v"), + (".gate_up_proj", ".gate_proj", 0), + (".gate_up_proj", ".up_proj", 1), + ] + + num_attention_heads = self.config.num_attention_heads + num_kv_heads = getattr( + self.config, "num_key_value_heads", self.config.num_attention_heads + ) + split_params_mapping = [ + (".gate_up_proj", ".gate_and_up_proj", 2, [(1, 1), (0, 1)], None), + ( + ".qkv_proj", + ".qkv_proj", + num_attention_heads + num_kv_heads * 2, + [("q", num_attention_heads), ("k", num_kv_heads), ("v", num_kv_heads)], + self._split_qkv_weight, + ), + ] + + if _is_moe(self.config): + expert_params_mapping = FusedMoE.make_expert_params_mapping( + self, + ckpt_gate_proj_name="gate_proj", + ckpt_down_proj_name="down_proj", + ckpt_up_proj_name="up_proj", + num_experts=self.config.num_experts, + ) + else: + expert_params_mapping = {} + + params_dict = dict(self.named_parameters()) + + # V3 shared weights mapping: + # - embed_tokens: from main model's model.embed_tokens.weight + # - lm_head: from main model's lm_head.weight → MTP shared_head.head + # (HF infer_mtp uses head_weight=self.lm_head.weight, not the + # checkpoint's model.layers..shared_head.weight) + # - No norm mapping (V3 MTP has no intermediate norm before lm_head) + mtp_start = self.config.num_hidden_layers + v3_shared_weights = { + "model.embed_tokens.weight": "model.embed_tokens.weight", + "lm_head.weight": f"model.layers.{mtp_start}.shared_head.head.weight", + } + + for name, loaded_weight in weights: + # Intercept shared weights before any other processing + if name in v3_shared_weights: + target_name = v3_shared_weights[name] + if target_name in params_dict: + param = params_dict[target_name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + continue + + if "rotary_emb.inv_freq" in name: + continue + if "gate_proj_bias" in name: + name = name.replace("gate_proj_bias", "gate_proj.bias") + if "up_proj_bias" in name: + name = name.replace("up_proj_bias", "up_proj.bias") + if "rotary_emb.cos_cached" in name or "rotary_emb.sin_cached" in name: + continue + if self.config.tie_word_embeddings and "lm_head.weight" in name: + continue + if self.quant_config is not None and ( + scale_name := self.quant_config.get_cache_scale(name) + ): + param = params_dict[scale_name] + weight_loader = getattr(param, "weight_loader", default_weight_loader) + loaded_weight = loaded_weight[0] + weight_loader(param, loaded_weight) + continue + spec_layer = get_spec_layer_idx_from_weight_name(self.config, name) + if spec_layer is None: + continue + name = self._rewrite_spec_layer_name(spec_layer, name) + # Skip weights that _rewrite_spec_layer_name marked for skipping + if name == "__skip__": + continue + if "scale" in name: + name = maybe_remap_kv_scale_name(name, params_dict) + if name is None: + continue + is_found = False + + for param_name, weight_name, shard_id in stacked_params_mapping: + if weight_name not in name: + continue + if "mlp.experts" in name: + continue + if weight_name == ".q_proj": + match = re.search(r"layers\.\d+", name) + if match: + layer_id = int(match.group(0).split(".")[-1]) + if cla_factor > 1 and layer_id % cla_factor != 0: + continue + name = name.replace(weight_name, param_name) + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader(param, loaded_weight, shard_id) + + is_found = True + break + if is_found: + continue + + for param_name, weight_name, den, split_param, func in split_params_mapping: + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if name.endswith(".bias") and name not in params_dict: + continue + + if is_pp_missing_parameter(name, self): + continue + + assert loaded_weight.shape[0] % den == 0 + units = loaded_weight.shape[0] // den + + param = params_dict[name] + weight_loader = param.weight_loader + offset = 0 + for shard_id, num in split_param: + new_offset = offset + num * units + if func: + weight_loader( + param, func(loaded_weight)[offset:new_offset], shard_id + ) + else: + weight_loader(param, loaded_weight[offset:new_offset], shard_id) + offset = new_offset + + break + else: + if name.endswith(".bias") and name not in params_dict: + continue + for mapping in expert_params_mapping: + param_name, weight_name, expert_id, shard_id = mapping + if weight_name not in name: + continue + name = name.replace(weight_name, param_name) + if is_pp_missing_parameter(name, self): + continue + param = params_dict[name] + weight_loader = param.weight_loader + weight_loader( + param, + loaded_weight, + name, + shard_id=shard_id, + expert_id=expert_id, + ) + break + else: + if is_pp_missing_parameter(name, self): + continue + + if "mlp.gate.wg." in name: + name = name.replace("wg.", "") + # V3 checkpoint: mlp.router.gate -> mlp.gate + if "mlp.router.gate." in name: + name = name.replace("router.gate.", "gate.") + + param = params_dict[name] + weight_loader = getattr( + param, "weight_loader", default_weight_loader + ) + weight_loader(param, loaded_weight) + + def _rewrite_spec_layer_name(self, spec_layer: int, name: str) -> str: + """Rewrite spec layer weight names to match vLLM module structure.""" + # Skip embed_tokens (doesn't exist in V3 MTP checkpoint under spec + # layer) and shared_head (we use main model's lm_head instead) + if f"model.layers.{spec_layer}.embed_tokens" in name: + return "__skip__" + if f"model.layers.{spec_layer}.shared_head" in name: + return "__skip__" + + spec_layer_weight_names = ["enorm", "hnorm", "eh_proj", "final_layernorm"] + spec_layer_weight = False + for weight_name in spec_layer_weight_names: + if weight_name in name: + spec_layer_weight = True + break + if not spec_layer_weight: + # Transformer block weights go under .mtp_block + name = name.replace( + f"model.layers.{spec_layer}.", f"model.layers.{spec_layer}.mtp_block." + ) + return name diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index baac4a6c664..3f80f66161f 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -133,6 +133,7 @@ _TEXT_GENERATION_MODELS = { "Grok1ForCausalLM": ("grok1", "GrokForCausalLM"), "HunYuanMoEV1ForCausalLM": ("hunyuan_v1", "HunYuanMoEV1ForCausalLM"), "HunYuanDenseV1ForCausalLM": ("hunyuan_v1", "HunYuanDenseV1ForCausalLM"), + "HYV3ForCausalLM": ("hy_v3", "HYV3ForCausalLM"), "HCXVisionForCausalLM": ("hyperclovax_vision", "HCXVisionForCausalLM"), "HCXVisionV2ForCausalLM": ("hyperclovax_vision_v2", "HCXVisionV2ForCausalLM"), "HyperCLOVAXForCausalLM": ("hyperclovax", "HyperCLOVAXForCausalLM"), @@ -599,6 +600,7 @@ _SPECULATIVE_DECODING_MODELS = { "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"), "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"), "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"), + "HYV3MTPModel": ("hy_v3_mtp", "HYV3MTP"), # Temporarily disabled. # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1. # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"), diff --git a/vllm/reasoning/__init__.py b/vllm/reasoning/__init__.py index 37d8a9b1dab..42b522f691e 100644 --- a/vllm/reasoning/__init__.py +++ b/vllm/reasoning/__init__.py @@ -56,6 +56,10 @@ _REASONING_PARSERS_TO_REGISTER = { "hunyuan_a13b_reasoning_parser", "HunyuanA13BReasoningParser", ), + "hy_v3": ( + "hy_v3_reasoning_parser", + "HYV3ReasoningParser", + ), "kimi_k2": ( "kimi_k2_reasoning_parser", "KimiK2ReasoningParser", diff --git a/vllm/reasoning/hy_v3_reasoning_parser.py b/vllm/reasoning/hy_v3_reasoning_parser.py new file mode 100644 index 00000000000..6acaa13bb76 --- /dev/null +++ b/vllm/reasoning/hy_v3_reasoning_parser.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from collections.abc import Iterable, Sequence + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, +) +from vllm.entrypoints.openai.engine.protocol import DeltaMessage +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.logger import init_logger +from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser +from vllm.reasoning.identity_reasoning_parser import IdentityReasoningParser +from vllm.tokenizers import TokenizerLike + +logger = init_logger(__name__) + + +class HYV3ReasoningParser(BaseThinkingReasoningParser): + """ + HYV3 parser that delegates to either HYV3ReasoningParser or + IdentityReasoningParser based on `reasoning_effort`. + + The HYV3 model uses ... tokens to denote reasoning text. + This parser extracts the reasoning content from the model output. + """ + + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): + super().__init__(tokenizer, *args, **kwargs) + + # First, If there is reasoning_effort in chat_kwargs, + # prioritize using chat_kwargs.reasoning_effort. + # If it's not present, use the "reasoning_effort" field + # at the outer level of the chat message. + # Otherwise, If both are empty, assign "no_think". + + chat_kwargs = kwargs.pop("chat_template_kwargs", {}) or {} + reasoning_effort = chat_kwargs.pop("reasoning_effort", "no_think") + + logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort) + + self._identity_parser: IdentityReasoningParser | None + if reasoning_effort == "no_think": + self._identity_parser = IdentityReasoningParser(tokenizer, *args, **kwargs) + else: + self._identity_parser = None + + @property + def start_token(self) -> str: + """The token that starts reasoning content.""" + return "" + + @property + def end_token(self) -> str: + """The token that ends reasoning content.""" + return "" + + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + if self._identity_parser is not None: + return self._identity_parser.is_reasoning_end(input_ids) + + return super().is_reasoning_end(input_ids) + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + if self._identity_parser is not None: + return self._identity_parser.is_reasoning_end_streaming( + input_ids, delta_ids + ) + + return super().is_reasoning_end_streaming(input_ids, delta_ids) + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + if self._identity_parser is not None: + return self._identity_parser.extract_content_ids(input_ids) + + return super().extract_content_ids(input_ids) + + def extract_reasoning( + self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" + ) -> tuple[str | None, str | None]: + if self._identity_parser is not None: + return self._identity_parser.extract_reasoning(model_output, request) + + return super().extract_reasoning(model_output, request) + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + if self._identity_parser is not None: + return self._identity_parser.extract_reasoning_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) + + ret = super().extract_reasoning_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + delta_token_ids, + ) + if ( + ret is not None + and self.start_token_id not in previous_token_ids + and self.start_token_id not in delta_token_ids + ): + if self.end_token_id in delta_token_ids: + # end token in delta with more tokens, + # extract reasoning content and content + end_index = delta_text.find(self.end_token) + reasoning = delta_text[:end_index] + content = delta_text[end_index + len(self.end_token) :] + return DeltaMessage( + reasoning=reasoning, + content=content if content else None, + ) + elif self.end_token_id in previous_token_ids: + # end token in previous, thinking content ends + return DeltaMessage(content=delta_text) + else: + # no end token in previous or delta, reasoning content continues + return DeltaMessage(reasoning=delta_text) + + return ret diff --git a/vllm/tool_parsers/__init__.py b/vllm/tool_parsers/__init__.py index abfa8f3fdbe..7d5ea8d5ea7 100644 --- a/vllm/tool_parsers/__init__.py +++ b/vllm/tool_parsers/__init__.py @@ -66,6 +66,10 @@ _TOOL_PARSERS_TO_REGISTER = { "hunyuan_a13b_tool_parser", "HunyuanA13BToolParser", ), + "hy_v3": ( + "hy_v3_tool_parser", + "HYV3ToolParser", + ), "internlm": ( "internlm2_tool_parser", "Internlm2ToolParser", diff --git a/vllm/tool_parsers/hy_v3_tool_parser.py b/vllm/tool_parsers/hy_v3_tool_parser.py new file mode 100644 index 00000000000..809a85ce417 --- /dev/null +++ b/vllm/tool_parsers/hy_v3_tool_parser.py @@ -0,0 +1,645 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import ast +import json +from collections.abc import Sequence +from typing import Any + +import regex as re + +from vllm.entrypoints.chat_utils import make_tool_call_id +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.entrypoints.openai.engine.protocol import ( + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + ExtractedToolCallInformation, + FunctionCall, + ToolCall, +) +from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.abstract_tool_parser import ( + Tool, + ToolParser, +) + +logger = init_logger(__name__) + + +class HYV3ToolParser(ToolParser): + _TYPE_ALIASES: dict[str, str] = { + "str": "string", + "text": "string", + "varchar": "string", + "char": "string", + "enum": "string", + "bool": "boolean", + "binary": "boolean", + "int": "integer", + "float": "number", + "double": "number", + "list": "array", + "dict": "object", + "map": "object", + } + + # Prefix-based wildcard matching for non-standard type names. + # Following the same approach as + # qwen3coder_tool_parser._convert_param_value which uses + # param_type.startswith("int"), startswith("uint"), etc. + _INTEGER_PREFIXES: tuple[str, ...] = ( + "int", + "uint", + "long", + "short", + "unsigned", + ) + _NUMBER_PREFIXES: tuple[str, ...] = ("num", "float") + + @staticmethod + def _normalize_type(raw_type: str) -> str: + """Map non-standard type aliases to JSON Schema standard names. + + First performs exact lookup in _TYPE_ALIASES. On miss, falls back + to prefix-based matching using startswith() + - int*/uint*/long*/short*/unsigned* → "integer" + - num*/float* → "number" + """ + exact = HYV3ToolParser._TYPE_ALIASES.get(raw_type) + if exact is not None: + return exact + lower = raw_type.lower() + if any(lower.startswith(p) for p in HYV3ToolParser._INTEGER_PREFIXES): + return "integer" + if any(lower.startswith(p) for p in HYV3ToolParser._NUMBER_PREFIXES): + return "number" + return raw_type + + @staticmethod + def _get_arg_schema( + function_name: str, + arg_key: str, + tools: list[ChatCompletionToolsParam] | None, + ) -> dict: + """Look up a specific argument's property schema from the tools list.""" + if tools is None: + return {} + for tool in tools: + if tool.function.name == function_name: + if tool.function.parameters is None: + return {} + return tool.function.parameters.get("properties", {}).get(arg_key, {}) + logger.warning("No tool named '%s'.", function_name) + return {} + + @staticmethod + def _get_schema_options(arg_schema: dict) -> list[dict]: + """Normalize any property schema into a list of sub-schemas. + - has type (single type) → return [arg_schema] + - anyOf → return the anyOf list + - oneOf → return the oneOf list + - fallback → [{"type": "string"}] + + Note: single ``type`` has the highest priority. + """ + if "type" in arg_schema: + return [arg_schema] + if "anyOf" in arg_schema: + return arg_schema["anyOf"] + if "oneOf" in arg_schema: + return arg_schema["oneOf"] + + return [{"type": "string"}] + + @staticmethod + def _get_types(arg_schema: dict) -> set[str]: + """Extract normalized, non-null type set from a property schema.""" + schemas = HYV3ToolParser._get_schema_options(arg_schema) + return { + HYV3ToolParser._normalize_type(s.get("type", "string")) for s in schemas + } - {"null"} + + @staticmethod + def _is_only_string_type( + function_name: str, + arg_key: str, + tools: list[ChatCompletionToolsParam] | None, + ) -> bool: + """Return True if the parameter's type set is exactly {"string"}. + + Only pure string types get partial value streaming; compound types + like anyOf(string | array) do not, since the partial value might + end up being a JSON array or object. + """ + arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools) + types = HYV3ToolParser._get_types(arg_schema) + return types == {"string"} + + @staticmethod + def _try_parse_bool(value: str) -> bool | None: + """Try to parse a string as bool; return None on failure.""" + lower = value.lower() + if lower == "true": + return True + elif lower == "false": + return False + return None + + @staticmethod + def _try_parse_int(value: str) -> int | None: + """Try to parse a string as int; return None on failure.""" + try: + return int(value) + except (ValueError, TypeError): + return None + + @staticmethod + def _try_parse_wildcard_number(value: str) -> int | float | None: + """Try to parse a string as a number (int or float). + + Decision rule: if the string contains '.' or 'e'/'E' (scientific + notation), parse as float; otherwise parse as int. + + Examples: + "5" → int(5) + "5.0" → float(5.0) + "5.3" → float(5.3) + "1e3" → float(1000.0) + "-3" → int(-3) + + Return None on failure. + """ + try: + if "." in value or "e" in value or "E" in value: + return float(value) + return int(value) + except (ValueError, TypeError): + return None + + @staticmethod + def _deserialize(value: str) -> Any: + """Deserialize a string value using json.loads then ast.literal_eval.""" + try: + return json.loads(value) + except Exception: + pass + try: + return ast.literal_eval(value) + except Exception: + pass + return value + + @staticmethod + def _parse_value( + value: str, + function_name: str, + arg_key: str, + tools: list[ChatCompletionToolsParam] | None, + ) -> Any: + """Unified argument value parser with anyOf/oneOf support. + + Fallthrough chain: + bool → int → number(wildcard_number) + → json.loads for array/object + → string → _deserialize + """ + arg_schema = HYV3ToolParser._get_arg_schema(function_name, arg_key, tools) + types = HYV3ToolParser._get_types(arg_schema) + + # 1. Try bool + if "boolean" in types: + result_bool = HYV3ToolParser._try_parse_bool(value) + if result_bool is not None: + return result_bool + + # 2. Try int + if "integer" in types: + result_int = HYV3ToolParser._try_parse_int(value) + if result_int is not None: + return result_int + + # 3. Try number (wildcard_number: int if no '.'/e/E, float otherwise) + if "number" in types: + result_number = HYV3ToolParser._try_parse_wildcard_number(value) + if result_number is not None: + return result_number + + # 4. Try json.loads (covers array/object and other unlisted types) + if types - {"string", "boolean", "integer", "number"}: + try: + return json.loads(value) + except (json.JSONDecodeError, ValueError): + pass + + # 5. String fallback + if "string" in types: + return value + + # 6. Final fallback + return HYV3ToolParser._deserialize(value) + + def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): + super().__init__(tokenizer, tools) + + self.current_tool_name_sent: bool = False + self.prev_tool_call_arr: list[dict] = [] + self.current_tool_id: int = -1 + self.streamed_args_for_tool: list[ + str + ] = [] # map what has been streamed for each tool so far to a list + + # Streaming state: send tool name first, then return arguments at once + self._streaming_tool_name: str | None = None # tool name being streamed + + # State fields for incremental argument streaming + self._completed_args: dict = {} # closed {key: parsed_value} + self._current_arg_key: str | None = None # key being collected + self._current_arg_is_string: bool = False # is current arg pure string? + self._streamed_json_len: int = 0 # bytes of JSON already sent + + self.tool_calls_start_token: str = "" + self.tool_calls_end_token: str = "" + + self.tool_call_start_token: str = "" + self.tool_call_end_token: str = "" + + self.tool_sep_token: str = "" + + self.arg_key_start_token: str = "" + self.arg_key_end_token: str = "" + + self.arg_value_start_token: str = "" + self.arg_value_end_token: str = "" + + self.tool_call_regex = re.compile( + rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}" + rf"(.*?){self.tool_call_end_token}", + re.DOTALL, + ) + + self.tool_call_portion_regex = re.compile( + rf"{self.tool_call_start_token}(.*?){self.tool_sep_token}(.*)", re.DOTALL + ) + + self.func_args_regex = re.compile( + rf"{self.arg_key_start_token}(.*?){self.arg_key_end_token}\s*" + rf"{self.arg_value_start_token}(.*?){self.arg_value_end_token}", + re.DOTALL, + ) + + if not self.model_tokenizer: + raise ValueError( + "The model tokenizer must be passed to the ToolParser " + "constructor during construction." + ) + self.tool_calls_start_token_id = self.vocab.get(self.tool_calls_start_token) + self.tool_calls_end_token_id = self.vocab.get(self.tool_calls_end_token) + + self.tool_call_start_token_id = self.vocab.get(self.tool_call_start_token) + self.tool_call_end_token_id = self.vocab.get(self.tool_call_end_token) + self._buffer = "" + + if ( + self.tool_calls_start_token_id is None + or self.tool_calls_end_token_id is None + ): + raise RuntimeError( + "HYV3 Tool parser could not locate tool call " + "start/end tokens in the tokenizer!" + ) + + def _extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> list[ToolCall]: + try: + function_call_tuples = [] + # start_token{name}sep_token{args}end_token... + function_calls = self.tool_call_regex.findall(model_output) + if function_calls: + function_call_tuples.extend(function_calls) + remaining = model_output.split(self.tool_call_end_token)[-1] + function_calls = self.tool_call_portion_regex.findall(remaining) + function_call_tuples += function_calls + else: + function_calls = self.tool_call_portion_regex.findall(model_output) + if function_calls: + function_call_tuples.extend(function_calls) + tool_calls = [] + for match in function_call_tuples: + function_name, function_args = match + function_name = function_name.strip() + function_args = function_args.strip() + + arg_pairs = self.func_args_regex.findall(function_args) + arg_dict = {} + for key, value in arg_pairs: + parsed_value = HYV3ToolParser._parse_value( + value, function_name, key, request.tools + ) + arg_dict[key] = parsed_value + tool_calls.append( + ToolCall( + type="function", + function=FunctionCall( + name=function_name, + arguments=json.dumps(arg_dict, ensure_ascii=False), + ), + ) + ) + return tool_calls + except Exception: + logger.exception("Error in extracting tool call from response.") + return [] + + def extract_tool_calls( + self, + model_output: str, + request: ChatCompletionRequest, + ) -> ExtractedToolCallInformation: + # sanity check; avoid unnecessary processing + if self.tool_calls_start_token not in model_output: + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + else: + try: + tool_calls = self._extract_tool_calls(model_output, request) + + s_index = model_output.find(self.tool_calls_start_token) + content = model_output[:s_index] if s_index != -1 else model_output + return ExtractedToolCallInformation( + tools_called=True, + tool_calls=tool_calls, + content=content if content else None, + ) + + except Exception: + logger.exception("Error in extracting tool call from response.") + return ExtractedToolCallInformation( + tools_called=False, tool_calls=[], content=model_output + ) + + def _reset_streaming_tool_state(self): + """Reset the streaming state for a single tool call.""" + self._streaming_tool_name = None + self._completed_args = {} + self._current_arg_key = None + self._current_arg_is_string = False + self._streamed_json_len = 0 + + def extract_tool_calls_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + request: ChatCompletionRequest, + ) -> DeltaMessage | None: + # Check whether current tokens contain the tool_calls start token + if self.tool_calls_start_token_id not in current_token_ids: + return DeltaMessage(content=delta_text) + + # Encountered tool_calls start tag; extract preceding content and buffer + if self.tool_calls_start_token in delta_text: + text_parts = delta_text.split(self.tool_calls_start_token) + self._buffer += text_parts[-1] + if text_parts[0]: + return DeltaMessage(content=text_parts[0]) + # Don't return None; continue processing buffer for complete content + else: + self._buffer += delta_text + + # Encountered finish, extract valid arguments + if ( + current_text.find(self.tool_call_end_token + self.tool_calls_end_token) + != -1 + and self._buffer.find(self.tool_call_end_token) == -1 + ): + self._buffer += self.tool_call_end_token + self.tool_calls_end_token + + cur_text = self._buffer + + # Haven't encountered tool_call start tag yet; keep buffering + start_idx = cur_text.find(self.tool_call_start_token) + if start_idx == -1 and self._streaming_tool_name is None: + self._buffer = "" + return None + + # === Phase 1: Detect tool name (send when tool_sep_token is seen) === + name_delta: DeltaMessage | None = None + if self._streaming_tool_name is None: + sep_idx = cur_text.find(self.tool_sep_token) + if sep_idx == -1: + # tool_sep not yet seen; keep buffering from tool_call_start + self._buffer = cur_text[start_idx:] + return None + + # Extract tool name: between tool_call_start_token and tool_sep_token + name_start = start_idx + len(self.tool_call_start_token) + tool_name = cur_text[name_start:sep_idx].strip() + self._streaming_tool_name = tool_name + + # Update buffer: keep only content after tool_sep (i.e. the args portion) + self._buffer = cur_text[sep_idx + len(self.tool_sep_token) :] + + # Increment tool_id and send a chunk containing only the name + self.current_tool_id += 1 + self._current_tool_call_id = make_tool_call_id() + name_delta = DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_id, + id=self._current_tool_call_id, + type="function", + function=DeltaFunctionCall( + name=tool_name, + ), + ) + ] + ) + + # Check if buffer already has complete arguments (all-in-one-delta) + if self.tool_call_end_token not in self._buffer: + return name_delta + # Buffer already has a complete tool call; continue to phase 2 below + + # === Phase 2: Incremental argument streaming === + return self._extract_streaming_incremental(name_delta, request) + + def _make_args_delta(self, argument_diff: str) -> DeltaMessage: + """Build a DeltaMessage containing only an arguments diff.""" + return DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_id, + function=DeltaFunctionCall(arguments=argument_diff), + ) + ] + ) + + def _extract_streaming_incremental( + self, + name_delta: DeltaMessage | None, + request: ChatCompletionRequest, + ) -> DeltaMessage | None: + """Incremental phase-2: scan tags in buffer, emit JSON diffs. + + Strategy: + - Track completed args and emit each one as a JSON fragment. + - For string-typed args, stream the value character-by-character. + - Withhold the closing ``}`` until ```` is seen. + + We build JSON manually via fragments rather than using json.dumps + with a cursor, because json.dumps of partial-vs-full string values + produces incompatible prefixes (e.g. ``""}`` vs ``"Hello"}``). + """ + buf = self._buffer + is_complete = self.tool_call_end_token in buf + + if is_complete: + end_idx = buf.find(self.tool_call_end_token) + args_text = buf[:end_idx] + remaining = buf[end_idx + len(self.tool_call_end_token) :] + else: + args_text = buf + remaining = "" + + # --- scan all fully closed kv pairs --- + arg_pairs = self.func_args_regex.findall(args_text) + for key, value in arg_pairs: + key = key.strip() + if key not in self._completed_args: + parsed_value = HYV3ToolParser._parse_value( + value, self._streaming_tool_name or "", key, request.tools + ) + self._completed_args[key] = parsed_value + + # --- detect partial (unclosed) kv at the tail --- + last_closed_end = 0 + for m in self.func_args_regex.finditer(args_text): + last_closed_end = m.end() + tail = args_text[last_closed_end:] + + partial_key: str | None = None + partial_value: str | None = None + + ak_start = tail.find(self.arg_key_start_token) + if ak_start != -1: + ak_end = tail.find( + self.arg_key_end_token, + ak_start + len(self.arg_key_start_token), + ) + if ak_end != -1: + partial_key = tail[ + ak_start + len(self.arg_key_start_token) : ak_end + ].strip() + self._current_arg_key = partial_key + self._current_arg_is_string = HYV3ToolParser._is_only_string_type( + self._streaming_tool_name or "", + partial_key, + request.tools, + ) + + av_start = tail.find(self.arg_value_start_token, ak_end) + if av_start != -1: + val_content_start = av_start + len(self.arg_value_start_token) + if self._current_arg_is_string: + partial_value = tail[val_content_start:] + else: + # key not yet closed + self._current_arg_key = None + self._current_arg_is_string = False + + # --- build the current JSON snapshot as a string --- + # We construct JSON manually so we can precisely control + # what gets sent incrementally. + snapshot_parts: list[str] = [] + for k, v in self._completed_args.items(): + k_json = json.dumps(k, ensure_ascii=False) + v_json = json.dumps(v, ensure_ascii=False) + snapshot_parts.append(f"{k_json}: {v_json}") + + if partial_key is not None and partial_value is not None: + k_json = json.dumps(partial_key, ensure_ascii=False) + # For string partial value, we build the JSON string + # WITHOUT the closing quote, so the prefix stays stable + # as the value grows. The closing `"` and `}` will be + # sent when the value or tool_call closes. + escaped_val = ( + partial_value.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("\n", "\\n") + .replace("\r", "\\r") + .replace("\t", "\\t") + ) + # Note: no closing " here – it's appended only on close + snapshot_parts.append(f'{k_json}: "{escaped_val}') + + snapshot = "{" + ", ".join(snapshot_parts) + "}" + + # --- compute diff --- + argument_diff: str | None = None + + if is_complete: + # Tool call finished – send everything remaining. + # Build final snapshot with proper JSON (all values closed). + final_args = dict(self._completed_args) + final_json = json.dumps(final_args, ensure_ascii=False) + if self._streamed_json_len < len(final_json): + argument_diff = final_json[self._streamed_json_len :] + self._streamed_json_len = len(final_json) + + # Record into prev_tool_call_arr + self.prev_tool_call_arr.append( + { + "name": self._streaming_tool_name, + "arguments": final_args, + } + ) + self.streamed_args_for_tool.append(final_json) + + self._reset_streaming_tool_state() + self._buffer = remaining + else: + # Still in progress – withhold the tail. + # For open strings: snapshot ends with ...partial_val} + # we withhold "}" (1 char) – the missing closing " will + # be sent when the value closes. + # For no open string: snapshot ends with ...value"} + # we withhold "}" (1 char). + end = len(snapshot) - 1 # exclude trailing "}" + if end > self._streamed_json_len: + argument_diff = snapshot[self._streamed_json_len : end] + self._streamed_json_len = end + + # --- construct return DeltaMessage --- + if name_delta is not None and argument_diff: + nd_func = name_delta.tool_calls[0].function + return DeltaMessage( + tool_calls=[ + DeltaToolCall( + index=self.current_tool_id, + id=self._current_tool_call_id, + type="function", + function=DeltaFunctionCall( + name=nd_func.name if nd_func else None, + arguments=argument_diff, + ), + ) + ] + ) + elif name_delta is not None: + return name_delta + elif argument_diff: + return self._make_args_delta(argument_diff) + else: + return None diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index cf2676a8f72..93dba4fd2f3 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -94,6 +94,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( funaudiochat="FunAudioChatConfig", granite4_vision="Granite4VisionConfig", hunyuan_vl="HunYuanVLConfig", + hy_v3="HYV3Config", isaac="IsaacConfig", kimi_k2="DeepseekV3Config", # Kimi K2 uses same architecture as DeepSeek V3 kimi_linear="KimiLinearConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index a45ea865db8..45eff21513b 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -36,6 +36,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl", "HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl", "HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl", + "HYV3Config": "vllm.transformers_utils.configs.hy_v3", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", "IsaacConfig": "vllm.transformers_utils.configs.isaac", # RWConfig is for the original tiiuae/falcon-40b(-instruct) and @@ -97,6 +98,7 @@ __all__ = [ "HunYuanVLConfig", "HunYuanVLTextConfig", "HunYuanVLVisionConfig", + "HYV3Config", "HyperCLOVAXConfig", "IsaacConfig", "RWConfig", diff --git a/vllm/transformers_utils/configs/hy_v3.py b/vllm/transformers_utils/configs/hy_v3.py new file mode 100644 index 00000000000..9425caf4e03 --- /dev/null +++ b/vllm/transformers_utils/configs/hy_v3.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +from transformers.configuration_utils import PretrainedConfig + + +class HYV3Config(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`HYV3Model`]. + It is used to instantiate a HYV3 model (HY V3 MoE language model) according to + the specified arguments. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to + control the model outputs. Read the documentation from [`PretrainedConfig`] + for more information. + + Args: + vocab_size (`int`, *optional*, defaults to 120832): + Vocabulary size of the model. + hidden_size (`int`, *optional*, defaults to 4096): + Dimension of the hidden representations. + intermediate_size (`int`, *optional*, defaults to 13312): + Dimension of the dense FFN intermediate representations. + num_hidden_layers (`int`, *optional*, defaults to 80): + Number of hidden layers in the Transformer decoder. + num_attention_heads (`int`, *optional*, defaults to 64): + Number of attention heads for each attention layer. + num_key_value_heads (`int`, *optional*, defaults to 8): + Number of key-value heads for grouped-query attention. + head_dim (`int`, *optional*, defaults to 128): + Dimension per attention head. + hidden_act (`str`, *optional*, defaults to `"silu"`): + Activation function used in FFN layers. + max_position_embeddings (`int`, *optional*, defaults to 131072): + Maximum sequence length supported by the model. + initializer_range (`float`, *optional*, defaults to 0.006): + Standard deviation of the truncated normal initializer for weight + initialization. + rms_norm_eps (`float`, *optional*, defaults to 1e-5): + Epsilon for RMS normalization layers. + use_cache (`bool`, *optional*, defaults to `True`): + Whether to use KV cache for decoding. + pad_token_id (`int`, *optional*): + Padding token id. + bos_token_id (`int`, *optional*): + Beginning-of-sequence token id. + eos_token_id (`int` or `List[int]`, *optional*): + End-of-sequence token id(s). + rope_parameters (`dict`, *optional*): + The parameters of the RoPE embeddings. + qk_norm (`bool`, *optional*, defaults to `True`): + Whether to apply RMSNorm to query and key states before attention. + tie_word_embeddings (`bool`, *optional*, defaults to `False`): + Whether to tie input and output embedding weights. + enable_attention_fp32_softmax (`bool`, *optional*, defaults to `False`): + Whether to upcast attention softmax to float32. Note: the eager attention + path always computes softmax in float32 regardless of this setting; this + flag is reserved for future use with custom attention backends. + enable_lm_head_fp32 (`bool`, *optional*, defaults to `True`): + Whether to upcast the LM head computation to float32. + num_experts (`int`, *optional*, defaults to 192): + Total number of MoE experts. + num_experts_per_tok (`int`, *optional*, defaults to 8): + Number of experts selected per token (top-k routing). + num_shared_experts (`int`, *optional*, defaults to 1): + Number of always-active shared experts combined into a single MLP. + expert_hidden_dim (`int`, *optional*, defaults to 1536): + Intermediate dimension of each individual MoE expert. + moe_router_enable_expert_bias (`bool`, *optional*, defaults to `True`): + Whether to use per-expert load-balancing bias in the router. + moe_router_use_sigmoid (`bool`, *optional*, defaults to `True`): + Whether to use sigmoid (instead of softmax) for router scoring. + route_norm (`bool`, *optional*, defaults to `True`): + Whether to normalize routing scores when using sigmoid routing. + router_scaling_factor (`float`, *optional*): + Optional multiplicative scaling factor applied to routing scores. + use_grouped_mm (`bool`, *optional*, defaults to `False`): + Whether to use grouped GEMM for expert computation (not yet implemented). + enable_moe_fp32_combine (`bool`, *optional*, defaults to `False`): + Whether to accumulate expert outputs in float32. + first_k_dense_replace (`int`, *optional*, defaults to 1): + Number of initial decoder layers that use a dense FFN instead of MoE. + output_router_logits (`bool`, *optional*, defaults to `False`): + Whether to output router logits from each MoE layer. Useful for computing + auxiliary load-balancing loss during training. Disabled by default to avoid + the memory overhead of storing per-layer router tensors during inference. + + Example: + ```python + >>> from transformers import HYV3Config, HYV3Model + + >>> config = HYV3Config() + >>> model = HYV3Model(config) + ``` + """ + + model_type = "hy_v3" + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__( + self, + vocab_size=120832, + hidden_size=4096, + intermediate_size=13312, + num_hidden_layers=80, + num_attention_heads=64, + num_key_value_heads=8, + head_dim=128, + hidden_act="silu", + max_position_embeddings=131072, + initializer_range=0.006, + rms_norm_eps=1e-5, + use_cache=True, + pad_token_id=None, + bos_token_id=None, + eos_token_id=None, + rope_parameters: dict[str, Any] | None = None, + qk_norm=True, + tie_word_embeddings=False, + enable_attention_fp32_softmax=False, + enable_lm_head_fp32=True, + # MoE specific + num_experts=192, + num_experts_per_tok=8, + num_shared_experts=1, + expert_hidden_dim=1536, + moe_router_enable_expert_bias=True, + moe_router_use_sigmoid=True, + route_norm=True, + router_scaling_factor=None, + use_grouped_mm=False, + enable_moe_fp32_combine=False, + # Dense/MoE layer control + first_k_dense_replace=1, + output_router_logits=False, + **kwargs, + ): + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.hidden_act = hidden_act + self.max_position_embeddings = max_position_embeddings + self.initializer_range = initializer_range + self.rms_norm_eps = rms_norm_eps + self.use_cache = use_cache + rope_theta = kwargs.pop("rope_theta", 11158840.0) + if rope_parameters is None: + rope_parameters = {"rope_type": "default", "rope_theta": rope_theta} + self.rope_parameters = rope_parameters + self.qk_norm = qk_norm + self.tie_word_embeddings = tie_word_embeddings + self.enable_lm_head_fp32 = enable_lm_head_fp32 + self.enable_attention_fp32_softmax = enable_attention_fp32_softmax + + # MoE specific + self.num_experts = num_experts + self.num_experts_per_tok = num_experts_per_tok + self.num_shared_experts = num_shared_experts + self.expert_hidden_dim = expert_hidden_dim + self.moe_router_enable_expert_bias = moe_router_enable_expert_bias + self.moe_router_use_sigmoid = moe_router_use_sigmoid + self.route_norm = route_norm + self.use_grouped_mm = use_grouped_mm + self.router_scaling_factor = router_scaling_factor + self.enable_moe_fp32_combine = enable_moe_fp32_combine + + # Dense/MoE layer control + self.first_k_dense_replace = first_k_dense_replace + self.output_router_logits = output_router_logits + + if eos_token_id is not None and isinstance(eos_token_id, int): + eos_token_id = [eos_token_id] + + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + tie_word_embeddings=tie_word_embeddings, + **kwargs, + ) From b7a26050200e20917871a4a6b09df0bc9ea3fdc7 Mon Sep 17 00:00:00 2001 From: Srreyansh Sethi <107075589+WorldExplored@users.noreply.github.com> Date: Thu, 23 Apr 2026 07:57:03 -0700 Subject: [PATCH 069/153] [Bugfix] Make Attention Backend Auto-Selection Batch-Invariance-Aware (#40193) Signed-off-by: Srreyansh Sethi Signed-off-by: Matthew Bonanni Co-authored-by: Matthew Bonanni Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- examples/rl/rlhf_async_new_apis.py | 9 +--- vllm/model_executor/layers/batch_invariant.py | 42 ++----------------- vllm/v1/attention/backend.py | 7 ++++ vllm/v1/attention/backends/flash_attn.py | 4 ++ .../attention/backends/mla/flashattn_mla.py | 4 ++ vllm/v1/attention/backends/mla/triton_mla.py | 4 ++ vllm/v1/attention/backends/triton_attn.py | 4 ++ vllm/v1/attention/selector.py | 11 ++++- vllm/v1/worker/gpu_worker.py | 3 +- 9 files changed, 38 insertions(+), 50 deletions(-) diff --git a/examples/rl/rlhf_async_new_apis.py b/examples/rl/rlhf_async_new_apis.py index 1d264d77985..f8af9537579 100644 --- a/examples/rl/rlhf_async_new_apis.py +++ b/examples/rl/rlhf_async_new_apis.py @@ -131,16 +131,9 @@ class TrainModel: from vllm.model_executor.layers.batch_invariant import ( init_batch_invariance, ) - from vllm.platforms import current_platform - from vllm.v1.attention.backends.registry import AttentionBackendEnum # need to init all env vars for batch invariance which affect nccl ops - attn_backend = ( - AttentionBackendEnum.TRITON_ATTN - if current_platform.is_rocm() - else AttentionBackendEnum.FLASH_ATTN - ) - init_batch_invariance(attn_backend) + init_batch_invariance() self.model = AutoModelForCausalLM.from_pretrained( model_name, dtype=torch.bfloat16 diff --git a/vllm/model_executor/layers/batch_invariant.py b/vllm/model_executor/layers/batch_invariant.py index 08756ee04de..fe051d6b397 100644 --- a/vllm/model_executor/layers/batch_invariant.py +++ b/vllm/model_executor/layers/batch_invariant.py @@ -14,7 +14,6 @@ from vllm.triton_utils import tl, triton from vllm.utils.mem_utils import get_max_shared_memory_bytes from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import is_torch_equal_or_newer -from vllm.v1.attention.backends.registry import AttentionBackendEnum logger = init_logger(__name__) @@ -991,40 +990,7 @@ def enable_batch_invariant_mode(): torch.backends.cuda.preferred_blas_library(backend="cublaslt") -def override_envs_for_invariance( - attention_backend: AttentionBackendEnum | None, -): - decode_invariant_backends = [ - AttentionBackendEnum.FLASH_ATTN, # best supported backend - AttentionBackendEnum.TRITON_ATTN, - ] - supported_backends = decode_invariant_backends + [ - # FlashInfer temporarily disabled due to invariant CTA sizes. - # See FlashInfer issue #2424 - # AttentionBackendEnum.FLASHINFER, - AttentionBackendEnum.FLASH_ATTN_MLA, - AttentionBackendEnum.TRITON_MLA, - # Not yet supported MLA backends - # AttentionBackendEnum.FLASHMLA, - # AttentionBackendEnum.FLEX_ATTENTION, # IMA issue - # AttentionBackendEnum.FLASHINFER_MLA, # PR #28967 - ] - if attention_backend not in supported_backends: - supported_names = [b.name for b in supported_backends] - backend_name = attention_backend.name if attention_backend else None - error = ( - "VLLM batch_invariant mode requires an attention backend in " - f"{supported_names}, but got '{backend_name}'. " - "Please use --attention-backend or attention_config to set " - "one of the supported backends before enabling batch_invariant." - ) - raise RuntimeError(error) - if attention_backend not in decode_invariant_backends: - warning = ( - "You are using a non-decode-invariant form of batch invariance. " - "This will not be invariant between prefill and decode." - ) - logger.warning_once(warning) +def override_envs_for_invariance(): os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0" os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" @@ -1045,12 +1011,10 @@ def override_envs_for_invariance( os.environ["VLLM_USE_AOT_COMPILE"] = "0" -def init_batch_invariance( - attention_backend: AttentionBackendEnum | None, -): +def init_batch_invariance(): # this will hit all the csrc overrides as well if envs.VLLM_BATCH_INVARIANT: - override_envs_for_invariance(attention_backend) + override_envs_for_invariance() enable_batch_invariant_mode() # Disable TF32 for batch invariance - it causes non-deterministic rounding diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index d2005181992..7d6bba4189d 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -236,6 +236,10 @@ class AttentionBackend(ABC): """ return False + @classmethod + def supports_batch_invariance(cls) -> bool: + return False + @classmethod def supports_attn_type(cls, attn_type: str) -> bool: """Check if backend supports a given attention type. @@ -278,6 +282,7 @@ class AttentionBackend(ABC): device_capability: "DeviceCapability", attn_type: str, use_non_causal: bool = False, + use_batch_invariant: bool = False, ) -> list[str]: invalid_reasons = [] if not cls.supports_head_size(head_size): @@ -312,6 +317,8 @@ class AttentionBackend(ABC): invalid_reasons.append(f"attention type {attn_type} not supported") if use_non_causal and not cls.supports_non_causal(): invalid_reasons.append("non-causal attention not supported") + if use_batch_invariant and not cls.supports_batch_invariance(): + invalid_reasons.append("batch invariance not supported") combination_reason = cls.supports_combination( head_size, dtype, diff --git a/vllm/v1/attention/backends/flash_attn.py b/vllm/v1/attention/backends/flash_attn.py index 19bcdfdc98e..1c9ff3f79e4 100755 --- a/vllm/v1/attention/backends/flash_attn.py +++ b/vllm/v1/attention/backends/flash_attn.py @@ -103,6 +103,10 @@ class FlashAttentionBackend(AttentionBackend): def get_name() -> str: return "FLASH_ATTN" + @classmethod + def supports_batch_invariance(cls) -> bool: + return True + @classmethod def supports_non_causal(cls) -> bool: return True diff --git a/vllm/v1/attention/backends/mla/flashattn_mla.py b/vllm/v1/attention/backends/mla/flashattn_mla.py index f58d9aeb302..bd947296e8b 100644 --- a/vllm/v1/attention/backends/mla/flashattn_mla.py +++ b/vllm/v1/attention/backends/mla/flashattn_mla.py @@ -56,6 +56,10 @@ class FlashAttnMLABackend(MLACommonBackend): def get_name() -> str: return "FLASH_ATTN_MLA" + @classmethod + def supports_batch_invariance(cls) -> bool: + return True + @staticmethod def get_builder_cls() -> type["FlashAttnMLAMetadataBuilder"]: return FlashAttnMLAMetadataBuilder diff --git a/vllm/v1/attention/backends/mla/triton_mla.py b/vllm/v1/attention/backends/mla/triton_mla.py index 0f8eb1c49a5..7aa8a646f41 100644 --- a/vllm/v1/attention/backends/mla/triton_mla.py +++ b/vllm/v1/attention/backends/mla/triton_mla.py @@ -55,6 +55,10 @@ class TritonMLABackend(MLACommonBackend): def get_name() -> str: return "TRITON_MLA" + @classmethod + def supports_batch_invariance(cls) -> bool: + return True + @staticmethod def get_impl_cls() -> type["TritonMLAImpl"]: return TritonMLAImpl diff --git a/vllm/v1/attention/backends/triton_attn.py b/vllm/v1/attention/backends/triton_attn.py index 4739d48e870..f254d95a414 100644 --- a/vllm/v1/attention/backends/triton_attn.py +++ b/vllm/v1/attention/backends/triton_attn.py @@ -296,6 +296,10 @@ class TritonAttentionBackend(AttentionBackend): def get_name() -> str: return "TRITON_ATTN" + @classmethod + def supports_batch_invariance(cls) -> bool: + return True + @staticmethod def get_impl_cls() -> type["TritonAttentionImpl"]: return TritonAttentionImpl diff --git a/vllm/v1/attention/selector.py b/vllm/v1/attention/selector.py index 066c5fcc9c2..f05d9664ef7 100644 --- a/vllm/v1/attention/selector.py +++ b/vllm/v1/attention/selector.py @@ -6,6 +6,7 @@ from typing import NamedTuple, cast, get_args import torch +import vllm.envs as envs from vllm.config.cache import CacheDType from vllm.logger import init_logger from vllm.utils.import_utils import resolve_obj_by_qualname @@ -30,6 +31,7 @@ class AttentionSelectorConfig(NamedTuple): use_per_head_quant_scales: bool = False attn_type: str = AttentionType.DECODER use_non_causal: bool = False + use_batch_invariant: bool = False def __repr__(self): return ( @@ -43,7 +45,8 @@ class AttentionSelectorConfig(NamedTuple): f"use_mm_prefix={self.use_mm_prefix}, " f"use_per_head_quant_scales={self.use_per_head_quant_scales}, " f"attn_type={self.attn_type}, " - f"use_non_causal={self.use_non_causal})" + f"use_non_causal={self.use_non_causal}, " + f"use_batch_invariant={self.use_batch_invariant})" ) @@ -95,6 +98,7 @@ def get_attn_backend( use_per_head_quant_scales=use_per_head_quant_scales, attn_type=attn_type or AttentionType.DECODER, use_non_causal=use_non_causal, + use_batch_invariant=envs.VLLM_BATCH_INVARIANT, ) return _cached_get_attn_backend( @@ -162,4 +166,9 @@ def _cached_get_mamba_attn_backend( ) from e mamba_attn_backend = selected_backend.get_class() + if envs.VLLM_BATCH_INVARIANT and not mamba_attn_backend.supports_batch_invariance(): + raise RuntimeError( + "VLLM batch_invariant mode is not supported for " + f"{mamba_attn_backend.get_name()}." + ) return mamba_attn_backend diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index afbee95c4d7..30d05308539 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -1027,11 +1027,10 @@ def init_worker_distributed_environment( backend: str = "nccl", ) -> None: """Initialize the distributed environment.""" - attention_config = vllm_config.attention_config parallel_config = vllm_config.parallel_config from vllm.model_executor.layers.batch_invariant import init_batch_invariance - init_batch_invariance(attention_config.backend) + init_batch_invariance() override_envs_for_eplb(parallel_config) set_custom_all_reduce(not parallel_config.disable_custom_all_reduce) From 53ecc807c0e323aea2f2a48dfdae71be838c4f5c Mon Sep 17 00:00:00 2001 From: Kunshang Ji Date: Thu, 23 Apr 2026 23:07:35 +0800 Subject: [PATCH 070/153] [XPU] Upgrade torch 2.11 for xpu (#37947) Signed-off-by: Kunshang Ji --- docker/Dockerfile.xpu | 8 ++-- .../installation/gpu.xpu.inc.md | 2 +- requirements/test/xpu.txt | 46 +++++++++---------- requirements/xpu.txt | 2 +- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/docker/Dockerfile.xpu b/docker/Dockerfile.xpu index 555c1f14420..ab3f6f40d87 100644 --- a/docker/Dockerfile.xpu +++ b/docker/Dockerfile.xpu @@ -50,9 +50,9 @@ RUN curl -LsSf https://astral.sh/uv/install.sh | sh RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV} ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2. -ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh" -RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \ +# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3. +ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh" +RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \ bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \ rm "${ONECCL_INSTALLER}" && \ echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \ @@ -164,7 +164,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # FIX triton RUN --mount=type=cache,target=/root/.cache/uv \ uv pip uninstall triton triton-xpu && \ - uv pip install triton-xpu==3.6.0 + uv pip install triton-xpu==3.7.0 # remove torch bundled oneccl to avoid conflicts RUN --mount=type=cache,target=/root/.cache/uv \ diff --git a/docs/getting_started/installation/gpu.xpu.inc.md b/docs/getting_started/installation/gpu.xpu.inc.md index 9e71860d62f..8c282582281 100644 --- a/docs/getting_started/installation/gpu.xpu.inc.md +++ b/docs/getting_started/installation/gpu.xpu.inc.md @@ -46,7 +46,7 @@ pip install -v -r requirements/xpu.txt !!! note - `triton` (without suffix) is for NVIDIA GPUs only. On XPU, using it instead of `triton-xpu` can cause correctness or runtime issues. - - For torch 2.10 (the version used in `requirements/xpu.txt`), the matching package is `triton-xpu==3.6.0`. If you use a different version of torch, check the corresponding `triton-xpu` version in [docker/Dockerfile.xpu](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.xpu). + - For torch 2.11 (the version used in `requirements/xpu.txt`), the matching package is `triton-xpu==3.7.0`. If you use a different version of torch, check the corresponding `triton-xpu` version in [docker/Dockerfile.xpu](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.xpu). - Finally, build and install vLLM XPU backend: diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index 4ddc0aa1c92..547269d4a2c 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -93,7 +93,7 @@ docker==7.1.0 # via gpt-oss docopt==0.6.2 # via num2words -dpcpp-cpp-rt==2025.3.1 +dpcpp-cpp-rt==2025.3.2 # via # onemkl-sycl-blas # onemkl-sycl-dft @@ -172,27 +172,27 @@ idna==3.11 # yarl imageio==2.37.3 # via scikit-image -impi-rt==2021.17.0 +impi-rt==2021.17.2 # via # oneccl # torch iniconfig==2.3.0 # via pytest -intel-cmplr-lib-rt==2025.3.1 +intel-cmplr-lib-rt==2025.3.2 # via # intel-sycl-rt # torch -intel-cmplr-lib-ur==2025.3.1 +intel-cmplr-lib-ur==2025.3.2 # via # intel-openmp # intel-sycl-rt # torch -intel-cmplr-lic-rt==2025.3.1 +intel-cmplr-lic-rt==2025.3.2 # via # intel-opencl-rt # intel-sycl-rt # torch -intel-opencl-rt==2025.3.1 +intel-opencl-rt==2025.3.2 # via # dpcpp-cpp-rt # onemkl-sycl-blas @@ -201,14 +201,14 @@ intel-opencl-rt==2025.3.1 # onemkl-sycl-rng # onemkl-sycl-sparse # torch -intel-openmp==2025.3.1 +intel-openmp==2025.3.2 # via # dpcpp-cpp-rt # mkl # torch -intel-pti==0.15.0 +intel-pti==0.16.0 # via torch -intel-sycl-rt==2025.3.1 +intel-sycl-rt==2025.3.2 # via # dpcpp-cpp-rt # oneccl @@ -270,7 +270,7 @@ mistral-common==1.11.0 # via # -c requirements/common.txt # -r requirements/test/xpu.in -mkl==2025.3.0 +mkl==2025.3.1 # via # onemkl-sycl-blas # onemkl-sycl-dft @@ -335,28 +335,28 @@ numpy==2.2.6 # tifffile # torchvision # transformers -oneccl==2021.17.1 +oneccl==2021.17.2 # via # oneccl-devel # torch -oneccl-devel==2021.17.1 +oneccl-devel==2021.17.2 # via torch -onemkl-license==2025.3.0 +onemkl-license==2025.3.1 # via # mkl # torch -onemkl-sycl-blas==2025.3.0 +onemkl-sycl-blas==2025.3.1 # via # onemkl-sycl-lapack # onemkl-sycl-sparse # torch -onemkl-sycl-dft==2025.3.0 +onemkl-sycl-dft==2025.3.1 # via torch -onemkl-sycl-lapack==2025.3.0 +onemkl-sycl-lapack==2025.3.1 # via torch -onemkl-sycl-rng==2025.3.0 +onemkl-sycl-rng==2025.3.1 # via torch -onemkl-sycl-sparse==2025.3.0 +onemkl-sycl-sparse==2025.3.1 # via torch openai-harmony==0.0.8 # via @@ -608,7 +608,7 @@ tabledata==1.3.4 # via pytablewriter tabulate==0.10.0 # via sacrebleu -tbb==2022.3.0 +tbb==2022.3.1 # via # intel-opencl-rt # mkl @@ -645,7 +645,7 @@ tokenizers==0.22.2 # via # -c requirements/common.txt # transformers -torch==2.10.0+xpu +torch==2.11.0+xpu # via # -c requirements/xpu.txt # accelerate @@ -653,7 +653,7 @@ torch==2.10.0+xpu # sentence-transformers # timm # torchvision -torchvision==0.25.0+xpu +torchvision==0.26.0+xpu # via timm tqdm==4.67.3 # via @@ -671,7 +671,7 @@ transformers==5.5.3 # via # -c requirements/common.txt # sentence-transformers -triton-xpu==3.6.0 +triton-xpu==3.7.0 # via torch typepy==1.3.4 # via @@ -710,7 +710,7 @@ typing-inspection==0.4.2 # via # fastapi # pydantic -umf==1.0.2 +umf==1.0.3 # via # intel-cmplr-lib-ur # torch diff --git a/requirements/xpu.txt b/requirements/xpu.txt index 26ba38f3efa..3be85dcb5f4 100644 --- a/requirements/xpu.txt +++ b/requirements/xpu.txt @@ -11,7 +11,7 @@ jinja2>=3.1.6 datasets # for benchmark scripts numba == 0.61.2 # Required for N-gram speculative decoding --extra-index-url=https://download.pytorch.org/whl/xpu -torch==2.10.0+xpu +torch==2.11.0+xpu torchaudio torchvision From 0098db9ec1b19138843fee3147b61bbdbec0cd05 Mon Sep 17 00:00:00 2001 From: pschlan-amd Date: Thu, 23 Apr 2026 17:08:48 +0200 Subject: [PATCH 071/153] [ROCm] Implement GPU-to-NUMA-node detection (#40015) Signed-off-by: Patrick Schlangen Co-authored-by: TJian --- docs/configuration/optimization.md | 6 +++--- vllm/platforms/rocm.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/docs/configuration/optimization.md b/docs/configuration/optimization.md index 26eda1246b1..472e1cf57ff 100644 --- a/docs/configuration/optimization.md +++ b/docs/configuration/optimization.md @@ -155,9 +155,9 @@ switch to `--physcpubind= --membind=`. These `--numa-bind*` options only apply to GPU execution processes. They do not configure the CPU backend's separate thread-affinity controls. Automatic -GPU-to-NUMA detection is currently implemented for CUDA/NVML-based platforms; -other GPU backends must provide explicit binding lists if they use these -options. +GPU-to-NUMA detection is currently implemented for CUDA/NVML-based as well as +ROCM-based platforms; other GPU backends must provide explicit binding lists if +they use these options. `--numa-bind-nodes` takes one non-negative NUMA node index per visible GPU, in the same order as the GPU indices. diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 0801c852423..52773338620 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -33,6 +33,7 @@ try: amdsmi_init, amdsmi_shut_down, amdsmi_topo_get_link_type, + amdsmi_topo_get_numa_node_number, ) except ImportError as e: logger.warning("Failed to import from amdsmi with %r", e) @@ -955,3 +956,30 @@ class RocmPlatform(Platform): rms_norm = default return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm) + + @classmethod + @with_amdsmi_context + def get_all_device_numa_nodes(cls) -> list[int] | None: + """Get NUMA nodes for all visible GPU devices.""" + try: + handles = amdsmi_get_processor_handles() + numa_nodes = [] + for device_id in range(cls.device_count()): + physical_device_id = cls.device_id_to_physical_device_id(device_id) + try: + numa_node = amdsmi_topo_get_numa_node_number( + handles[physical_device_id] + ) + except AmdSmiException as e: + logger.warning( + "Could not detect NUMA node for GPU %d, " + "disabling automatic NUMA binding: %s", + device_id, + e, + ) + return None + numa_nodes.append(numa_node) + return numa_nodes + except Exception as e: + logger.warning("Failed to get NUMA nodes for GPUs: %s", e) + return None From 8824f50f1f1475fb07cc0c20da260e2e5a355cbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Thu, 23 Apr 2026 17:20:12 +0200 Subject: [PATCH 072/153] [CI] Split disaggregated tests into own test-area (#40623) Signed-off-by: NickLucche --- .buildkite/test_areas/disaggregated.yaml | 98 +++++++++++++++++++ .buildkite/test_areas/distributed.yaml | 85 ---------------- .../config_sweep_accuracy_test.sh | 3 +- 3 files changed, 100 insertions(+), 86 deletions(-) create mode 100644 .buildkite/test_areas/disaggregated.yaml diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml new file mode 100644 index 00000000000..a10fda41ef0 --- /dev/null +++ b/.buildkite/test_areas/disaggregated.yaml @@ -0,0 +1,98 @@ +group: Disaggregated +depends_on: + - image-build +steps: +- label: Distributed NixlConnector PD accuracy (4 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +- label: Distributed FlashInfer NixlConnector PD accuracy (4 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - FLASHINFER=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) + timeout_in_minutes: 20 + working_dir: "/vllm-workspace/tests" + num_devices: 4 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh + +- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 2 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh + +- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) + timeout_in_minutes: 30 + device: a100 + working_dir: "/vllm-workspace/tests" + num_devices: 2 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/v1/worker/kv_connector_model_runner_mixin.py + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh + +- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) + timeout_in_minutes: 30 + working_dir: "/vllm-workspace/tests" + num_devices: 2 + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py + - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ + - tests/v1/kv_connector/nixl_integration/ + commands: + - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt + - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh \ No newline at end of file diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index e13618eb65d..093f3ab4fe1 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -226,91 +226,6 @@ steps: commands: - ./.buildkite/scripts/run-multi-node-test.sh /vllm-workspace/tests 2 2 $IMAGE_TAG "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=0 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_multi_node_assignment.py && VLLM_MULTI_NODE=1 pytest -v -s distributed/test_pipeline_parallel.py" "VLLM_TEST_SAME_HOST=0 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_same_node.py | grep 'Same node test passed' && NUM_NODES=2 torchrun --nnodes 2 --nproc-per-node=2 --rdzv_backend=c10d --rdzv_endpoint=192.168.10.10 distributed/test_node_count.py | grep 'Node count test passed' && python3 ../examples/offline_inference/data_parallel.py -dp=2 -tp=1 --dp-num-nodes=2 --dp-node-rank=1 --dp-master-addr=192.168.10.10 --dp-master-port=12345 --enforce-eager --trust-remote-code" -- label: Distributed NixlConnector PD accuracy (4 GPUs) - timeout_in_minutes: 30 - working_dir: "/vllm-workspace/tests" - num_devices: 4 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - -- label: DP EP Distributed NixlConnector PD accuracy tests (4 GPUs) - timeout_in_minutes: 30 - working_dir: "/vllm-workspace/tests" - num_devices: 4 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - DP_EP=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - -- label: CrossLayer KV layout Distributed NixlConnector PD accuracy tests (4 GPUs) - timeout_in_minutes: 30 - working_dir: "/vllm-workspace/tests" - num_devices: 4 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - -- label: Hyrbid SSM NixlConnector PD accuracy tests (4 GPUs) - timeout_in_minutes: 20 - working_dir: "/vllm-workspace/tests" - num_devices: 4 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - HYBRID_SSM=1 bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh - -- label: MultiConnector (Nixl+Offloading) PD accuracy (2 GPUs) - timeout_in_minutes: 30 - working_dir: "/vllm-workspace/tests" - num_devices: 2 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py - - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py - - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - bash v1/kv_connector/nixl_integration/run_multi_connector_accuracy_test.sh - -- label: NixlConnector PD + Spec Decode acceptance (2 GPUs) - timeout_in_minutes: 30 - device: a100 - working_dir: "/vllm-workspace/tests" - num_devices: 2 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - vllm/v1/worker/kv_connector_model_runner_mixin.py - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - bash v1/kv_connector/nixl_integration/spec_decode_acceptance_test.sh - -- label: MultiConnector (Nixl+Offloading) PD edge cases (2 GPUs) - timeout_in_minutes: 30 - working_dir: "/vllm-workspace/tests" - num_devices: 2 - source_file_dependencies: - - vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py - - vllm/distributed/kv_transfer/kv_connector/v1/multi_connector.py - - vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py - - vllm/distributed/kv_transfer/kv_connector/v1/offloading/ - - tests/v1/kv_connector/nixl_integration/ - commands: - - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - - bash v1/kv_connector/nixl_integration/run_multi_connector_edge_case_test.sh - - label: Pipeline + Context Parallelism (4 GPUs) timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" diff --git a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh index b0794bfa38a..9bc0f3135ed 100755 --- a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh @@ -12,7 +12,6 @@ tp_configs=( "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA case "GPU_MEMORY_UTILIZATION=0.8 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" "GPU_MEMORY_UTILIZATION=0.8 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" - "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" # SW model ) dp_ep_configs=( "DP_EP=1 GPU_MEMORY_UTILIZATION=0.8 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 MODEL_NAMES=deepseek-ai/deepseek-vl2-tiny" # MLA+P-TP1, D-DPEP=2 (TP=1) @@ -24,6 +23,8 @@ hybrid_ssm_configs=( "VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code,--no-async-scheduling" ) sw_attn_configs=( + # NOTE: gemma3 does not work with FlashInfer + "GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" # SW model "ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" "ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=google/gemma-3-4b-it PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=1 VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192" ) From 1c2c1eb8b9fdd2e67c45afb6123ccc07c0177555 Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Thu, 23 Apr 2026 11:22:34 -0400 Subject: [PATCH 073/153] [MoE Refactor] Rename FusedMoE.make_expert_params_mapping to fused_moe_make_expert_params_mapping (#40671) Signed-off-by: Bill Nell --- .../layers/fused_moe/__init__.py | 2 ++ vllm/model_executor/layers/fused_moe/layer.py | 19 +++++++++++++++++++ vllm/model_executor/models/AXK1.py | 9 ++++++--- vllm/model_executor/models/afmoe.py | 7 +++++-- vllm/model_executor/models/arctic.py | 5 ++++- vllm/model_executor/models/aria.py | 4 +++- vllm/model_executor/models/bailing_moe.py | 7 +++++-- .../models/bailing_moe_linear.py | 7 +++++-- vllm/model_executor/models/dbrx.py | 4 +++- vllm/model_executor/models/deepseek_eagle.py | 6 ++++-- vllm/model_executor/models/deepseek_mtp.py | 6 ++++-- vllm/model_executor/models/deepseek_v2.py | 5 +++-- vllm/model_executor/models/dots1.py | 7 +++++-- vllm/model_executor/models/ernie45_moe.py | 7 +++++-- vllm/model_executor/models/ernie45_vl_moe.py | 7 +++++-- vllm/model_executor/models/exaone_moe.py | 7 +++++-- vllm/model_executor/models/flex_olmo.py | 4 +++- vllm/model_executor/models/gemma4.py | 5 ++++- vllm/model_executor/models/glm4_moe.py | 7 +++++-- vllm/model_executor/models/glm4_moe_lite.py | 10 ++++++---- .../models/glm4_moe_lite_mtp.py | 7 +++++-- vllm/model_executor/models/glm4_moe_mtp.py | 7 +++++-- vllm/model_executor/models/gpt_oss.py | 7 +++++-- vllm/model_executor/models/granitemoe.py | 7 +++++-- vllm/model_executor/models/grok1.py | 7 +++++-- vllm/model_executor/models/hunyuan_v1.py | 7 +++++-- vllm/model_executor/models/interns1_pro.py | 4 +++- vllm/model_executor/models/jamba.py | 7 +++++-- vllm/model_executor/models/kimi_linear.py | 7 +++++-- vllm/model_executor/models/lfm2_moe.py | 7 +++++-- vllm/model_executor/models/llama4.py | 11 +++++++---- vllm/model_executor/models/longcat_flash.py | 7 +++++-- vllm/model_executor/models/mimo_v2_flash.py | 7 +++++-- vllm/model_executor/models/minimax_m2.py | 7 +++++-- vllm/model_executor/models/minimax_text_01.py | 4 +++- vllm/model_executor/models/mixtral.py | 7 +++++-- vllm/model_executor/models/mllama4.py | 6 ++++-- vllm/model_executor/models/nemotron_h.py | 3 ++- vllm/model_executor/models/nemotron_h_mtp.py | 6 ++++-- vllm/model_executor/models/olmoe.py | 7 +++++-- vllm/model_executor/models/openpangu.py | 7 +++++-- vllm/model_executor/models/openpangu_mtp.py | 6 ++++-- vllm/model_executor/models/param2moe.py | 7 +++++-- vllm/model_executor/models/phimoe.py | 7 +++++-- vllm/model_executor/models/qwen2_moe.py | 7 +++++-- vllm/model_executor/models/qwen3_5_mtp.py | 6 ++++-- vllm/model_executor/models/qwen3_moe.py | 7 +++++-- vllm/model_executor/models/qwen3_next.py | 7 +++++-- vllm/model_executor/models/qwen3_next_mtp.py | 6 ++++-- vllm/model_executor/models/sarvam.py | 7 +++++-- vllm/model_executor/models/step3_text.py | 4 +++- vllm/model_executor/models/step3p5.py | 7 +++++-- .../model_executor/models/transformers/moe.py | 7 +++++-- 53 files changed, 254 insertions(+), 98 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 1b2ce61f7c8..a154ede547b 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import ( from vllm.model_executor.layers.fused_moe.layer import ( FusedMoE, FusedMoeWeightScaleSupported, + fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.fused_moe.modular_kernel import ( FusedMoEActivationFormat, @@ -65,6 +66,7 @@ __all__ = [ "RoutingMethodType", "activation_without_mul", "apply_moe_activation", + "fused_moe_make_expert_params_mapping", "override_config", "get_config", ] diff --git a/vllm/model_executor/layers/fused_moe/layer.py b/vllm/model_executor/layers/fused_moe/layer.py index 7adac0374cf..012c7328503 100644 --- a/vllm/model_executor/layers/fused_moe/layer.py +++ b/vllm/model_executor/layers/fused_moe/layer.py @@ -1618,6 +1618,25 @@ class FusedMoE(PluggableLayer): return s +# This is a temporary forwarding method which will be removed/modified layer. +def fused_moe_make_expert_params_mapping( + model: torch.nn.Module, + ckpt_gate_proj_name: str, + ckpt_down_proj_name: str, + ckpt_up_proj_name: str, + num_experts: int, + num_redundant_experts: int = 0, +) -> list[tuple[str, str, int, str]]: + return FusedMoE.make_expert_params_mapping( + model, + ckpt_gate_proj_name, + ckpt_down_proj_name, + ckpt_up_proj_name, + num_experts, + num_redundant_experts, + ) + + # Mark the FusedMoE weight_loader as supporting MoE-specific parameters # to avoid expensive runtime reflection in model loading code FusedMoE.weight_loader.supports_moe_loading = True # type: ignore[attr-defined] diff --git a/vllm/model_executor/models/AXK1.py b/vllm/model_executor/models/AXK1.py index c33d5b97372..c8f56ca97fc 100644 --- a/vllm/model_executor/models/AXK1.py +++ b/vllm/model_executor/models/AXK1.py @@ -42,7 +42,10 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -916,7 +919,7 @@ class AXK1ForCausalLM( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -950,7 +953,7 @@ class AXK1ForCausalLM( # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/afmoe.py b/vllm/model_executor/models/afmoe.py index e34a418c981..2216e4948bd 100644 --- a/vllm/model_executor/models/afmoe.py +++ b/vllm/model_executor/models/afmoe.py @@ -18,7 +18,10 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -479,7 +482,7 @@ class AfmoeModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/arctic.py b/vllm/model_executor/models/arctic.py index 0c9267994b0..6ab55a4b1bf 100644 --- a/vllm/model_executor/models/arctic.py +++ b/vllm/model_executor/models/arctic.py @@ -18,7 +18,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import fused_experts, fused_topk +from vllm.model_executor.layers.fused_moe import ( + fused_experts, + fused_topk, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, diff --git a/vllm/model_executor/models/aria.py b/vllm/model_executor/models/aria.py index 9696dec6d87..48c8d9a441e 100644 --- a/vllm/model_executor/models/aria.py +++ b/vllm/model_executor/models/aria.py @@ -14,7 +14,9 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.distributed import get_tensor_model_parallel_rank from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.activation import get_act_fn -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, +) from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig diff --git a/vllm/model_executor/models/bailing_moe.py b/vllm/model_executor/models/bailing_moe.py index ef4f66614a3..56e119207da 100644 --- a/vllm/model_executor/models/bailing_moe.py +++ b/vllm/model_executor/models/bailing_moe.py @@ -41,7 +41,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -461,7 +464,7 @@ class BailingMoeModel(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/bailing_moe_linear.py b/vllm/model_executor/models/bailing_moe_linear.py index df36659b10c..e26adc17430 100644 --- a/vllm/model_executor/models/bailing_moe_linear.py +++ b/vllm/model_executor/models/bailing_moe_linear.py @@ -21,7 +21,10 @@ from vllm.model_executor.layers.fla.ops.layernorm_guard import ( RMSNormGated, layernorm_fn, ) -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -990,7 +993,7 @@ class BailingMoeV25Model(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: """Get expert parameter mapping for MoE layers.""" - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/dbrx.py b/vllm/model_executor/models/dbrx.py index a72f4e48716..6c798bf2f36 100644 --- a/vllm/model_executor/models/dbrx.py +++ b/vllm/model_executor/models/dbrx.py @@ -15,7 +15,9 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, +) from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, diff --git a/vllm/model_executor/models/deepseek_eagle.py b/vllm/model_executor/models/deepseek_eagle.py index 5c439cdf486..f975b32adc1 100644 --- a/vllm/model_executor/models/deepseek_eagle.py +++ b/vllm/model_executor/models/deepseek_eagle.py @@ -8,7 +8,9 @@ import torch.nn as nn from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -105,7 +107,7 @@ class DeepseekV2Model(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/deepseek_mtp.py b/vllm/model_executor/models/deepseek_mtp.py index 898e4333409..37f94c687a2 100644 --- a/vllm/model_executor/models/deepseek_mtp.py +++ b/vllm/model_executor/models/deepseek_mtp.py @@ -11,7 +11,9 @@ from vllm._aiter_ops import rocm_aiter_ops from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -252,7 +254,7 @@ class DeepSeekMTP(nn.Module, DeepseekV2MixtureOfExperts): ] stacked_params_mapping.extend(indexer_fused_mapping) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 53bcf87c6cc..3d0b1c42458 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -51,6 +51,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoE, GateLinear, RoutingMethodType, + fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import LayerNorm, RMSNorm from vllm.model_executor.layers.linear import ( @@ -1432,7 +1433,7 @@ class DeepseekV2ForCausalLM( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -1474,7 +1475,7 @@ class DeepseekV2ForCausalLM( # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/dots1.py b/vllm/model_executor/models/dots1.py index 181bd598e8e..f58fc4da92b 100644 --- a/vllm/model_executor/models/dots1.py +++ b/vllm/model_executor/models/dots1.py @@ -40,7 +40,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -413,7 +416,7 @@ class Dots1Model(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/ernie45_moe.py b/vllm/model_executor/models/ernie45_moe.py index 58dd61e9d92..a2b0eccde65 100644 --- a/vllm/model_executor/models/ernie45_moe.py +++ b/vllm/model_executor/models/ernie45_moe.py @@ -42,7 +42,10 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -485,7 +488,7 @@ class Ernie4_5_MoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/ernie45_vl_moe.py b/vllm/model_executor/models/ernie45_vl_moe.py index b4e7af9304b..38ed756ba41 100644 --- a/vllm/model_executor/models/ernie45_vl_moe.py +++ b/vllm/model_executor/models/ernie45_vl_moe.py @@ -36,7 +36,10 @@ from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -649,7 +652,7 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/exaone_moe.py b/vllm/model_executor/models/exaone_moe.py index dd91a189628..80b7e0957e8 100644 --- a/vllm/model_executor/models/exaone_moe.py +++ b/vllm/model_executor/models/exaone_moe.py @@ -30,7 +30,10 @@ from vllm.distributed import ( get_pp_group, get_tensor_model_parallel_world_size, ) -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -326,7 +329,7 @@ class ExaoneMoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/flex_olmo.py b/vllm/model_executor/models/flex_olmo.py index 1b2047eb231..2ff9d860567 100644 --- a/vllm/model_executor/models/flex_olmo.py +++ b/vllm/model_executor/models/flex_olmo.py @@ -20,7 +20,9 @@ from torch import nn from vllm.config import VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.models.olmoe import OlmoeAttention, OlmoeForCausalLM diff --git a/vllm/model_executor/models/gemma4.py b/vllm/model_executor/models/gemma4.py index 42762e36f81..bb91fd601e7 100644 --- a/vllm/model_executor/models/gemma4.py +++ b/vllm/model_executor/models/gemma4.py @@ -37,7 +37,10 @@ from vllm.forward_context import get_forward_context from vllm.logger import init_logger from vllm.model_executor.layers.activation import GeluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE, GateLinear +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + GateLinear, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, diff --git a/vllm/model_executor/models/glm4_moe.py b/vllm/model_executor/models/glm4_moe.py index 680e7460992..aeec6fefa23 100644 --- a/vllm/model_executor/models/glm4_moe.py +++ b/vllm/model_executor/models/glm4_moe.py @@ -42,7 +42,10 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -466,7 +469,7 @@ class Glm4MoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/glm4_moe_lite.py b/vllm/model_executor/models/glm4_moe_lite.py index 5dc33ec18bf..77aaa179aa5 100644 --- a/vllm/model_executor/models/glm4_moe_lite.py +++ b/vllm/model_executor/models/glm4_moe_lite.py @@ -41,7 +41,9 @@ from vllm.distributed import ( get_pp_group, ) from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -308,7 +310,7 @@ class Glm4MoeLiteModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -334,7 +336,7 @@ class Glm4MoeLiteModel(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -616,7 +618,7 @@ class Glm4MoeLiteForCausalLM( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/glm4_moe_lite_mtp.py b/vllm/model_executor/models/glm4_moe_lite_mtp.py index e00476abac6..596cb48face 100644 --- a/vllm/model_executor/models/glm4_moe_lite_mtp.py +++ b/vllm/model_executor/models/glm4_moe_lite_mtp.py @@ -32,7 +32,10 @@ from transformers import PretrainedConfig from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -260,7 +263,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts): ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), ] - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/glm4_moe_mtp.py b/vllm/model_executor/models/glm4_moe_mtp.py index cde94673e53..791ecabebeb 100644 --- a/vllm/model_executor/models/glm4_moe_mtp.py +++ b/vllm/model_executor/models/glm4_moe_mtp.py @@ -31,7 +31,10 @@ import torch.nn as nn from transformers import PretrainedConfig from vllm.config import CacheConfig, ParallelConfig, VllmConfig -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig @@ -247,7 +250,7 @@ class Glm4MoeMTP(nn.Module, Glm4MixtureOfExperts): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/gpt_oss.py b/vllm/model_executor/models/gpt_oss.py index b6edc344302..d12db96c5d4 100644 --- a/vllm/model_executor/models/gpt_oss.py +++ b/vllm/model_executor/models/gpt_oss.py @@ -20,7 +20,10 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -331,7 +334,7 @@ class GptOssModel(nn.Module, EagleModelMixin): # Params for weights, weight scales, activation scales # (param_name, weight_name, expert_id, shard_id) # NOTE: this is only used for quark. - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/granitemoe.py b/vllm/model_executor/models/granitemoe.py index f57a8c942bb..e3585a6dd74 100644 --- a/vllm/model_executor/models/granitemoe.py +++ b/vllm/model_executor/models/granitemoe.py @@ -39,7 +39,10 @@ from vllm.distributed import ( tensor_model_parallel_all_gather, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -351,7 +354,7 @@ class GraniteMoeModel(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/grok1.py b/vllm/model_executor/models/grok1.py index c9aa3d2068f..f06122a7fd1 100644 --- a/vllm/model_executor/models/grok1.py +++ b/vllm/model_executor/models/grok1.py @@ -38,7 +38,10 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.activation import GeluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -519,7 +522,7 @@ class Grok1Model(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Map expert parameter names to standard names num_experts = _get_num_experts(self.config) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name=self.ckpt_gate_proj_name, ckpt_down_proj_name=self.ckpt_down_proj_name, diff --git a/vllm/model_executor/models/hunyuan_v1.py b/vllm/model_executor/models/hunyuan_v1.py index 9d3ebe4ed9c..fca801b7482 100644 --- a/vllm/model_executor/models/hunyuan_v1.py +++ b/vllm/model_executor/models/hunyuan_v1.py @@ -42,7 +42,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -712,7 +715,7 @@ class HunYuanModel(nn.Module, EagleModelMixin): if _is_moe(self.config): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/interns1_pro.py b/vllm/model_executor/models/interns1_pro.py index 9612ea57b2c..36f669179c5 100644 --- a/vllm/model_executor/models/interns1_pro.py +++ b/vllm/model_executor/models/interns1_pro.py @@ -41,7 +41,9 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, diff --git a/vllm/model_executor/models/jamba.py b/vllm/model_executor/models/jamba.py index b4b3b6873db..84e96def6c1 100644 --- a/vllm/model_executor/models/jamba.py +++ b/vllm/model_executor/models/jamba.py @@ -14,7 +14,10 @@ from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed.parallel_state import get_pp_group from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -378,7 +381,7 @@ class JambaModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/kimi_linear.py b/vllm/model_executor/models/kimi_linear.py index 21940fb2e1f..29d827d196f 100644 --- a/vllm/model_executor/models/kimi_linear.py +++ b/vllm/model_executor/models/kimi_linear.py @@ -14,7 +14,10 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.kda import KimiDeltaAttention from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -476,7 +479,7 @@ class KimiLinearModel(nn.Module): if self.config.is_moe: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 4b49430c1fa..55b00d2b9ea 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -15,7 +15,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -482,7 +485,7 @@ class Lfm2MoeModel(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index c9495a743b7..bfcb72a6a74 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -36,7 +36,10 @@ from vllm.model_executor.layers.attention import ( Attention, ChunkedLocalAttention, ) -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -414,7 +417,7 @@ class Llama4Model(LlamaModel): params_dict: The dictionary of module parameters. loaded_params: The set of already loaded parameters. expert_params_mapping: The mapping of expert parameters. Must be - generated by FusedMoE.make_expert_params_mapping(). + generated by fused_moe_make_expert_params_mapping(). fused: Whether the expert weights are fused into a single weight tensor or are separate weight tensors for each expert. When fused is True, loaded_weight should have shape of: @@ -554,7 +557,7 @@ class Llama4Model(LlamaModel): fused_experts_params = False # Expert parameter mapping for the case where the expert weights are # not fused into a single weight tensor. - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", @@ -564,7 +567,7 @@ class Llama4Model(LlamaModel): ) # Expert parameter mapping for the case where the expert weights are # fused into a single weight tensor. - expert_params_mapping_fused = FusedMoE.make_expert_params_mapping( + expert_params_mapping_fused = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_up_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/longcat_flash.py b/vllm/model_executor/models/longcat_flash.py index 945fcb61509..d81df6f3373 100644 --- a/vllm/model_executor/models/longcat_flash.py +++ b/vllm/model_executor/models/longcat_flash.py @@ -46,7 +46,10 @@ from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -622,7 +625,7 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/mimo_v2_flash.py b/vllm/model_executor/models/mimo_v2_flash.py index 0b466f16601..0fe31c129e0 100644 --- a/vllm/model_executor/models/mimo_v2_flash.py +++ b/vllm/model_executor/models/mimo_v2_flash.py @@ -22,7 +22,10 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -511,7 +514,7 @@ class MiMoV2Model(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/minimax_m2.py b/vllm/model_executor/models/minimax_m2.py index 84d8dda533f..c4a00f41012 100644 --- a/vllm/model_executor/models/minimax_m2.py +++ b/vllm/model_executor/models/minimax_m2.py @@ -38,7 +38,10 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -393,7 +396,7 @@ class MiniMaxM2Model(nn.Module, EagleModelMixin): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/minimax_text_01.py b/vllm/model_executor/models/minimax_text_01.py index 67d7cb2d8bc..c73fbf7009d 100644 --- a/vllm/model_executor/models/minimax_text_01.py +++ b/vllm/model_executor/models/minimax_text_01.py @@ -24,7 +24,9 @@ from vllm.distributed.parallel_state import ( from vllm.forward_context import get_forward_context from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, diff --git a/vllm/model_executor/models/mixtral.py b/vllm/model_executor/models/mixtral.py index c182444f667..cbfc254dda3 100644 --- a/vllm/model_executor/models/mixtral.py +++ b/vllm/model_executor/models/mixtral.py @@ -40,7 +40,10 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -364,7 +367,7 @@ class MixtralModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 227ef2fa669..8fe1be721c7 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -40,7 +40,9 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.distributed import get_tensor_model_parallel_world_size from vllm.inputs import MultiModalDataDict from vllm.model_executor.layers.attention import MMEncoderAttention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.linear import ( ColumnParallelLinear, QKVParallelLinear, @@ -1072,7 +1074,7 @@ class Llama4ForConditionalGeneration( def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index 9b8ed68560c..537e19afbca 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -37,6 +37,7 @@ from vllm.model_executor.layers.fused_moe import ( FusedMoE, GateLinear, activation_without_mul, + fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -652,7 +653,7 @@ class NemotronHModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: if self.has_moe: # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( # - FusedMoe.w1 (aka gate_proj) should be up_proj since that's # what the activation is applied to # - FusedMoe.w3 (aka up_proj) should be ignored since we're diff --git a/vllm/model_executor/models/nemotron_h_mtp.py b/vllm/model_executor/models/nemotron_h_mtp.py index 12551d4254e..fe737438c30 100644 --- a/vllm/model_executor/models/nemotron_h_mtp.py +++ b/vllm/model_executor/models/nemotron_h_mtp.py @@ -11,7 +11,9 @@ import torch.nn as nn from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.config.parallel import ParallelConfig -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor @@ -399,7 +401,7 @@ class NemotronHMTP(nn.Module, SupportsPP): if getattr(self.config, "model_type", None) == "nemotron_h_puzzle": num_experts = self.config.mtp_n_routed_experts if num_experts is not None: - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="up_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/olmoe.py b/vllm/model_executor/models/olmoe.py index fcde2e41afb..1f342ad1733 100644 --- a/vllm/model_executor/models/olmoe.py +++ b/vllm/model_executor/models/olmoe.py @@ -32,7 +32,10 @@ from vllm.distributed import ( from vllm.distributed.utils import split_tensor_along_last_dim from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( QKVParallelLinear, @@ -336,7 +339,7 @@ class OlmoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/openpangu.py b/vllm/model_executor/models/openpangu.py index 96b837e42a8..68ab4a9ae4c 100644 --- a/vllm/model_executor/models/openpangu.py +++ b/vllm/model_executor/models/openpangu.py @@ -44,7 +44,10 @@ from vllm.model_executor.layers.attention import ( Attention, StaticSinkAttention, ) -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -1149,7 +1152,7 @@ class OpenPanguModel(nn.Module): ] has_experts = hasattr(self.config, "n_routed_experts") if has_experts: - expert_merge_mapping = FusedMoE.make_expert_params_mapping( + expert_merge_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/openpangu_mtp.py b/vllm/model_executor/models/openpangu_mtp.py index 91b454a4bc3..3a04ccdff5b 100644 --- a/vllm/model_executor/models/openpangu_mtp.py +++ b/vllm/model_executor/models/openpangu_mtp.py @@ -28,7 +28,9 @@ from vllm.config import VllmConfig # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -147,7 +149,7 @@ class OpenPanguMTP(nn.Module): ("fused_qkv_a_proj", "kv_a_proj_with_mqa", 1), ] - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/param2moe.py b/vllm/model_executor/models/param2moe.py index 4d1b3ff1b99..e8ea2dbc0e6 100644 --- a/vllm/model_executor/models/param2moe.py +++ b/vllm/model_executor/models/param2moe.py @@ -32,7 +32,10 @@ from vllm.distributed import ( ) from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -690,7 +693,7 @@ class Param2MoEModel(nn.Module): return loaded_params def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/phimoe.py b/vllm/model_executor/models/phimoe.py index 7d6083f202e..5770420ce56 100644 --- a/vllm/model_executor/models/phimoe.py +++ b/vllm/model_executor/models/phimoe.py @@ -35,7 +35,10 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import CacheConfig, VllmConfig from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.linear import ( QKVParallelLinear, ReplicatedLinear, @@ -514,7 +517,7 @@ class PhiMoEModel(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="w1", ckpt_down_proj_name="w2", diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index 7fc3c6a7dde..77eea390eda 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -40,7 +40,10 @@ from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -418,7 +421,7 @@ class Qwen2MoeModel(nn.Module): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index bbb296d28c9..e86b205b9f3 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -12,7 +12,9 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -194,7 +196,7 @@ class Qwen3_5MultiTokenPredictor(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index 520126718fd..4ec1be3367d 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -43,7 +43,10 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, @@ -516,7 +519,7 @@ class Qwen3MoeModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 2a4021be6e4..96d7e9c713c 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -23,7 +23,10 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import ( GemmaRMSNorm as Qwen3NextRMSNorm, ) @@ -533,7 +536,7 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/qwen3_next_mtp.py b/vllm/model_executor/models/qwen3_next_mtp.py index 751d7c23eb9..2f411c48a63 100644 --- a/vllm/model_executor/models/qwen3_next_mtp.py +++ b/vllm/model_executor/models/qwen3_next_mtp.py @@ -11,7 +11,9 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import VllmConfig from vllm.distributed.parallel_state import get_pp_group from vllm.logger import init_logger -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.linear import ColumnParallelLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -145,7 +147,7 @@ class Qwen3NextMultiTokenPredictor(nn.Module): # Params for weights, fp8 weight scales, fp8 activation scales # (param_name, weight_name, expert_id, shard_id) - expert_params_mapping = FusedMoE.make_expert_params_mapping( + expert_params_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/sarvam.py b/vllm/model_executor/models/sarvam.py index c770e203200..a0ab6c0ce26 100644 --- a/vllm/model_executor/models/sarvam.py +++ b/vllm/model_executor/models/sarvam.py @@ -35,7 +35,10 @@ from vllm.distributed import ( get_tensor_model_parallel_world_size, ) from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -529,7 +532,7 @@ class SarvamMLAModel(nn.Module): return hidden_states def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: - return FusedMoE.make_expert_params_mapping( + return fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/step3_text.py b/vllm/model_executor/models/step3_text.py index 912a1b07546..8f08f6c6071 100644 --- a/vllm/model_executor/models/step3_text.py +++ b/vllm/model_executor/models/step3_text.py @@ -18,7 +18,9 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, +) from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, diff --git a/vllm/model_executor/models/step3p5.py b/vllm/model_executor/models/step3p5.py index a0bc1211bfe..df051fb8735 100644 --- a/vllm/model_executor/models/step3p5.py +++ b/vllm/model_executor/models/step3p5.py @@ -23,7 +23,10 @@ from vllm.distributed import ( from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul, SwigluStepAndMul from vllm.model_executor.layers.attention import Attention -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.layers.layernorm import GemmaRMSNorm from vllm.model_executor.layers.linear import ( ColumnParallelLinear, @@ -637,7 +640,7 @@ class Step3p5Model(nn.Module): ] # New per-expert format: .moe.experts.E.gate_proj.weight_packed [out, in] - per_expert_mapping = FusedMoE.make_expert_params_mapping( + per_expert_mapping = fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name="gate_proj", ckpt_down_proj_name="down_proj", diff --git a/vllm/model_executor/models/transformers/moe.py b/vllm/model_executor/models/transformers/moe.py index cf13958ef76..51a51799ffc 100644 --- a/vllm/model_executor/models/transformers/moe.py +++ b/vllm/model_executor/models/transformers/moe.py @@ -25,7 +25,10 @@ from vllm.config.utils import getattr_iter from vllm.distributed import get_dp_group, get_ep_group from vllm.forward_context import ForwardContext, get_forward_context from vllm.model_executor.custom_op import PluggableLayer -from vllm.model_executor.layers.fused_moe import FusedMoE +from vllm.model_executor.layers.fused_moe import ( + FusedMoE, + fused_moe_make_expert_params_mapping, +) from vllm.model_executor.models.interfaces import MixtureOfExperts from vllm.model_executor.models.utils import maybe_prefix from vllm.platforms import current_platform @@ -179,7 +182,7 @@ class MoEMixin(MixtureOfExperts): num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts for gate_proj, down_proj, up_proj in ckpt_names: expert_mapping.extend( - FusedMoE.make_expert_params_mapping( + fused_moe_make_expert_params_mapping( self, ckpt_gate_proj_name=gate_proj, ckpt_down_proj_name=down_proj, From 5ef33ab250b3904da375ecb18bdda00a4a73c3a8 Mon Sep 17 00:00:00 2001 From: Or Ozeri Date: Thu, 23 Apr 2026 20:00:45 +0300 Subject: [PATCH 074/153] [kv_offload+HMA][10/N]: Support load with multiple KV groups (#39402) Signed-off-by: Or Ozeri --- .../kv_connector/v1/offloading/scheduler.py | 78 ++++++++++++------- 1 file changed, 50 insertions(+), 28 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index bff512815a6..5cee750811e 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -14,6 +14,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import ( ReqId, ) from vllm.logger import init_logger +from vllm.utils.math_utils import cdiv from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_offload.abstract import ( @@ -271,45 +272,66 @@ class OffloadingConnectorScheduler: return req_status = self._req_status[request.request_id] - block_groups = blocks.get_block_ids() - # Below assertions will be removed once this function supports HMA - assert len(self.config.kv_group_configs) == 1 - assert len(req_status.group_states) == 1 - assert len(block_groups) == 1 - block_ids = block_groups[0] - group_config = self.config.kv_group_configs[0] - group_state = req_status.group_states[0] + num_locally_computed_tokens = req_status.num_locally_computed_tokens + num_cached_tokens = num_locally_computed_tokens + num_external_tokens - num_computed_gpu_blocks = sum( - block.block_hash is not None for block in blocks.blocks[0] - ) - num_computed_tokens = num_computed_gpu_blocks * group_config.gpu_block_size - full_block_tokens = num_computed_tokens + num_external_tokens - assert full_block_tokens % group_config.offloaded_block_size == 0 + keys_to_load: list[OffloadKey] = [] + dst_block_ids: list[int] = [] + # per group + group_sizes: list[int] = [] + block_indices: list[int] = [] + for group_config, group_state, group_blocks in zip( + self.config.kv_group_configs, + req_status.group_states, + blocks.blocks, + ): + gpu_block_size = group_config.gpu_block_size + offloaded_block_size = group_config.offloaded_block_size + offload_keys = group_state.offload_keys + num_gpu_blocks = cdiv(num_cached_tokens, gpu_block_size) - num_pending_gpu_blocks = len(block_ids) - num_computed_gpu_blocks - assert ( - num_external_tokens == num_pending_gpu_blocks * group_config.gpu_block_size - ) + assert len(group_blocks) >= num_gpu_blocks + num_locally_computed_gpu_blocks = num_gpu_blocks + # Skip null placeholder blocks (used for sliding window or mamba padding). + for i, block in enumerate(group_blocks[:num_gpu_blocks]): + if not block.is_null and block.block_hash is None: + num_locally_computed_gpu_blocks = i + break - start_block_idx = num_computed_tokens // group_config.offloaded_block_size - num_blocks = full_block_tokens // group_config.offloaded_block_size + assert ( + num_locally_computed_tokens + <= num_locally_computed_gpu_blocks * gpu_block_size + ) + num_pending_gpu_blocks = num_gpu_blocks - num_locally_computed_gpu_blocks - assert len(request.block_hashes) // self.config.block_size_factor >= num_blocks - offload_keys = group_state.offload_keys[start_block_idx:num_blocks] + num_blocks = cdiv(num_cached_tokens, offloaded_block_size) + assert len(offload_keys) >= num_blocks + if num_pending_gpu_blocks: + start_block_idx = ( + num_locally_computed_gpu_blocks // self.config.block_size_factor + ) + keys_to_load.extend(offload_keys[start_block_idx:num_blocks]) - src_spec = self.manager.prepare_load(offload_keys, req_status.req_context) + dst_block_ids.extend( + block.block_id + for block in group_blocks[ + num_locally_computed_gpu_blocks:num_gpu_blocks + ] + ) + group_sizes.append(num_pending_gpu_blocks) + block_indices.append(num_locally_computed_gpu_blocks) + + group_state.next_stored_block_idx = num_blocks + + src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context) dst_spec = GPULoadStoreSpec( - block_ids[num_computed_gpu_blocks:], - group_sizes=(num_pending_gpu_blocks,), - block_indices=(num_computed_gpu_blocks,), + dst_block_ids, group_sizes=group_sizes, block_indices=block_indices ) self._reqs_to_load[request.request_id] = (src_spec, dst_spec) req_blocks_being_loaded = self._reqs_being_loaded[request.request_id] - req_blocks_being_loaded.update(offload_keys) - group_state.next_stored_block_idx = num_blocks + req_blocks_being_loaded.update(keys_to_load) if self._blocks_being_loaded is not None: self._blocks_being_loaded.update(req_blocks_being_loaded) From e9ba519f450fd0c3eea5cda44e73eec3ad34f654 Mon Sep 17 00:00:00 2001 From: shaharmor98 <17088876+shaharmor98@users.noreply.github.com> Date: Thu, 23 Apr 2026 20:21:13 +0300 Subject: [PATCH 075/153] [DP][Ray] Pin DP control bundle to same node as first GPU bundle (#39167) Signed-off-by: Shahar Mor Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/v1/engine/utils.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/vllm/v1/engine/utils.py b/vllm/v1/engine/utils.py index 0de9b9ba4d9..53cad2bc153 100644 --- a/vllm/v1/engine/utils.py +++ b/vllm/v1/engine/utils.py @@ -80,6 +80,21 @@ class EngineHandshakeMetadata: parallel_config: dict[str, int | str | list[int]] +def _make_control_bundle(node_ip: str) -> dict[str, float]: + # The engine actor is scheduled on the final CPU-only bundle. Keep that + # bundle colocated with the group's first GPU bundle so the actor does not + # float to an unrelated node and reorder worker ranks away from the + # advertised DP bootstrap host. + return {"CPU": 1.0, "node:" + node_ip: 0.001} + + +def _get_bundle_node_ip(bundle: dict[str, float]) -> str: + for key in bundle: + if key.startswith("node:"): + return key.split(":", 1)[1] + raise ValueError(f"Missing node affinity in placement bundle: {bundle}") + + class CoreEngineProcManager: """ Utility class to handle creation, readiness, and shutdown @@ -597,10 +612,20 @@ class CoreEngineActorManager: if len(collected_bundles) < world_size: continue - bundles = collected_bundles + [{"CPU": 1.0}] + control_node_ip = _get_bundle_node_ip(collected_bundles[0]) + bundles = collected_bundles + [ + _make_control_bundle(control_node_ip) + ] collected_bundles = [] else: - bundles = device_bundle * world_size + [{"CPU": 1.0}] + # STRICT_PACK already keeps every bundle in the placement + # group on one node, so the explicit node affinity on the + # control bundle is redundant for correctness here. Keep it + # anyway for consistency with the span path and to preserve + # intent if this scheduling strategy changes later. + bundles = device_bundle * world_size + [ + _make_control_bundle(node_ip) + ] pg = ray.util.placement_group( name=f"dp_rank_{len(placement_groups)}", From 1b1c01de39425f5ccce2ffc45f0ce3eb9fc2ce2c Mon Sep 17 00:00:00 2001 From: Jackmin801 <56836461+Jackmin801@users.noreply.github.com> Date: Thu, 23 Apr 2026 10:38:10 -0700 Subject: [PATCH 076/153] [MoE] Move xpu moe to fused_moe/experts/ (#40568) Signed-off-by: Jackmin801 Co-authored-by: Claude Co-authored-by: Kunshang Ji --- .github/mergify.yml | 2 +- vllm/model_executor/layers/fused_moe/__init__.py | 10 ++++++---- .../fused_moe/{xpu_fused_moe.py => experts/xpu_moe.py} | 0 vllm/model_executor/layers/fused_moe/oracle/fp8.py | 4 ++-- vllm/model_executor/layers/fused_moe/oracle/mxfp4.py | 2 +- .../layers/fused_moe/oracle/unquantized.py | 2 +- 6 files changed, 11 insertions(+), 9 deletions(-) rename vllm/model_executor/layers/fused_moe/{xpu_fused_moe.py => experts/xpu_moe.py} (100%) diff --git a/.github/mergify.yml b/.github/mergify.yml index baf65e14a88..b96d6b81ac0 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -262,7 +262,7 @@ pull_request_rules: - files~=^docker/Dockerfile.xpu - files~=^\\.buildkite/intel_jobs/ - files=\.buildkite/ci_config_intel.yaml - - files=vllm/model_executor/layers/fused_moe/xpu_fused_moe.py + - files=vllm/model_executor/layers/fused_moe/experts/xpu_moe.py - files=vllm/model_executor/kernels/linear/mixed_precision/xpu.py - files=vllm/model_executor/kernels/linear/mxfp8/xpu.py - files=vllm/model_executor/kernels/linear/scaled_mm/xpu.py diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index a154ede547b..1d273bd31e4 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -85,6 +85,11 @@ if HAS_TRITON: from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( DeepGemmExperts, ) + from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( + XPUExperts, + XPUExpertsFp8, + XPUExpertsMXFp4, + ) from vllm.model_executor.layers.fused_moe.fused_batched_moe import ( BatchedTritonExperts, ) @@ -106,10 +111,6 @@ if HAS_TRITON: from vllm.model_executor.layers.fused_moe.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, ) - from vllm.model_executor.layers.fused_moe.xpu_fused_moe import ( - XPUExperts, - XPUExpertsFp8, - ) __all__ += [ "AiterExperts", @@ -129,6 +130,7 @@ if HAS_TRITON: "TritonOrDeepGemmExperts", "XPUExperts", "XPUExpertsFp8", + "XPUExpertsMXFp4", ] else: # Some model classes directly use the custom ops. Add placeholders diff --git a/vllm/model_executor/layers/fused_moe/xpu_fused_moe.py b/vllm/model_executor/layers/fused_moe/experts/xpu_moe.py similarity index 100% rename from vllm/model_executor/layers/fused_moe/xpu_fused_moe.py rename to vllm/model_executor/layers/fused_moe/experts/xpu_moe.py diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 584c2bf7928..ca13d0d901d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -180,7 +180,7 @@ def backend_to_kernel_cls( return [CutlassBatchedExpertsFp8] elif backend == Fp8MoeBackend.XPU: - from vllm.model_executor.layers.fused_moe.xpu_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( XPUExpertsFp8, ) @@ -470,7 +470,7 @@ def convert_to_fp8_moe_kernel_format( is_trtllm=(fp8_backend == Fp8MoeBackend.FLASHINFER_TRTLLM), ) elif fp8_backend == Fp8MoeBackend.XPU: - from vllm.model_executor.layers.fused_moe.xpu_fused_moe import ( + from vllm.model_executor.layers.fused_moe.experts.xpu_moe import ( prepare_fp8_moe_layer_for_xpu, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index 6306d0e2e9d..9d2c9f8baff 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -141,7 +141,7 @@ def backend_to_kernel_cls( return [AiterExperts] elif backend == Mxfp4MoeBackend.XPU: - from vllm.model_executor.layers.fused_moe.xpu_fused_moe import XPUExpertsMXFp4 + from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMXFp4 return [XPUExpertsMXFp4] diff --git a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py index cdfd6bb8c02..00fe914ad9d 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/unquantized.py +++ b/vllm/model_executor/layers/fused_moe/oracle/unquantized.py @@ -121,7 +121,7 @@ def backend_to_kernel_cls( return BatchedTritonExperts elif backend == UnquantizedMoeBackend.XPU: - from vllm.model_executor.layers.fused_moe.xpu_fused_moe import XPUExperts + from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExperts return XPUExperts From 7f95a66cbffcd111c6d37abdcb7ca297cec47b78 Mon Sep 17 00:00:00 2001 From: Johnny Date: Thu, 23 Apr 2026 21:42:14 +0200 Subject: [PATCH 077/153] [NVIDIA] Add sm_110 (Jetson Thor) to CUDA 13.0 build targets (#39233) --- docker/Dockerfile | 4 ++-- docker/docker-bake.hcl | 2 +- docker/versions.json | 2 +- tools/flashinfer-build.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d76a2e986b7..258754b777d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -188,7 +188,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Explicitly set the list to avoid issues with torch 2.2 # See https://github.com/pytorch/pytorch/pull/123243 # From versions.json: .torch.cuda_arch_list -ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' +ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0+PTX' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} #################### BUILD BASE IMAGE #################### @@ -765,7 +765,7 @@ ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL ENV UV_HTTP_TIMEOUT=500 # install kv_connectors if requested -ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX' +ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0+PTX' ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list} RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=requirements/kv_connectors.txt,target=/tmp/kv_connectors.txt,ro \ diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl index 055287ca3be..785d598d608 100644 --- a/docker/docker-bake.hcl +++ b/docker/docker-bake.hcl @@ -20,7 +20,7 @@ variable "NVCC_THREADS" { } variable "TORCH_CUDA_ARCH_LIST" { - default = "8.0 8.9 9.0 10.0" + default = "8.0 8.9 9.0 10.0 11.0 12.0" } variable "COMMIT" { diff --git a/docker/versions.json b/docker/versions.json index f4e05914afa..f3d848cba10 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -32,7 +32,7 @@ "default": "false" }, "TORCH_CUDA_ARCH_LIST": { - "default": "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX" + "default": "7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0+PTX" }, "MAX_JOBS": { "default": "2" diff --git a/tools/flashinfer-build.sh b/tools/flashinfer-build.sh index 8bb63007024..fb148f056f6 100755 --- a/tools/flashinfer-build.sh +++ b/tools/flashinfer-build.sh @@ -35,7 +35,7 @@ elif [[ "${CUDA_VERSION}" == 12.[8-9]* ]]; then FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0a 10.3a 12.0" else # CUDA 13.0+ - FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0f 12.0" + FI_TORCH_CUDA_ARCH_LIST="7.5 8.0 8.9 9.0a 10.0f 11.0 12.0f" fi echo "🏗️ Building FlashInfer AOT for arches: ${FI_TORCH_CUDA_ARCH_LIST}" From 7ff65b19003be4955d2d5d1428e7d94d082559d0 Mon Sep 17 00:00:00 2001 From: czhu-cohere Date: Thu, 23 Apr 2026 13:50:05 -0700 Subject: [PATCH 078/153] [Bugfix] Fix workspace resize leaking reserved GPU memory (#39226) Signed-off-by: root Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/v1/worker/workspace.py | 39 +++++++++++++++++-------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/vllm/v1/worker/workspace.py b/vllm/v1/worker/workspace.py index 7e21d89f703..1c502bfd8ff 100644 --- a/vllm/v1/worker/workspace.py +++ b/vllm/v1/worker/workspace.py @@ -161,36 +161,33 @@ class WorkspaceManager: "Workspace growth is not allowed after locking." ) - for ubatch_id in range(self._num_ubatches): - current_workspace = self._current_workspaces[ubatch_id] - if ( - current_workspace is None - or self._workspace_size_bytes(current_workspace) < required_bytes - ): - # Delete old tensor before allocating new one to avoid - # memory spike from resize_(). resize_() allocates new - # memory before freeing old, which can cause OOM. - # Must clear the list reference first since local var - # is just a copy of the reference. - self._current_workspaces[ubatch_id] = None - del current_workspace - self._current_workspaces[ubatch_id] = torch.empty( - (required_bytes,), dtype=torch.uint8, device=self._device - ) + # Only resize the requesting ubatch's workspace. Other + # ubatches resize lazily on their next get_simultaneous call. + # Resizing all ubatches here would orphan the other ubatch's + # old tensor when it still holds views into it (DBO leak). + self._current_workspaces[ubatch_id] = None + del current_workspace + # Release the freed segment back to CUDA so the caching + # allocator can reuse the GPU memory for the larger + # allocation below. Without this, each resize may leave a + # dead segment in reserved memory which can cause higher peak + # memory usage. + torch.accelerator.empty_cache() + self._current_workspaces[ubatch_id] = torch.empty( + (required_bytes,), dtype=torch.uint8, device=self._device + ) + current_workspace = self._current_workspaces[ubatch_id] if envs.VLLM_DEBUG_WORKSPACE: logger.info( "[WORKSPACE DEBUG] Resized workspace from '%s': %.2f MB -> " - "%.2f MB (%d ubatches, total memory %.2f MB)", + "%.2f MB (ubatch %d)", get_caller_info(), current_size / _MB, required_bytes / _MB, - self._num_ubatches, - required_bytes * self._num_ubatches / _MB, + ubatch_id, ) - current_workspace = self._current_workspaces[dbo_current_ubatch_id()] - return current_workspace From 4a6dd1c3cc4aabae616977bbddf3d6c53e20204b Mon Sep 17 00:00:00 2001 From: bnellnm <49004751+bnellnm@users.noreply.github.com> Date: Thu, 23 Apr 2026 18:11:37 -0400 Subject: [PATCH 079/153] [Bugfix] Fix DeepSeek V2-Lite Accuracy drop (#40673) Signed-off-by: Bill Nell --- .../layers/fused_moe/runner/moe_runner.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py index 00be12780a1..d6a6c0502c7 100644 --- a/vllm/model_executor/layers/fused_moe/runner/moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/moe_runner.py @@ -335,11 +335,16 @@ class MoERunner(MoERunnerInterface): """All-reduce shared expert output when the combine kernel already reduced fused output. - This is the "early" all-reduce path. When the combine kernel produces - already-reduced fused output, shared output must be reduced separately - to match. + * If the combine kernel does the reduction for fused_output, reduce + shared_output separately. O.w, reduce fused_output+shared_output later. + * If we have SP (TP=N, DP=M, EP), there is a separate AG step handled + in the model. """ - if shared_output is not None and self._fused_output_is_reduced: + if ( + shared_output is not None + and not self.moe_config.is_sequence_parallel + and self._fused_output_is_reduced + ): shared_output = tensor_model_parallel_all_reduce(shared_output) return shared_output From cde8d247102652713eca27101aae731d8f041cbd Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 23 Apr 2026 18:28:27 -0400 Subject: [PATCH 080/153] [Spec Decode] Move `SpecDecodeBaseProposer` out of `eagle.py` (#40732) Signed-off-by: Matthew Bonanni --- tests/v1/spec_decode/test_eagle.py | 6 +- tests/v1/spec_decode/test_mtp.py | 6 +- vllm/v1/spec_decode/dflash.py | 2 +- vllm/v1/spec_decode/draft_model.py | 2 +- vllm/v1/spec_decode/eagle.py | 1775 +-------------------- vllm/v1/spec_decode/llm_base_proposer.py | 1778 ++++++++++++++++++++++ vllm/v1/worker/cpu_model_runner.py | 8 +- 7 files changed, 1792 insertions(+), 1785 deletions(-) create mode 100644 vllm/v1/spec_decode/llm_base_proposer.py diff --git a/tests/v1/spec_decode/test_eagle.py b/tests/v1/spec_decode/test_eagle.py index 188e84abca0..462ddfdfe50 100644 --- a/tests/v1/spec_decode/test_eagle.py +++ b/tests/v1/spec_decode/test_eagle.py @@ -741,9 +741,9 @@ def test_set_inputs_first_pass_parallel_drafting(): @pytest.mark.parametrize("pp_size", [1, 2]) @pytest.mark.parametrize("use_distinct_embed_tokens", [True, False]) @pytest.mark.parametrize("use_distinct_lm_head", [True, False]) -@mock.patch("vllm.v1.spec_decode.eagle.get_pp_group") -@mock.patch("vllm.v1.spec_decode.eagle.get_layers_from_vllm_config") -@mock.patch("vllm.v1.spec_decode.eagle.get_model") +@mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_pp_group") +@mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_layers_from_vllm_config") +@mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_model") def test_load_model( mock_get_model, mock_get_layers, diff --git a/tests/v1/spec_decode/test_mtp.py b/tests/v1/spec_decode/test_mtp.py index 094611e05c1..7c478f81d86 100644 --- a/tests/v1/spec_decode/test_mtp.py +++ b/tests/v1/spec_decode/test_mtp.py @@ -61,9 +61,9 @@ def _create_mtp_proposer(num_speculative_tokens: int) -> EagleProposer: return EagleProposer(vllm_config=vllm_config, device=DEVICE_TYPE) -@mock.patch("vllm.v1.spec_decode.eagle.get_pp_group") -@mock.patch("vllm.v1.spec_decode.eagle.get_layers_from_vllm_config") -@mock.patch("vllm.v1.spec_decode.eagle.get_model") +@mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_pp_group") +@mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_layers_from_vllm_config") +@mock.patch("vllm.v1.spec_decode.llm_base_proposer.get_model") def test_mtp_load_model_unified(mock_get_model, mock_get_layers, mock_get_pp_group): """Test MTP-specific model loading with unified model approach.""" diff --git a/vllm/v1/spec_decode/dflash.py b/vllm/v1/spec_decode/dflash.py index 51916053c7d..cb31a97a131 100644 --- a/vllm/v1/spec_decode/dflash.py +++ b/vllm/v1/spec_decode/dflash.py @@ -11,7 +11,7 @@ from vllm.forward_context import set_forward_context from vllm.logger import init_logger from vllm.triton_utils import triton from vllm.v1.attention.backend import CommonAttentionMetadata -from vllm.v1.spec_decode.eagle import SpecDecodeBaseProposer +from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer from vllm.v1.spec_decode.utils import copy_and_expand_dflash_inputs_kernel logger = init_logger(__name__) diff --git a/vllm/v1/spec_decode/draft_model.py b/vllm/v1/spec_decode/draft_model.py index 9633e2ef6ca..a8c8ab03b61 100644 --- a/vllm/v1/spec_decode/draft_model.py +++ b/vllm/v1/spec_decode/draft_model.py @@ -9,7 +9,7 @@ from vllm.config import VllmConfig from vllm.config.utils import replace from vllm.logger import init_logger from vllm.model_executor.model_loader import get_model -from vllm.v1.spec_decode.eagle import SpecDecodeBaseProposer +from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer logger = init_logger(__name__) diff --git a/vllm/v1/spec_decode/eagle.py b/vllm/v1/spec_decode/eagle.py index f22e15b79f6..002d0b7833a 100644 --- a/vllm/v1/spec_decode/eagle.py +++ b/vllm/v1/spec_decode/eagle.py @@ -1,1735 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import ast -from importlib.util import find_spec -from typing import Any, cast -import numpy as np import torch -import torch.nn as nn -from vllm.config import ( - CUDAGraphMode, - VllmConfig, - get_layers_from_vllm_config, - replace, -) -from vllm.distributed.parallel_state import get_pp_group -from vllm.forward_context import set_forward_context -from vllm.logger import init_logger -from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase -from vllm.model_executor.model_loader import get_model -from vllm.model_executor.models import supports_multimodal -from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausalLM -from vllm.model_executor.models.interfaces import SupportsMultiModal -from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM -from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM -from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.platforms import current_platform -from vllm.utils.platform_utils import is_pin_memory_available -from vllm.v1.attention.backend import CommonAttentionMetadata -from vllm.v1.attention.backends.registry import AttentionBackendEnum -from vllm.v1.attention.backends.tree_attn import ( - TreeAttentionMetadata, - TreeAttentionMetadataBuilder, -) -from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata -from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher -from vllm.v1.kv_cache_interface import KVCacheConfig, UniformTypeKVCacheSpecs -from vllm.v1.sample.metadata import SamplingMetadata -from vllm.v1.sample.sampler import _SAMPLING_EPS -from vllm.v1.spec_decode.metadata import SpecDecodeMetadata -from vllm.v1.spec_decode.utils import ( - PADDING_SLOT_ID, - compute_new_slot_mapping, - copy_and_expand_eagle_inputs_kernel, - eagle_prepare_inputs_padded_kernel, - eagle_prepare_next_token_padded_kernel, - eagle_step_update_slot_mapping_and_metadata, - extend_all_queries_by_N, - next_power_of_2, -) -from vllm.v1.utils import CpuGpuBuffer -from vllm.v1.worker.dp_utils import coordinate_batch_across_dp -from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch -from vllm.v1.worker.utils import AttentionGroup - -logger = init_logger(__name__) - - -class SpecDecodeBaseProposer: - def __init__( - self, - vllm_config: VllmConfig, - device: torch.device, - pass_hidden_states_to_model: bool, - runner=None, - ): - self.vllm_config = vllm_config - assert vllm_config.speculative_config is not None - self.speculative_config = vllm_config.speculative_config - self.draft_model_config = self.speculative_config.draft_model_config - self.method = self.speculative_config.method - self.pass_hidden_states_to_model = pass_hidden_states_to_model - - self.device = device - self.dtype = vllm_config.model_config.dtype - self.max_model_len = vllm_config.model_config.max_model_len - self.dp_rank = vllm_config.parallel_config.data_parallel_rank - self.num_speculative_tokens = self.speculative_config.num_speculative_tokens - - # We need to get the hidden size from the draft model config because - # the draft model's hidden size can be different from the target model's - # hidden size (e.g., Llama 3.3 70B). - self.hidden_size = self.draft_model_config.get_hidden_size() - self.inputs_embeds_size = self.draft_model_config.get_inputs_embeds_size() - - # Unifying eagle, draft model, and parallel drafting support. - # DFlash always uses parallel drafting (all tokens in one pass), - # but has an additional slot for the next_token_id (does not shift like EAGLE) - self.parallel_drafting: bool = self.speculative_config.parallel_drafting - self.extra_slots_per_request = ( - 1 if not self.parallel_drafting else self.num_speculative_tokens - ) - self.net_num_new_slots_per_request = self.extra_slots_per_request - ( - 1 if (self.pass_hidden_states_to_model and self.method != "dflash") else 0 - ) - self.needs_extra_input_slots = self.net_num_new_slots_per_request > 0 - - self.parallel_drafting_token_id: int = 0 - self.parallel_drafting_hidden_state_tensor: torch.Tensor | None = None - if self.parallel_drafting: - self._init_parallel_drafting_params() - self.use_local_argmax_reduction: bool = ( - self.speculative_config.use_local_argmax_reduction - ) - - self.max_batch_size = vllm_config.scheduler_config.max_num_seqs - self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens - self.token_arange_np = np.arange(self.max_num_tokens) - - # Can be specialized by methods like DFlash to reduce the limit - self.max_query_tokens = self.max_num_tokens - self.max_positions = self.max_num_tokens - - # Multi-modal data support - self.mm_registry = MULTIMODAL_REGISTRY - self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs( - vllm_config.model_config - ) - - self.draft_attn_groups: list[AttentionGroup] = [] - self.kv_cache_gid: int = -1 - self.eagle3_use_aux_hidden_state: bool = ( - self._get_eagle3_use_aux_hidden_state_from_config() - ) - - self.compilation_config = self.vllm_config.compilation_config - - # Cudagraph dispatcher for PIECEWISE-only dispatching in eagle. - # Keys are initialized later via initialize_cudagraph_keys() called from - # gpu_model_runner._check_and_update_cudagraph_mode after - # adjust_cudagraph_sizes_for_spec_decode is called. - self.cudagraph_dispatcher = CudagraphDispatcher(self.vllm_config) - - # persistent buffers for cuda graph - self.input_ids = torch.zeros( - self.max_num_tokens, dtype=torch.int32, device=device - ) - # Use draft model's M-RoPE setting, not target model's - # Draft models may be text-only even if target is multimodal - self.uses_mrope = self.draft_model_config.uses_mrope - self.uses_xdrope_dim = self.vllm_config.model_config.uses_xdrope_dim - self.draft_uses_xdrope_dim = self.draft_model_config.uses_xdrope_dim - if self.uses_mrope: - # NOTE: `mrope_positions` is implemented with one additional dummy - # position on purpose to make it non-contiguous so that it can work - # with torch compile. - # See detailed explanation in https://github.com/vllm-project/vllm/pull/12128#discussion_r1926431923 - - # NOTE: When M-RoPE is enabled, position ids are 3D regardless of - # the modality of inputs. For text-only inputs, each dimension has - # identical position IDs, making M-RoPE functionally equivalent to - # 1D-RoPE. - # See page 5 of https://arxiv.org/abs/2409.12191 - self.mrope_positions = torch.zeros( - (3, self.max_positions + 1), dtype=torch.int64, device=device - ) - elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: - self.xdrope_positions = torch.zeros( - (self.uses_xdrope_dim, self.max_positions + 1), - dtype=torch.int64, - device=device, - ) - else: - # RoPE need (max_num_tokens,) - self.positions = torch.zeros( - self.max_positions, - dtype=torch.int64, - device=device, - ) - self.hidden_states = torch.zeros( - (self.max_num_tokens, self.hidden_size), dtype=self.dtype, device=device - ) - - # Will be set when we initialize the attention backend - self.block_size: int = -1 - - # We need +1 here because the arange is used to set query_start_loc, - # which has one more element than batch_size. - max_num_slots_for_arange = max(self.max_batch_size + 1, self.max_num_tokens) - self.arange = torch.arange( - max_num_slots_for_arange, device=device, dtype=torch.int32 - ) - - if self.needs_extra_input_slots: - self._raise_if_padded_drafter_batch_disabled() - self._raise_if_multimodal() - self._raise_if_mrope() - - self.is_rejected_token_mask: torch.Tensor | None = None - self.is_masked_token_mask: torch.Tensor | None = None - if self.needs_extra_input_slots: - # For draft models and parallel drafting, we need to keep track of - # which tokens are rejected to update the slot mapping with padding slots. - self.is_rejected_token_mask = torch.zeros( - (self.max_num_tokens,), dtype=torch.bool, device=device - ) - # For parallel drafting, we also need to keep track of which tokens - # are parallel-padding tokens used to sample at later positions. - # We populate this tensor even when using draft models for simplicity. - self.is_masked_token_mask = torch.zeros( - (self.max_num_tokens,), dtype=torch.bool, device=device - ) - - self.inputs_embeds = torch.zeros( - (self.max_num_tokens, self.inputs_embeds_size), - dtype=self.dtype, - device=device, - ) - - self.backup_next_token_ids = CpuGpuBuffer( - self.max_batch_size, - dtype=torch.int32, - pin_memory=is_pin_memory_available(), - device=device, - with_numpy=True, - ) - - self._slot_mapping_buffer = torch.zeros( - self.max_positions, - dtype=torch.int64, - device=device, - ) - - # Determine allowed attention backends once during initialization. - self.allowed_attn_types: tuple | None = None - if current_platform.is_rocm(): - from vllm.v1.attention.backends.mla.indexer import ( - DeepseekV32IndexerMetadata, - ) - from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( - ROCMAiterMLASparseMetadata, - ) - from vllm.v1.attention.backends.rocm_attn import RocmAttentionMetadata - - rocm_types = [ - TritonAttentionMetadata, - RocmAttentionMetadata, - ROCMAiterMLASparseMetadata, - DeepseekV32IndexerMetadata, - ] - # ROCM_AITER_FA is an optional backend - # We check is_enabled() here to avoid importing the backend module during - # auto-discovery when VLLM_ROCM_USE_AITER=0, which would trigger aiter - # import and JIT compilation warnings. Explicit backend selection via - # attention_config still works because the backend module is loaded - # directly when selected, not through this auto-discovery path. - # Check if backend module exists to allow explicit selection - if find_spec( - AttentionBackendEnum.ROCM_AITER_FA.get_path(include_classname=False) - ): - from vllm.v1.attention.backends.rocm_aiter_fa import ( - AiterFlashAttentionMetadata, - ) - - rocm_types.append(AiterFlashAttentionMetadata) - - # TRITON_MLA backend support for MLA models (e.g., DeepSeek) - from vllm.model_executor.layers.attention.mla_attention import ( - MLACommonMetadata, - ) - - rocm_types.append(MLACommonMetadata) - - # FlexAttention backend support - from vllm.v1.attention.backends.flex_attention import FlexAttentionMetadata - - rocm_types.append(FlexAttentionMetadata) - - self.allowed_attn_types = tuple(rocm_types) - - # Parse the speculative token tree. - spec_token_tree = self.speculative_config.speculative_token_tree - assert spec_token_tree is not None - self.tree_choices: list[tuple[int, ...]] = ast.literal_eval(spec_token_tree) - tree_depth = len(self.tree_choices[-1]) - # Precompute per-level properties of the tree. - num_drafts_per_level = [0] * tree_depth - for node in self.tree_choices: - num_drafts_per_level[len(node) - 1] += 1 - self.cu_drafts_per_level = [num_drafts_per_level[0]] - self.child_drafts_per_level = [num_drafts_per_level[0]] - for level in range(1, tree_depth): - self.cu_drafts_per_level.append( - self.cu_drafts_per_level[-1] + num_drafts_per_level[level] - ) - self.child_drafts_per_level.append( - num_drafts_per_level[level] // num_drafts_per_level[level - 1] - ) - # Precompute draft position offsets in flattened tree. - self.tree_draft_pos_offsets = torch.arange( - 1, len(self.tree_choices) + 1, device=device, dtype=torch.int32 - ).repeat(self.max_batch_size, 1) - - def _raise_if_padded_drafter_batch_disabled(self): - if self.speculative_config.disable_padded_drafter_batch: - raise NotImplementedError( - "Speculative Decoding with draft models or parallel drafting only " - "supports padded drafter batch. Please unset " - "disable_padded_drafter_batch in the speculative_config." - ) - - def _raise_if_multimodal(self): - if self.supports_mm_inputs: - raise NotImplementedError( - "Speculative Decoding with draft models or parallel drafting " - "does not support multimodal models yet" - ) - - def _raise_if_mrope(self): - if self.draft_model_config.uses_mrope: - raise NotImplementedError( - "Speculative Decoding with draft models or parallel drafting " - "does not support M-RoPE yet" - ) - - def _init_parallel_drafting_params(self): - # For parallel drafting, we need the token ID to use for masked slots - # And for EAGLE + parallel drafting, we need the hidden state tensor to use - # for those masked slots. - - model_hf_config = self.draft_model_config.hf_config - # DFlash stores mask_token_id in dflash_config - dflash_config = getattr(model_hf_config, "dflash_config", None) - if dflash_config and "mask_token_id" in dflash_config: - self.parallel_drafting_token_id = dflash_config["mask_token_id"] - elif hasattr(model_hf_config, "pard_token"): - self.parallel_drafting_token_id = model_hf_config.pard_token - elif hasattr(model_hf_config, "ptd_token_id"): - self.parallel_drafting_token_id = model_hf_config.ptd_token_id - else: - raise ValueError( - "For parallel drafting, the draft model config must have " - "`pard_token`, `ptd_token_id`, or " - "`dflash_config.mask_token_id` specified in its config.json." - ) - - if self.pass_hidden_states_to_model: - self.parallel_drafting_hidden_state_tensor = torch.empty( - self.hidden_size, dtype=self.dtype, device=self.device - ) - - def _get_positions(self, num_tokens: int): - if self.uses_mrope: - return self.mrope_positions[:, :num_tokens] - if self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: - return self.xdrope_positions[:, :num_tokens] - return self.positions[:num_tokens] - - def _set_positions(self, num_tokens: int, positions: torch.Tensor): - if self.uses_mrope: - self.mrope_positions[:, :num_tokens] = positions - elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: - self.xdrope_positions[:, :num_tokens] = positions - else: - # Convert M-RoPE positions if target model uses M-RoPE - # but draft doesn't, For text inputs, all M-RoPE - # dimensions are identical - if self.vllm_config.model_config.uses_mrope: - positions = positions[0] - self.positions[:num_tokens] = positions - - def _get_slot_mapping( - self, - num_tokens: int, - slot_mapping: torch.Tensor | None = None, - ) -> dict[str, torch.Tensor]: - """Return slot_mapping dict for EAGLE layers. - - If slot_mapping is provided, copies it into the buffer first. - """ - if slot_mapping is not None: - num_actual = slot_mapping.shape[0] - self._slot_mapping_buffer[:num_actual].copy_(slot_mapping) - if num_tokens > num_actual: - self._slot_mapping_buffer[num_actual:num_tokens].fill_(PADDING_SLOT_ID) - - view = self._slot_mapping_buffer[:num_tokens] - return {name: view for name in self._draft_attn_layer_names} - - def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode) -> None: - """Initialize cudagraph dispatcher keys for eagle. - - Eagle only supports PIECEWISE cudagraphs (via mixed_mode). - This should be called after adjust_cudagraph_sizes_for_spec_decode. - """ - if ( - not self.speculative_config.enforce_eager - and cudagraph_mode.mixed_mode() - in [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL] - ): - eagle_cudagraph_mode = CUDAGraphMode.PIECEWISE - else: - eagle_cudagraph_mode = CUDAGraphMode.NONE - - self.cudagraph_dispatcher.initialize_cudagraph_keys(eagle_cudagraph_mode) - - def _greedy_sample(self, hidden_states: torch.Tensor) -> torch.Tensor: - """Greedy-sample draft tokens from hidden states.""" - if self.use_local_argmax_reduction: - return self.model.get_top_tokens(hidden_states) - return self.model.compute_logits(hidden_states).argmax(dim=-1) - - def propose( - self, - # [num_tokens] - target_token_ids: torch.Tensor, - # [num_tokens] or [3, num_tokens] when M-RoPE is enabled - target_positions: torch.Tensor, - # [num_tokens, hidden_size] - target_hidden_states: torch.Tensor, - # [batch_size] - next_token_ids: torch.Tensor, - token_indices_to_sample: torch.Tensor | None, - common_attn_metadata: CommonAttentionMetadata, - sampling_metadata: SamplingMetadata, - mm_embed_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, - num_rejected_tokens_gpu: torch.Tensor | None = None, - slot_mappings: dict[str, torch.Tensor] - | list[dict[str, torch.Tensor]] - | None = None, - ) -> torch.Tensor: - batch_size = common_attn_metadata.batch_size() - - if self.method in ("eagle3", "dflash"): - assert isinstance( - self.model, - ( - Eagle3LlamaForCausalLM, - Eagle3DeepseekV2ForCausalLM, - DFlashQwen3ForCausalLM, - ), - ) - target_hidden_states = self.model.combine_hidden_states( - target_hidden_states - ) - assert target_hidden_states.shape[-1] == self.hidden_size - - num_tokens, token_indices_to_sample, common_attn_metadata = ( - self.set_inputs_first_pass( - target_token_ids=target_token_ids, - next_token_ids=next_token_ids, - target_positions=target_positions, - target_hidden_states=target_hidden_states, - token_indices_to_sample=token_indices_to_sample, - cad=common_attn_metadata, - num_rejected_tokens_gpu=num_rejected_tokens_gpu, - ) - ) - - per_group_attn_metadata, per_layer_attn_metadata = ( - self.build_per_group_and_layer_attn_metadata(common_attn_metadata) - ) - - cudagraph_runtime_mode, num_input_tokens, num_tokens_across_dp = ( - self._determine_batch_execution_and_padding(num_tokens) - ) - - model_kwargs, slot_mapping_size = self.build_model_inputs_first_pass( - num_tokens, num_input_tokens, mm_embed_inputs - ) - - with set_forward_context( - per_layer_attn_metadata, - self.vllm_config, - num_tokens=num_input_tokens, - num_tokens_across_dp=num_tokens_across_dp, - cudagraph_runtime_mode=cudagraph_runtime_mode, - slot_mapping=self._get_slot_mapping( - slot_mapping_size, common_attn_metadata.slot_mapping - ), - ): - ret_hidden_states = self.model(**model_kwargs) - if not self.model_returns_tuple(): - last_hidden_states = ret_hidden_states - hidden_states = last_hidden_states - else: - last_hidden_states, hidden_states = ret_hidden_states - - sample_hidden_states = last_hidden_states[token_indices_to_sample] - - # Early exit if there is only one draft token to be generated. - if self.num_speculative_tokens == 1 or self.parallel_drafting: - draft_token_ids = self._greedy_sample(sample_hidden_states) - return draft_token_ids.view(-1, self.num_speculative_tokens) - - if self.uses_mrope: - positions = self.mrope_positions[:, token_indices_to_sample] - else: - positions = self.positions[token_indices_to_sample] - hidden_states = hidden_states[token_indices_to_sample] - - if any(isinstance(md, TreeAttentionMetadata) for md in per_group_attn_metadata): - # Draft using tree attention - requires full logits for top-k - logits = self.model.compute_logits(sample_hidden_states) - draft_token_ids_list = self.propose_tree( - batch_size=batch_size, - logits=logits, - positions=positions, - hidden_states=hidden_states, - common_attn_metadata=common_attn_metadata, - slot_mappings=slot_mappings, - ) - # [batch_size, num_tree_tokens] - return torch.cat(draft_token_ids_list, dim=1) - - draft_token_ids = self._greedy_sample(sample_hidden_states) - - if self.allowed_attn_types is not None: - for group_md in per_group_attn_metadata: - if not isinstance(group_md, self.allowed_attn_types): - raise ValueError( - f"Unsupported attention metadata type for speculative " - "decoding with num_speculative_tokens > 1: " - f"{type(group_md)}. Supported types are: " - f"{self.allowed_attn_types}" - ) - - # Generate the remaining draft tokens. - draft_token_ids_list = [draft_token_ids] - - cudagraph_runtime_mode, input_batch_size, batch_size_across_dp = ( - self._determine_batch_execution_and_padding(batch_size) - ) - - common_attn_metadata.num_actual_tokens = batch_size - common_attn_metadata.max_query_len = 1 - common_attn_metadata.query_start_loc = self.arange[: batch_size + 1] - common_attn_metadata.query_start_loc_cpu = torch.from_numpy( - self.token_arange_np[: batch_size + 1] - ).clone() - - # In padded drafter batch, we need to adjust the sequence lengths - # to remove the "padding" (i.e. rejected tokens). - # Only apply this adjustment when we have rejected tokens - # (i.e., not the first proposal). - if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None: - common_attn_metadata.seq_lens -= num_rejected_tokens_gpu - # Invalidate the CPU-side shadows to avoid H<>D sync. - common_attn_metadata._seq_lens_cpu = None - common_attn_metadata._num_computed_tokens_cpu = None - - block_size = self.block_size - assert block_size > 0, "block_size has not been initialized." - for token_index in range(self.num_speculative_tokens - 1): - # Update the inputs. - # cast to int32 is crucial when eagle model is compiled. - # tensor.argmax() returns int64 by default. - input_ids = draft_token_ids_list[-1].int() - # Use fused kernel for slot mapping and metadata updates. - # Write clamped positions directly into the positions buffer to - # avoid an extra D2D copy for the common (non-mrope) case. - positions_1d = positions[0] if self.uses_mrope else positions - if self.uses_mrope: - out_pos = self.mrope_positions[0, :batch_size] - elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: - out_pos = self.xdrope_positions[0, :batch_size] - else: - out_pos = self.positions[:batch_size] - eagle_step_update_slot_mapping_and_metadata( - positions_1d=positions_1d, - block_table_tensor=common_attn_metadata.block_table_tensor, - seq_lens=common_attn_metadata.seq_lens, - block_size=block_size, - max_model_len=self.max_model_len, - out_clamped_positions=out_pos, - out_slot_mapping=self._slot_mapping_buffer[:input_batch_size], - input_batch_size=input_batch_size, - ) - common_attn_metadata.slot_mapping = self._slot_mapping_buffer[:batch_size] - if self.uses_mrope: - self.mrope_positions[1:, :batch_size] = self.mrope_positions[ - 0, :batch_size - ] - positions = self.mrope_positions[:, :batch_size] - elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: - self.xdrope_positions[1:, :batch_size] = self.xdrope_positions[ - 0, :batch_size - ] - positions = self.xdrope_positions[0, :batch_size] - else: - positions = self.positions[:batch_size] - # Increment the maximum sequence length. We increment max_seq_len - # unconditionally even though some seq_lens may have been capped above, - # as max_seq_len serves as an upper bound for sequence lengths. - common_attn_metadata.max_seq_len = min( - common_attn_metadata.max_seq_len + 1, self.max_model_len - ) - - # Also update the CPU-side shadow; NOTE: this is hacky and should be - # removed in when common_attn_metadata.seq_lens_cpu is deprecated. - if common_attn_metadata._seq_lens_cpu is not None: - common_attn_metadata._seq_lens_cpu += 1 - if common_attn_metadata._num_computed_tokens_cpu is not None: - common_attn_metadata._num_computed_tokens_cpu += 1 - - # Rebuild attention metadata - _, per_layer_attn_metadata = self.build_per_group_and_layer_attn_metadata( - common_attn_metadata, draft_index=token_index + 1 - ) - - # copy inputs to buffer for cudagraph - self.input_ids[:batch_size] = input_ids - self.hidden_states[:batch_size] = hidden_states - if self.supports_mm_inputs: - self.inputs_embeds[:batch_size] = self.model.embed_input_ids(input_ids) - - input_ids = None - inputs_embeds = self.inputs_embeds[:input_batch_size] - else: - input_ids = self.input_ids[:input_batch_size] - inputs_embeds = None - - # Run the model. - model_kwargs = { - "input_ids": input_ids, - "positions": self._get_positions(input_batch_size), - "inputs_embeds": inputs_embeds, - } - if self.pass_hidden_states_to_model: - model_kwargs["hidden_states"] = self.hidden_states[:input_batch_size] - - with set_forward_context( - per_layer_attn_metadata, - self.vllm_config, - num_tokens=input_batch_size, - num_tokens_across_dp=batch_size_across_dp, - cudagraph_runtime_mode=cudagraph_runtime_mode, - slot_mapping=self._get_slot_mapping(input_batch_size), - ): - ret_hidden_states = self.model(**model_kwargs) - if not self.model_returns_tuple(): - last_hidden_states = ret_hidden_states - hidden_states = ret_hidden_states - else: - last_hidden_states, hidden_states = ret_hidden_states - - hidden_states = hidden_states[:batch_size] - draft_token_ids = self._greedy_sample(last_hidden_states[:batch_size]) - draft_token_ids_list.append(draft_token_ids) - - # [batch_size, num_speculative_tokens] - draft_token_ids = torch.stack(draft_token_ids_list, dim=1) - return draft_token_ids - - def set_inputs_first_pass( - self, - target_token_ids: torch.Tensor, - next_token_ids: torch.Tensor, - target_positions: torch.Tensor, - target_hidden_states: torch.Tensor, - token_indices_to_sample: torch.Tensor | None, - cad: CommonAttentionMetadata, - num_rejected_tokens_gpu: torch.Tensor | None, - ) -> tuple[int, torch.Tensor, CommonAttentionMetadata]: - if not self.needs_extra_input_slots: - # Default EAGLE pathway: no reshaping of input tensors needed. - # Simply rotate the input ids and leave the positions unchanged, - # Inserting the next token ids at the last slot in each request. - if token_indices_to_sample is None: - token_indices_to_sample = cad.query_start_loc[1:] - 1 - - num_tokens = target_token_ids.shape[0] - # Shift the input ids by one token. - # E.g., [a1, b1, b2, c1, c2, c3] -> [b1, b2, c1, c2, c3, c3] - self.input_ids[: num_tokens - 1] = target_token_ids[1:] - # Replace the last token with the next token. - # E.g., [b1, b2, c1, c2, c3, c3] -> [a2, b2, b3, c2, c3, c4] - self.input_ids[token_indices_to_sample] = next_token_ids - - # copy inputs to buffer for cudagraph - if self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim == 0: - target_positions = target_positions[0] - self._set_positions(num_tokens, target_positions) - - self.hidden_states[:num_tokens] = target_hidden_states - - return num_tokens, token_indices_to_sample, cad - else: - assert self.is_rejected_token_mask is not None - assert self.is_masked_token_mask is not None - # 1. - # Call a custom triton kernel to copy input_ids and positions - # into the correct slots in the preallocated buffers self.input_ids, - # self.positions. - batch_size = cad.batch_size() - # Since we might have to copy a lot of data for prefills, we select the - # block size based on the max query length and limit to max 256 slots/block. - max_num_tokens_per_request = ( - cad.max_query_len + self.net_num_new_slots_per_request - ) - BLOCK_SIZE_TOKENS = min(256, next_power_of_2(max_num_tokens_per_request)) - num_blocks = ( - max_num_tokens_per_request + BLOCK_SIZE_TOKENS - 1 - ) // BLOCK_SIZE_TOKENS - total_num_input_tokens = target_token_ids.shape[0] - total_num_output_tokens = total_num_input_tokens + ( - self.net_num_new_slots_per_request * batch_size - ) - - token_indices_to_sample = torch.empty( - batch_size * self.extra_slots_per_request, - dtype=torch.int32, - device=self.device, - ) - - # Destination indices to write target_hidden_states into drafting buffer. - out_hidden_state_mapping = torch.empty( - total_num_input_tokens, dtype=torch.int32, device=self.device - ) - - # Kernel grid: one program per request (row) - grid = (batch_size, num_blocks) - query_start_loc = cad.query_start_loc - query_end_loc = cad.query_start_loc[1:] - 1 - if num_rejected_tokens_gpu is not None: - query_end_loc = query_end_loc - num_rejected_tokens_gpu - - copy_and_expand_eagle_inputs_kernel[grid]( - # (Padded) Inputs from the target model - target_token_ids_ptr=target_token_ids, - target_positions_ptr=target_positions, - next_token_ids_ptr=next_token_ids, # sampled tokens, one per request - # Outputs to the drafting buffers - out_input_ids_ptr=self.input_ids, - out_positions_ptr=self.positions, # Doesn't support mrope for now - out_is_rejected_token_mask_ptr=self.is_rejected_token_mask, - out_is_masked_token_mask_ptr=self.is_masked_token_mask, - out_new_token_indices_ptr=token_indices_to_sample, - out_hidden_state_mapping_ptr=out_hidden_state_mapping, - # Input metadata - query_start_loc_ptr=query_start_loc, - query_end_loc_ptr=query_end_loc, - padding_token_id=0, - parallel_drafting_token_id=self.parallel_drafting_token_id, - # Sizing info - # Note that we can deduce batch_size for free from the grid size - total_input_tokens=total_num_input_tokens, - num_padding_slots_per_request=self.extra_slots_per_request, - shift_input_ids=self.pass_hidden_states_to_model, - BLOCK_SIZE_TOKENS=BLOCK_SIZE_TOKENS, - ) - if self.pass_hidden_states_to_model: - assert self.parallel_drafting_hidden_state_tensor is not None - self.hidden_states[out_hidden_state_mapping] = target_hidden_states - # Use torch.where to avoid DtoH sync from boolean indexing - mask = self.is_masked_token_mask[:total_num_output_tokens] - torch.where( - mask.unsqueeze(1), - self.parallel_drafting_hidden_state_tensor, - self.hidden_states[:total_num_output_tokens], - out=self.hidden_states[:total_num_output_tokens], - ) - - # 2. - # Recompute the slot mapping based on the new positions and - # rejection mask. - assert self.block_size > 0, "block_size has not been initialized." - new_slot_mapping = compute_new_slot_mapping( - cad=cad, - new_positions=self.positions[:total_num_output_tokens], - is_rejected_token_mask=self.is_rejected_token_mask[ - :total_num_output_tokens - ], - block_size=self.block_size, - num_new_tokens=self.net_num_new_slots_per_request, - max_model_len=self.max_model_len, - ) - - # 3. Update the common attention metadata with the new (meta)data - new_cad = extend_all_queries_by_N( - cad, - N=self.net_num_new_slots_per_request, - arange=self.arange, - new_slot_mapping=new_slot_mapping, - ) - - return total_num_output_tokens, token_indices_to_sample, new_cad - - def build_model_inputs_first_pass( - self, - num_tokens: int, - num_input_tokens: int, - mm_embed_inputs: tuple[list[torch.Tensor], torch.Tensor] | None, - ) -> tuple[dict[str, Any], int]: - if self.supports_mm_inputs: - mm_embeds, is_mm_embed = mm_embed_inputs or (None, None) - - self.inputs_embeds[:num_tokens] = self.model.embed_input_ids( - self.input_ids[:num_tokens], - multimodal_embeddings=mm_embeds, - is_multimodal=is_mm_embed, - ) - - input_ids = None - inputs_embeds = self.inputs_embeds[:num_input_tokens] - else: - input_ids = self.input_ids[:num_input_tokens] - inputs_embeds = None - - model_kwargs = { - "input_ids": input_ids, - "positions": self._get_positions(num_input_tokens), - "inputs_embeds": inputs_embeds, - } - if self.pass_hidden_states_to_model: - model_kwargs["hidden_states"] = self.hidden_states[:num_input_tokens] - - return model_kwargs, num_input_tokens - - def build_per_group_and_layer_attn_metadata( - self, common_attn_metadata: CommonAttentionMetadata, draft_index: int = 0 - ) -> tuple[list[object], dict[str, object]]: - per_group_attn_metadata: list[object] = [] - per_layer_attn_metadata: dict[str, object] = {} - for attn_group in self.draft_attn_groups: - attn_metadata = attn_group.get_metadata_builder().build_for_drafting( - common_attn_metadata=common_attn_metadata, draft_index=draft_index - ) - per_group_attn_metadata.append(attn_metadata) - for layer_name in attn_group.layer_names: - per_layer_attn_metadata[layer_name] = attn_metadata - return per_group_attn_metadata, per_layer_attn_metadata - - def model_returns_tuple(self) -> bool: - return self.method not in ("mtp", "draft_model", "dflash") - - def prepare_next_token_ids_cpu( - self, - sampled_token_ids: list[list[int]], - requests: dict[str, CachedRequestState], - gpu_input_batch: InputBatch, - num_scheduled_tokens: dict[str, int], - ) -> torch.Tensor: - """ - This function is used to prepare the inputs for speculative decoding. - It calculates the next token ids for each request based on the sampled - token ids from the CPU. If a request has no sampled token ids (e.g., - during the initial decoding steps), it falls back to using the request - state to get the next token id. - """ - req_ids = gpu_input_batch.req_ids - next_token_ids: list[int] = [] - for i, token_ids in enumerate(sampled_token_ids): - if token_ids: - # Common case. - next_token_id = token_ids[-1] - else: - # Partial prefill (rare case). - # Get the next token id from the request state. - req_id = req_ids[i] - req_state = requests[req_id] - seq_len = req_state.num_computed_tokens + num_scheduled_tokens[req_id] - next_token_id = req_state.get_token_id(seq_len) - next_token_ids.append(next_token_id) - next_token_ids = torch.tensor( - next_token_ids, dtype=torch.int32, device=self.input_ids.device - ) - return next_token_ids - - def prepare_next_token_ids_padded( - self, - sampled_token_ids: torch.Tensor, - requests: dict[str, CachedRequestState], - gpu_input_batch: InputBatch, - discard_request_mask: torch.Tensor, - ) -> tuple[torch.Tensor, torch.Tensor]: - """ - This function is used to prepare the inputs for speculative decoding. - It calculates the next token ids and the number of valid sampled tokens - for each request, considering the "discarded" requests whose next token - is not sampled and comes from `request.get_token_id()` instead. This is denoted - the "backup" token id. It also counts rejected tokens via `sampled_token_ids`. - """ - # Precompute get_token_id for when there is no valid next token - num_reqs = gpu_input_batch.num_reqs - seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist() - self.backup_next_token_ids.np[:num_reqs] = np.array( - [ - requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i]) - for i in range(num_reqs) - ], - dtype=np.int32, - ) - self.backup_next_token_ids.copy_to_gpu(num_reqs) - backup_tokens_gpu = self.backup_next_token_ids.gpu - - batch_size, num_tokens = sampled_token_ids.shape - device = sampled_token_ids.device - - assert discard_request_mask.dtype == torch.bool - assert backup_tokens_gpu.dtype == torch.int32 - - next_token_ids = torch.empty(batch_size, dtype=torch.int32, device=device) - valid_sampled_tokens_count = next_token_ids.new_empty(batch_size) - - # Kernel grid: one program per request (row) - grid = (batch_size,) - - # Find the next power of 2 for block sizes - BLOCK_SIZE_TOKENS = next_power_of_2(num_tokens) - eagle_prepare_next_token_padded_kernel[grid]( - sampled_token_ids, - discard_request_mask, - backup_tokens_gpu, - next_token_ids, - valid_sampled_tokens_count, - gpu_input_batch.vocab_size, - num_tokens, - batch_size, - sampled_token_ids.stride(0), - BLOCK_SIZE_TOKENS=BLOCK_SIZE_TOKENS, - ) - - return next_token_ids, valid_sampled_tokens_count - - def prepare_inputs_padded( - self, - common_attn_metadata: CommonAttentionMetadata, - spec_decode_metadata: SpecDecodeMetadata, - valid_sampled_tokens_count: torch.Tensor, - ) -> tuple[CommonAttentionMetadata, torch.Tensor, torch.Tensor]: - """ - This function is used to prepare the inputs for speculative decoding - It updates the common_attn_metadata for speculative decoding, - but does not consider the rejected tokens. Instead, all tokens - are included as inputs to the speculator, with the rejected tokens - used as padding and filtered out later by `token_indices_to_sample`. - No blocking CPU operations should be introduced in this function. - """ - num_reqs = common_attn_metadata.num_reqs - device = valid_sampled_tokens_count.device - - token_indices_to_sample = torch.empty( - (num_reqs,), dtype=torch.int32, device=device - ) - num_rejected_tokens_gpu = torch.empty( - (num_reqs,), dtype=torch.int32, device=device - ) - - grid = (num_reqs,) - eagle_prepare_inputs_padded_kernel[grid]( - spec_decode_metadata.cu_num_draft_tokens, - valid_sampled_tokens_count, - common_attn_metadata.query_start_loc, - token_indices_to_sample, - num_rejected_tokens_gpu, - num_reqs, - ) - - query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu - new_query_len_per_req = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] - - total_num_tokens = query_start_loc_cpu[-1].item() - - spec_common_attn_metadata = CommonAttentionMetadata( - query_start_loc=common_attn_metadata.query_start_loc, - seq_lens=common_attn_metadata.seq_lens, - query_start_loc_cpu=query_start_loc_cpu, - _seq_lens_cpu=common_attn_metadata._seq_lens_cpu, - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, - num_reqs=common_attn_metadata.num_reqs, - num_actual_tokens=total_num_tokens, - max_query_len=new_query_len_per_req.max().item(), - max_seq_len=common_attn_metadata.max_seq_len, - block_table_tensor=common_attn_metadata.block_table_tensor, - slot_mapping=common_attn_metadata.slot_mapping[:total_num_tokens], - causal=True, - dcp_local_seq_lens=common_attn_metadata.dcp_local_seq_lens, - ) - - return ( - spec_common_attn_metadata, - token_indices_to_sample, - num_rejected_tokens_gpu, - ) - - def propose_tree( - self, - batch_size: int, - # [num_tokens, vocab_size] - logits: torch.Tensor, - # [num_tokens] - positions: torch.Tensor, - # [num_tokens, hidden_size] - hidden_states: torch.Tensor, - common_attn_metadata: CommonAttentionMetadata, - slot_mappings: dict[str, torch.Tensor] - | list[dict[str, torch.Tensor]] - | None = None, - ) -> list[torch.Tensor]: - tree_attn_metadata_builder = self.draft_attn_groups[0].get_metadata_builder() - assert isinstance(tree_attn_metadata_builder, TreeAttentionMetadataBuilder) - - total_num_drafts = self.cu_drafts_per_level[0] - level_num_drafts = total_num_drafts - # Sample a draft token for each child at the tree root level. - num_children = self.child_drafts_per_level[0] - if num_children == 1: - draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1) - else: - draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view( - batch_size, -1 - ) - draft_token_ids_list = [draft_token_ids] - draft_hidden_states = hidden_states.view(batch_size, 1, -1) - - # Initialize empty tensors for concatenation with the level outputs. - tree_input_ids = torch.empty( - 0, device=self.input_ids.device, dtype=self.input_ids.dtype - ) - tree_positions = torch.empty( - 0, device=self.positions.device, dtype=self.positions.dtype - ) - tree_hidden_states = torch.empty( - 0, device=self.hidden_states.device, dtype=self.hidden_states.dtype - ) - # Precompute the draft token positions. - flattened_draft_positions = ( - positions.view(batch_size, -1) + self.tree_draft_pos_offsets[:batch_size, :] - ) - tree_depth = len(self.cu_drafts_per_level) - for level in range(tree_depth - 1): - # Get draft positions for RoPE. - draft_positions = positions + (level + 1) - exceeds_max_model_len = (positions + total_num_drafts) >= self.max_model_len - # Mask out the position ids that exceed the max model length. - # Otherwise, we may get out-of-range error in RoPE. - draft_positions = torch.where( - exceeds_max_model_len, - 0, - draft_positions, - ).view(batch_size, -1) - - if level_num_drafts > 1: - # Repeat the positions for each draft at this level. - draft_positions = draft_positions.repeat_interleave( - level_num_drafts, dim=1 - ) - - if num_children > 1: - # Repeat draft hidden states for each child. - draft_hidden_states = draft_hidden_states.repeat_interleave( - num_children, dim=1 - ) - - # Concatenate the draft tokens, positions, and hidden states. - tree_input_ids = torch.cat([tree_input_ids, draft_token_ids], dim=1) - tree_positions = torch.cat([tree_positions, draft_positions], dim=1) - tree_hidden_states = torch.cat( - [tree_hidden_states, draft_hidden_states], dim=1 - ) - - # Build new attention metadata for the next level of drafts. - # This is necessary to support tree attention. - query_len = total_num_drafts - common_attn_metadata = replace( - common_attn_metadata, - query_start_loc=query_len * self.arange[: batch_size + 1], - seq_lens=common_attn_metadata.seq_lens + level_num_drafts, - num_actual_tokens=batch_size * query_len, - max_query_len=query_len, - ) - attn_metadata = tree_attn_metadata_builder.build_for_drafting( - common_attn_metadata=common_attn_metadata, draft_index=level + 1 - ) - - # Apply new attention metadata to all draft layers. - per_layer_attn_metadata = {} - for attn_group in self.draft_attn_groups: - for layer_name in attn_group.layer_names: - per_layer_attn_metadata[layer_name] = attn_metadata - - # Consider max model length. - attn_metadata.max_seq_len = min( - attn_metadata.max_seq_len, self.max_model_len - ) - # For the requests that exceed the max model length, we set the - # sequence length to 1 to minimize their overheads in attention. - attn_metadata.seq_lens.masked_fill_(exceeds_max_model_len, 1) - - # Compute the slot mapping. - block_size = tree_attn_metadata_builder.kv_cache_spec.block_size - query_positions = flattened_draft_positions[:, level : level + query_len] - block_numbers = query_positions // block_size - block_ids = attn_metadata.block_table.gather(dim=1, index=block_numbers) - slot_mapping = block_ids * block_size + query_positions % block_size - # Mask out the slot mappings that exceed the max model length. - # Otherwise, the KV cache will be inadvertently updated with the - # padding tokens. - slot_mapping[exceeds_max_model_len] = PADDING_SLOT_ID - attn_metadata.slot_mapping = slot_mapping.view(-1) - - # Copy inputs to buffer for cudagraph. - num_tokens = attn_metadata.num_actual_tokens - input_ids = tree_input_ids.view(-1) - self.input_ids[:num_tokens] = input_ids - self.positions[:num_tokens] = tree_positions.view(-1) - self.hidden_states[:num_tokens] = tree_hidden_states.view(num_tokens, -1) - - cudagraph_runtime_mode, batch_desc = self.cudagraph_dispatcher.dispatch( - num_tokens - ) - num_input_tokens = batch_desc.num_tokens - # Run the model. - with set_forward_context( - per_layer_attn_metadata, - self.vllm_config, - num_tokens=num_input_tokens, - cudagraph_runtime_mode=cudagraph_runtime_mode, - slot_mapping=self._get_slot_mapping( - num_input_tokens, attn_metadata.slot_mapping - ), - ): - last_hidden_states, hidden_states = self.model( - input_ids=self.input_ids[:num_input_tokens], - positions=self.positions[:num_input_tokens], - hidden_states=self.hidden_states[:num_input_tokens], - inputs_embeds=None, - ) - - # Get the output hidden states for the draft tokens. - draft_hidden_states = hidden_states[:num_tokens].view( - batch_size, query_len, -1 - )[:, -level_num_drafts:] - draft_last_hidden_states = last_hidden_states[:num_tokens].view( - batch_size, query_len, -1 - )[:, -level_num_drafts:] - - # Get the output logits for the draft tokens. - logits = self.model.compute_logits( - draft_last_hidden_states.reshape(batch_size * level_num_drafts, -1) - ) - - # Sample a draft token for each child at the next tree level. - num_children = self.child_drafts_per_level[level + 1] - if num_children == 1: - draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1) - else: - draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view( - batch_size, -1 - ) - draft_token_ids_list.append(draft_token_ids) - - # Update the # drafts counters for the next tree level. - level_num_drafts = self.cu_drafts_per_level[level + 1] - total_num_drafts - total_num_drafts = self.cu_drafts_per_level[level + 1] - return draft_token_ids_list - - def prepare_inputs( - self, - common_attn_metadata: CommonAttentionMetadata, - sampled_token_ids: list[list[int]], - num_draft_tokens: list[int], - ) -> tuple[CommonAttentionMetadata, torch.Tensor]: - """ - This function is used to prepare the inputs for speculative decoding. - It updates to the common_attn_metadata to account for the rejected - tokens (and newly sampled tokens). It also returns the token indices - of the tokens that should be fed to the speculator. - """ - # E.g. - # common_attn_metadata.query_start_loc{_cpu}: - # [0, q1, q1 + q2, q1 + q2 + q3] - # common_attn_metadata.seq_lens{_cpu}: [s1, s2, s3] - # num_rejected_tokens: [n1, n2, n3] - # This function computes the intermediate values: - # num_tokens_per_req: [q1 - n1, q2 - n2, q3 - n3] - # And returns: - # common_attn_metadata.query_start_loc{_cpu}: - # [0, q1 - n1, q1 + q2 - n1 - n2, q1 + q2 + q3 - n1 - n2 - n3] - # common_attn_metadata.seq_lens{_cpu}: - # [s1 - n1 + 1, s2 - n2 + 1, s3 - n3 + 1] - # token_indices: [0, 1, ..., q1 - n1 - 1, - # q1, q1 + 1, ..., q1 + q2 - n2 - 1, - # q1 + q2, q1 + q2 + 1, ..., q1 + q2 + q3 - n3 - 1] - - num_rejected_tokens = [ - n + 1 - len(sampled_token_ids[i]) if n > 0 else 0 - for i, n in enumerate(num_draft_tokens) - ] - num_rejected_tokens = torch.tensor(num_rejected_tokens, dtype=torch.int32) - - device = common_attn_metadata.query_start_loc.device - query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu - new_seq_lens_cpu = common_attn_metadata.seq_lens_cpu - num_rejected_tokens - - # [0, q1, q1 + q2, q1 + q2 + q3] -> [q1, q2, q3] - new_query_len_per_req = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] - # [q1, q2, q3] -> [q1 - n1, q2 - n2, q3 - n3] - new_num_tokens_per_req = new_query_len_per_req - num_rejected_tokens - new_num_tokens_per_req_np = new_num_tokens_per_req.numpy() - - # [q1 - n1, q2 - n2, q3 - n3] -> - # [0, q1 - n1, q1 + q2 - n1 - n2, q1 + q2 + q3 - n1 - n2 - n3] - new_query_start_loc_cpu = torch.zeros( - query_start_loc_cpu.shape, - dtype=torch.int32, - pin_memory=is_pin_memory_available(), - ) - new_query_start_loc_np = new_query_start_loc_cpu.numpy() - np.cumsum(new_num_tokens_per_req_np, out=new_query_start_loc_np[1:]) - - total_num_tokens = new_query_start_loc_np[-1] - # Example assuming num_tokens_per_req_np = [2, 4, 3] - # this implies that `new_query_start_locs` is: - # [0, 2, 6, 9] -> - # [0, 0, 2, 2, 2, 2, 6, 6, 6] - # _r1_ ____r2____ ___r3__ - new_query_start_locs_expanded = np.repeat( - new_query_start_loc_np[:-1], new_num_tokens_per_req_np - ) - # [0, 1, 2, 3, 4, 5, 6, 7, 8] -> - # [0, 1, 0, 1, 2, 3, 0, 1, 2] - # _r1_ ____r2____ ___r3__ - token_offsets = ( - self.token_arange_np[:total_num_tokens] - new_query_start_locs_expanded - ) - - # Expand starting positions to match token pattern - # [0, q1, q1 + q2] -> - # [0, 0, q1, q1, q1, q1, q1 + q2, q1 + q2, q1 + q2] - # _r1_ _____r2_______ ___________r3____________ - old_query_start_locs_expanded = np.repeat( - query_start_loc_cpu[:-1].numpy(), new_num_tokens_per_req_np - ) - # Final token indices are: - # [0, 1, // req 1 - # q1 + 0, q1 + 1, q1 + 2, q1 + 3, // req 2 - # q1 + q2 + 0, q1 + q2 + 1, q1 + q2 + 2] // req 3 - token_indices_np = token_offsets + old_query_start_locs_expanded - token_indices = torch.from_numpy(token_indices_np).to(device, non_blocking=True) - - spec_common_attn_metadata = CommonAttentionMetadata( - query_start_loc=new_query_start_loc_cpu.to(device, non_blocking=True), - seq_lens=new_seq_lens_cpu.to(device, non_blocking=True), - query_start_loc_cpu=new_query_start_loc_cpu, - _seq_lens_cpu=new_seq_lens_cpu, - _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, - num_reqs=common_attn_metadata.num_reqs, - num_actual_tokens=total_num_tokens, - max_query_len=new_query_len_per_req.max().item(), - max_seq_len=new_seq_lens_cpu.max().item(), - block_table_tensor=common_attn_metadata.block_table_tensor, - slot_mapping=common_attn_metadata.slot_mapping[token_indices], - causal=True, - dcp_local_seq_lens=common_attn_metadata.dcp_local_seq_lens, - ) - - return spec_common_attn_metadata, token_indices - - def get_model_name(self, model: nn.Module) -> str: - if hasattr(model, "module"): # multi-GPU - model = model.module - return model.__class__.__name__ - - def _create_draft_vllm_config(self) -> VllmConfig: - """Return a VllmConfig with kernel-level overrides for the proposer. - Subclasses may override to apply additional config changes. - """ - spec_cfg = self.speculative_config - if spec_cfg.moe_backend is not None: - return replace( - self.vllm_config, - kernel_config=replace( - self.vllm_config.kernel_config, - moe_backend=spec_cfg.moe_backend, - ), - ) - return self.vllm_config - - def _get_model(self) -> nn.Module: - """ - Default method to call get_model(). Can be overridden by subclasses which - need to customize model loading. - """ - from vllm.compilation.backends import set_model_tag - - draft_vllm_config = self._create_draft_vllm_config() - with set_model_tag("eagle_head"): - model = get_model( - vllm_config=draft_vllm_config, - model_config=self.speculative_config.draft_model_config, - load_config=self.speculative_config.draft_load_config, - ) - return model - - def load_model(self, target_model: nn.Module) -> None: - target_attn_layer_names = set( - get_layers_from_vllm_config( - self.vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - ).keys() - ) - - self.model = self._get_model() - - # Find draft layers (attention layers added by draft model) - all_attn_layers = get_layers_from_vllm_config( - self.vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - ) - self._draft_attn_layer_names = ( - set(all_attn_layers.keys()) - target_attn_layer_names - ) - - if self.supports_mm_inputs: - # Even if the target model is multimodal, we can also use - # text-only draft models - try: - dummy_input_ids = torch.tensor([[1]], device=self.input_ids.device) - self.model.embed_input_ids(dummy_input_ids, multimodal_embeddings=None) - except (NotImplementedError, AttributeError, TypeError): - logger.warning( - "Draft model does not support multimodal inputs, " - "falling back to text-only mode" - ) - self.supports_mm_inputs = False - - if supports_multimodal(target_model): - # handle multimodality - assert hasattr(target_model, "config") - if self.get_model_name(target_model) in [ - "Exaone4_5_ForConditionalGeneration", - "GlmOcrForConditionalGeneration", - "HunYuanVLForConditionalGeneration", - "Qwen2_5_VLForConditionalGeneration", - "Qwen3_5ForConditionalGeneration", - "Qwen3_5MoeForConditionalGeneration", - "Qwen3VLForConditionalGeneration", - "Qwen3VLMoeForConditionalGeneration", - "Gemma4ForConditionalGeneration", - ]: - self.model.config.image_token_index = target_model.config.image_token_id - elif self.get_model_name(target_model) == "PixtralForConditionalGeneration": - self.model.config.image_token_index = ( - target_model.config.vision_config.image_token_id - ) - elif self.get_model_name(target_model) == "KimiK25ForConditionalGeneration": - self.model.config.image_token_index = ( - target_model.config.media_placeholder_token_id - ) - else: - self.model.config.image_token_index = ( - target_model.config.image_token_index - ) - target_language_model = cast( - SupportsMultiModal, target_model - ).get_language_model() - else: - target_language_model = target_model - - self._maybe_share_embeddings(target_language_model) - self._maybe_share_lm_head(target_language_model) - - if ( - self.parallel_drafting - and self.pass_hidden_states_to_model - and self.parallel_drafting_hidden_state_tensor is not None - ): - flat_mask = self.model.mask_hidden.view(-1) - if self.eagle3_use_aux_hidden_state: - # EAGLE3: mask_hidden stores all aux hidden states, - # project through combine_hidden_states - self.parallel_drafting_hidden_state_tensor.copy_( - self.model.combine_hidden_states(flat_mask) - ) - else: - self.parallel_drafting_hidden_state_tensor.copy_(flat_mask) - - def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None: - """ - Some draft models may not have their own embedding layers, and some may - have a duplicate copy of the target model's embedding layers. In these cases, - we share the target model's embedding layers with the draft model to save - memory. - """ - if get_pp_group().world_size == 1: - inner_model = getattr(target_language_model, "model", None) - if inner_model is None: - raise AttributeError("Target model does not have 'model' attribute") - if hasattr(inner_model, "embed_tokens"): - target_embed_tokens = inner_model.embed_tokens - elif hasattr(inner_model, "embedding"): - target_embed_tokens = inner_model.embedding - else: - raise AttributeError( - "Target model does not have 'embed_tokens' or 'embedding' attribute" - ) - - share_embeddings = False - if hasattr(self.model, "has_own_embed_tokens"): - # EAGLE model - if not self.model.has_own_embed_tokens: - share_embeddings = True - logger.info( - "Detected EAGLE model without its own embed_tokens in the" - " checkpoint. Sharing target model embedding weights with the" - " draft model." - ) - elif ( - isinstance(target_embed_tokens.weight, torch.Tensor) - and isinstance(self.model.model.embed_tokens.weight, torch.Tensor) - # TODO: Offload to CPU for comparison to avoid extra GPU memory - # usage in CI testing environments with limited GPU memory - and torch.equal( - target_embed_tokens.weight.cpu(), - self.model.model.embed_tokens.weight.cpu(), - ) - ): - share_embeddings = True - logger.info( - "Detected EAGLE model with embed_tokens identical to the target" - " model. Sharing target model embedding weights with the draft" - " model." - ) - else: - logger.info( - "Detected EAGLE model with distinct embed_tokens weights. " - "Keeping separate embedding weights from the target model." - ) - else: - # MTP model - share_embeddings = True - logger.info( - "Detected MTP model. " - "Sharing target model embedding weights with the draft model." - ) - - if share_embeddings: - if hasattr(self.model.model, "embed_tokens"): - del self.model.model.embed_tokens - self.model.model.embed_tokens = target_embed_tokens - else: - logger.info( - "The draft model's vocab embedding will be loaded separately" - " from the target model." - ) - - def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None: - """ - Some draft models may not have their own LM head, and some may have a - duplicate copy of the target model's LM head. In these cases, we share - the target model's LM head with the draft model to save memory. - """ - share_lm_head = False - if hasattr(self.model, "has_own_lm_head"): - # EAGLE model - if not self.model.has_own_lm_head: - share_lm_head = True - logger.info( - "Detected EAGLE model without its own lm_head in the checkpoint. " - "Sharing target model lm_head weights with the draft model." - ) - elif ( - hasattr(target_language_model, "lm_head") - and hasattr(target_language_model.lm_head, "weight") - and hasattr(self.model.lm_head, "weight") - and isinstance(target_language_model.lm_head.weight, torch.Tensor) - and isinstance(self.model.lm_head.weight, torch.Tensor) - # TODO: Offload to CPU for comparison to avoid extra GPU memory - # usage in CI testing environments with limited GPU memory - and torch.equal( - target_language_model.lm_head.weight.cpu(), - self.model.lm_head.weight.cpu(), - ) - ): - share_lm_head = True - logger.info( - "Detected EAGLE model with lm_head identical to the target model. " - "Sharing target model lm_head weights with the draft model." - ) - else: - logger.info( - "Detected EAGLE model with distinct lm_head weights. " - "Keeping separate lm_head weights from the target model." - ) - else: - # MTP model - share_lm_head = True - logger.info( - "Detected MTP model. " - "Sharing target model lm_head weights with the draft model." - ) - - if share_lm_head and hasattr(target_language_model, "lm_head"): - if hasattr(self.model, "lm_head"): - del self.model.lm_head - self.model.lm_head = target_language_model.lm_head - - # MTP models call compute_logits via shared_head.head (a - # ParallelLMHead inside each MTP layer), not self.model.lm_head. - # If the checkpoint omits a copy of the lm_head weights at the - # MTP layer path, shared_head.head stays uninitialised and - # produces NaN logits. Always share it explicitly. - inner = getattr(self.model, "model", None) - layers = getattr(inner, "layers", None) if inner else None - if layers is not None: - items = layers.values() if isinstance(layers, nn.ModuleDict) else layers - for layer in items: - sh = getattr(layer, "shared_head", None) - if sh is not None and hasattr(sh, "head"): - del sh.head - sh.head = target_language_model.lm_head - logger.info( - "Shared target model lm_head with MTP shared_head.head." - ) - - if self.use_local_argmax_reduction: - if not hasattr(self.model, "get_top_tokens"): - raise ValueError( - "use_local_argmax_reduction is enabled but draft model " - f"{self.model.__class__.__name__} does not implement " - "get_top_tokens()." - ) - # Warn if draft model has vocab remapping, which forces fallback - # to the full-logits path (negating the optimization). - if ( - hasattr(self.model, "draft_id_to_target_id") - and self.model.draft_id_to_target_id is not None - ): - logger.warning( - "use_local_argmax_reduction is enabled but draft model " - "uses draft_id_to_target_id vocab remapping. The " - "optimization will be bypassed (falling back to full " - "logits gather + argmax)." - ) - else: - logger.info( - "Using local argmax reduction for draft token generation " - "(communication: O(2*tp_size) vs O(vocab_size))." - ) - - @torch.inference_mode() - def dummy_run( - self, - num_tokens: int, - use_cudagraphs: bool = True, - is_graph_capturing: bool = False, - slot_mappings: dict[str, torch.Tensor] | None = None, - ) -> None: - # FIXME: when using tree-based specdec, adjust number of forward-passes - # according to the depth of the tree. - only_one_forward_pass = is_graph_capturing or self.parallel_drafting - for fwd_idx in range( - 1 if only_one_forward_pass else self.num_speculative_tokens - ): - if fwd_idx <= 1: - cudagraph_runtime_mode, num_input_tokens, num_tokens_across_dp = ( - self._determine_batch_execution_and_padding( - num_tokens, use_cudagraphs=use_cudagraphs - ) - ) - - # Make sure to use EAGLE's own buffer during cudagraph capture. - if ( - self._draft_attn_layer_names - and slot_mappings is not None - and next(iter(self._draft_attn_layer_names)) in slot_mappings - ): - slot_mapping_dict = self._get_slot_mapping(num_input_tokens) - else: - slot_mapping_dict = slot_mappings or {} - - with set_forward_context( - None, - self.vllm_config, - num_tokens=num_input_tokens, - num_tokens_across_dp=num_tokens_across_dp, - cudagraph_runtime_mode=cudagraph_runtime_mode, - slot_mapping=slot_mapping_dict, - ): - if self.supports_mm_inputs: - input_ids = None - inputs_embeds = self.inputs_embeds[:num_input_tokens] - else: - input_ids = self.input_ids[:num_input_tokens] - inputs_embeds = None - - kwargs = dict( - input_ids=input_ids, - positions=self._get_positions(num_input_tokens), - inputs_embeds=inputs_embeds, - ) - if self.pass_hidden_states_to_model: - kwargs["hidden_states"] = self.hidden_states[:num_input_tokens] - self.model(**kwargs) - - def _get_eagle3_use_aux_hidden_state_from_config(self) -> bool: - """ - Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary - hidden states and directly uses the last layer output just like eagle1. - They might indicate this by setting "use_aux_hidden_state" to False - inside the "eagle_config" dict of their hf_config. - """ - if self.method != "eagle3": - return False - # Assume that eagle3 heads use aux hidden states by default - use_aux_hidden_state = True - eagle_config = getattr(self.draft_model_config.hf_config, "eagle_config", None) - if eagle_config is not None: - use_aux_hidden_state = eagle_config.get("use_aux_hidden_state", True) - return use_aux_hidden_state - - def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None: - """ - Validate that all drafting layers belong to the same KVCacheGroup. - Need this assumption to ensure all drafting layers can use the - same AttentionMetadata. - May extend to multiple AttentionMetadata in the future. - """ - kv_cache_groups: dict[str, int] = {} - for id, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups): - for layer_name in kv_cache_group.layer_names: - kv_cache_groups[layer_name] = id - assert ( - len( - set( - [ - kv_cache_groups[layer_name] - for layer_name in self._draft_attn_layer_names - ] - ) - ) - == 1 - ), "All drafting layers should belong to the same kv cache group" - - def initialize_attn_backend( - self, - kv_cache_config: KVCacheConfig, - kernel_block_sizes: list[int] | None = None, - ) -> None: - """ - Initialize AttentionGroups for draft layers using kv_cache_config. - Called from the model runner's initialize_metadata_builders. - """ - all_attn_layers = get_layers_from_vllm_config( - self.vllm_config, - AttentionLayerBase, # type: ignore[type-abstract] - ) - - # Find which kv_cache_group the draft layers belong to - self.validate_same_kv_cache_group(kv_cache_config) - kv_cache_spec = None - for gid, group in enumerate(kv_cache_config.kv_cache_groups): - if self._draft_attn_layer_names & set(group.layer_names): - self.kv_cache_gid = gid - kv_cache_spec = group.kv_cache_spec - break - - attention_groups: dict[tuple[str, str], AttentionGroup] = {} - if kv_cache_spec is not None: - for layer_name in self._draft_attn_layer_names: - attn_backend = all_attn_layers[layer_name].get_attn_backend() - backend_key = attn_backend.full_cls_name() - if backend_key not in attention_groups: - layer_kv_cache_spec = kv_cache_spec - if isinstance(layer_kv_cache_spec, UniformTypeKVCacheSpecs): - layer_kv_cache_spec = layer_kv_cache_spec.kv_cache_specs[ - layer_name - ] - - kernel_block_size = ( - kernel_block_sizes[self.kv_cache_gid] - if kernel_block_sizes is not None - and self.kv_cache_gid < len(kernel_block_sizes) - else None - ) - attn_group = AttentionGroup( - backend=attn_backend, - layer_names=[layer_name], - kv_cache_spec=layer_kv_cache_spec, - kv_cache_group_id=self.kv_cache_gid, - ) - attn_group.create_metadata_builders( - self.vllm_config, - self.device, - kernel_block_size=kernel_block_size, - ) - attention_groups[backend_key] = attn_group - else: - attention_groups[backend_key].layer_names.append(layer_name) - - self.draft_attn_groups = list(attention_groups.values()) - self.block_size = ( - self.draft_attn_groups[0].get_metadata_builder().kv_cache_spec.block_size - ) - logger.debug("Using block size %d for drafting layers", self.block_size) - - def _determine_batch_execution_and_padding( - self, - num_tokens: int, - use_cudagraphs: bool = True, - ) -> tuple[CUDAGraphMode, int, torch.Tensor | None]: - cudagraph_mode, batch_desc = self.cudagraph_dispatcher.dispatch( - num_tokens, - valid_modes=({CUDAGraphMode.NONE} if not use_cudagraphs else None), - ) - num_tokens_padded = batch_desc.num_tokens - - # Extra coordination when running data-parallel since we need to - # coordinate across ranks - # TODO(Flechman): support DBO ubatching - should_ubatch, num_tokens_across_dp = False, None - if self.vllm_config.parallel_config.data_parallel_size > 1: - should_ubatch, num_tokens_across_dp, synced_cudagraph_mode = ( - coordinate_batch_across_dp( - num_tokens_unpadded=num_tokens, - parallel_config=self.vllm_config.parallel_config, - allow_microbatching=False, - num_tokens_padded=num_tokens_padded, - cudagraph_mode=cudagraph_mode.value, - ) - ) - assert not should_ubatch, "DBO ubatching not implemented for EAGLE" - - # Extract DP-synced values - if num_tokens_across_dp is not None: - dp_rank = self.dp_rank - num_tokens_padded = int(num_tokens_across_dp[dp_rank].item()) - # Re-dispatch with DP padding so we have the correct - # batch_descriptor - cudagraph_mode, batch_desc = self.cudagraph_dispatcher.dispatch( - num_tokens_padded, - valid_modes={CUDAGraphMode(synced_cudagraph_mode)}, - ) - # Assert to make sure the agreed upon token count is correct - # otherwise num_tokens_across_dp will no-longer be valid - assert batch_desc.num_tokens == num_tokens_padded - num_tokens_across_dp[dp_rank] = num_tokens_padded - - return cudagraph_mode, num_tokens_padded, num_tokens_across_dp +from vllm.config import VllmConfig +from vllm.v1.spec_decode.llm_base_proposer import SpecDecodeBaseProposer class EagleProposer(SpecDecodeBaseProposer): @@ -1745,49 +20,3 @@ class EagleProposer(SpecDecodeBaseProposer): pass_hidden_states_to_model=True, runner=runner, ) - - -# NOTE(woosuk): Currently, the below code is not used and we always use argmax -# to sample the draft tokens. We will use this after we find a way to manage -# the draft prob tensor. -# Refer to https://github.com/vllm-project/vllm/pull/16899 for the details. -# FIXME(woosuk): The logic here is duplicated with the main sampling code. -# We should refactor this to reuse the same sampling implementation. -def compute_probs_and_sample_next_token( - logits: torch.Tensor, - sampling_metadata: SamplingMetadata, -) -> tuple[torch.Tensor, torch.Tensor]: - if sampling_metadata.all_greedy: - # For greedy requests, draft_probs is not used in rejection sampling. - # Therefore, we can just return the logits. - probs = logits - next_token_ids = logits.argmax(dim=-1) - return next_token_ids, probs - - assert sampling_metadata.temperature is not None - - # Use epsilon comparison to detect greedy sampling (temperature ~ 0.0) - # consistent with sampler.py's _SAMPLING_EPS threshold - temperature = sampling_metadata.temperature - # Avoid division by zero if there are greedy requests. - if not sampling_metadata.all_random: - is_greedy = temperature < _SAMPLING_EPS - temperature = torch.where(is_greedy, 1.0, temperature) - logits.div_(temperature.view(-1, 1)) - probs = logits.softmax(dim=-1, dtype=torch.float32) - - # NOTE(woosuk): Currently, we ignore most of the sampling parameters in - # generating the draft tokens. We only use the temperature. While this - # could degrade the acceptance rate, it does not affect the distribution - # of the generated tokens after rejection sampling. - - # TODO(woosuk): Consider seeds. - q = torch.empty_like(probs) - q.exponential_() - # NOTE(woosuk): We shouldn't use `probs.div_(q)` because the draft_probs - # will be used later for rejection sampling. - next_token_ids = probs.div(q).argmax(dim=-1).view(-1) - if not sampling_metadata.all_random: - greedy_token_ids = probs.argmax(dim=-1) - next_token_ids = torch.where(is_greedy, greedy_token_ids, next_token_ids) - return next_token_ids, probs diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py new file mode 100644 index 00000000000..1764ae8db4d --- /dev/null +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -0,0 +1,1778 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import ast +from importlib.util import find_spec +from typing import Any, cast + +import numpy as np +import torch +import torch.nn as nn + +from vllm.config import ( + CUDAGraphMode, + VllmConfig, + get_layers_from_vllm_config, + replace, +) +from vllm.distributed.parallel_state import get_pp_group +from vllm.forward_context import set_forward_context +from vllm.logger import init_logger +from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.model_loader import get_model +from vllm.model_executor.models import supports_multimodal +from vllm.model_executor.models.deepseek_eagle3 import Eagle3DeepseekV2ForCausalLM +from vllm.model_executor.models.interfaces import SupportsMultiModal +from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM +from vllm.model_executor.models.qwen3_dflash import DFlashQwen3ForCausalLM +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.platforms import current_platform +from vllm.utils.platform_utils import is_pin_memory_available +from vllm.v1.attention.backend import CommonAttentionMetadata +from vllm.v1.attention.backends.registry import AttentionBackendEnum +from vllm.v1.attention.backends.tree_attn import ( + TreeAttentionMetadata, + TreeAttentionMetadataBuilder, +) +from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata +from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher +from vllm.v1.kv_cache_interface import KVCacheConfig, UniformTypeKVCacheSpecs +from vllm.v1.sample.metadata import SamplingMetadata +from vllm.v1.sample.sampler import _SAMPLING_EPS +from vllm.v1.spec_decode.metadata import SpecDecodeMetadata +from vllm.v1.spec_decode.utils import ( + PADDING_SLOT_ID, + compute_new_slot_mapping, + copy_and_expand_eagle_inputs_kernel, + eagle_prepare_inputs_padded_kernel, + eagle_prepare_next_token_padded_kernel, + eagle_step_update_slot_mapping_and_metadata, + extend_all_queries_by_N, + next_power_of_2, +) +from vllm.v1.utils import CpuGpuBuffer +from vllm.v1.worker.dp_utils import coordinate_batch_across_dp +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch +from vllm.v1.worker.utils import AttentionGroup + +logger = init_logger(__name__) + + +class SpecDecodeBaseProposer: + def __init__( + self, + vllm_config: VllmConfig, + device: torch.device, + pass_hidden_states_to_model: bool, + runner=None, + ): + self.vllm_config = vllm_config + assert vllm_config.speculative_config is not None + self.speculative_config = vllm_config.speculative_config + self.draft_model_config = self.speculative_config.draft_model_config + self.method = self.speculative_config.method + self.pass_hidden_states_to_model = pass_hidden_states_to_model + + self.device = device + self.dtype = vllm_config.model_config.dtype + self.max_model_len = vllm_config.model_config.max_model_len + self.dp_rank = vllm_config.parallel_config.data_parallel_rank + self.num_speculative_tokens = self.speculative_config.num_speculative_tokens + + # We need to get the hidden size from the draft model config because + # the draft model's hidden size can be different from the target model's + # hidden size (e.g., Llama 3.3 70B). + self.hidden_size = self.draft_model_config.get_hidden_size() + self.inputs_embeds_size = self.draft_model_config.get_inputs_embeds_size() + + # Unifying eagle, draft model, and parallel drafting support. + # DFlash always uses parallel drafting (all tokens in one pass), + # but has an additional slot for the next_token_id (does not shift like EAGLE) + self.parallel_drafting: bool = self.speculative_config.parallel_drafting + self.extra_slots_per_request = ( + 1 if not self.parallel_drafting else self.num_speculative_tokens + ) + self.net_num_new_slots_per_request = self.extra_slots_per_request - ( + 1 if (self.pass_hidden_states_to_model and self.method != "dflash") else 0 + ) + self.needs_extra_input_slots = self.net_num_new_slots_per_request > 0 + + self.parallel_drafting_token_id: int = 0 + self.parallel_drafting_hidden_state_tensor: torch.Tensor | None = None + if self.parallel_drafting: + self._init_parallel_drafting_params() + self.use_local_argmax_reduction: bool = ( + self.speculative_config.use_local_argmax_reduction + ) + + self.max_batch_size = vllm_config.scheduler_config.max_num_seqs + self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens + self.token_arange_np = np.arange(self.max_num_tokens) + + # Can be specialized by methods like DFlash to reduce the limit + self.max_query_tokens = self.max_num_tokens + self.max_positions = self.max_num_tokens + + # Multi-modal data support + self.mm_registry = MULTIMODAL_REGISTRY + self.supports_mm_inputs = self.mm_registry.supports_multimodal_inputs( + vllm_config.model_config + ) + + self.draft_attn_groups: list[AttentionGroup] = [] + self.kv_cache_gid: int = -1 + self.eagle3_use_aux_hidden_state: bool = ( + self._get_eagle3_use_aux_hidden_state_from_config() + ) + + self.compilation_config = self.vllm_config.compilation_config + + # Cudagraph dispatcher for PIECEWISE-only dispatching in eagle. + # Keys are initialized later via initialize_cudagraph_keys() called from + # gpu_model_runner._check_and_update_cudagraph_mode after + # adjust_cudagraph_sizes_for_spec_decode is called. + self.cudagraph_dispatcher = CudagraphDispatcher(self.vllm_config) + + # persistent buffers for cuda graph + self.input_ids = torch.zeros( + self.max_num_tokens, dtype=torch.int32, device=device + ) + # Use draft model's M-RoPE setting, not target model's + # Draft models may be text-only even if target is multimodal + self.uses_mrope = self.draft_model_config.uses_mrope + self.uses_xdrope_dim = self.vllm_config.model_config.uses_xdrope_dim + self.draft_uses_xdrope_dim = self.draft_model_config.uses_xdrope_dim + if self.uses_mrope: + # NOTE: `mrope_positions` is implemented with one additional dummy + # position on purpose to make it non-contiguous so that it can work + # with torch compile. + # See detailed explanation in https://github.com/vllm-project/vllm/pull/12128#discussion_r1926431923 + + # NOTE: When M-RoPE is enabled, position ids are 3D regardless of + # the modality of inputs. For text-only inputs, each dimension has + # identical position IDs, making M-RoPE functionally equivalent to + # 1D-RoPE. + # See page 5 of https://arxiv.org/abs/2409.12191 + self.mrope_positions = torch.zeros( + (3, self.max_positions + 1), dtype=torch.int64, device=device + ) + elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: + self.xdrope_positions = torch.zeros( + (self.uses_xdrope_dim, self.max_positions + 1), + dtype=torch.int64, + device=device, + ) + else: + # RoPE need (max_num_tokens,) + self.positions = torch.zeros( + self.max_positions, + dtype=torch.int64, + device=device, + ) + self.hidden_states = torch.zeros( + (self.max_num_tokens, self.hidden_size), dtype=self.dtype, device=device + ) + + # Will be set when we initialize the attention backend + self.block_size: int = -1 + + # We need +1 here because the arange is used to set query_start_loc, + # which has one more element than batch_size. + max_num_slots_for_arange = max(self.max_batch_size + 1, self.max_num_tokens) + self.arange = torch.arange( + max_num_slots_for_arange, device=device, dtype=torch.int32 + ) + + if self.needs_extra_input_slots: + self._raise_if_padded_drafter_batch_disabled() + self._raise_if_multimodal() + self._raise_if_mrope() + + self.is_rejected_token_mask: torch.Tensor | None = None + self.is_masked_token_mask: torch.Tensor | None = None + if self.needs_extra_input_slots: + # For draft models and parallel drafting, we need to keep track of + # which tokens are rejected to update the slot mapping with padding slots. + self.is_rejected_token_mask = torch.zeros( + (self.max_num_tokens,), dtype=torch.bool, device=device + ) + # For parallel drafting, we also need to keep track of which tokens + # are parallel-padding tokens used to sample at later positions. + # We populate this tensor even when using draft models for simplicity. + self.is_masked_token_mask = torch.zeros( + (self.max_num_tokens,), dtype=torch.bool, device=device + ) + + self.inputs_embeds = torch.zeros( + (self.max_num_tokens, self.inputs_embeds_size), + dtype=self.dtype, + device=device, + ) + + self.backup_next_token_ids = CpuGpuBuffer( + self.max_batch_size, + dtype=torch.int32, + pin_memory=is_pin_memory_available(), + device=device, + with_numpy=True, + ) + + self._slot_mapping_buffer = torch.zeros( + self.max_positions, + dtype=torch.int64, + device=device, + ) + + # Determine allowed attention backends once during initialization. + self.allowed_attn_types: tuple | None = None + if current_platform.is_rocm(): + from vllm.v1.attention.backends.mla.indexer import ( + DeepseekV32IndexerMetadata, + ) + from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import ( + ROCMAiterMLASparseMetadata, + ) + from vllm.v1.attention.backends.rocm_attn import RocmAttentionMetadata + + rocm_types = [ + TritonAttentionMetadata, + RocmAttentionMetadata, + ROCMAiterMLASparseMetadata, + DeepseekV32IndexerMetadata, + ] + # ROCM_AITER_FA is an optional backend + # We check is_enabled() here to avoid importing the backend module during + # auto-discovery when VLLM_ROCM_USE_AITER=0, which would trigger aiter + # import and JIT compilation warnings. Explicit backend selection via + # attention_config still works because the backend module is loaded + # directly when selected, not through this auto-discovery path. + # Check if backend module exists to allow explicit selection + if find_spec( + AttentionBackendEnum.ROCM_AITER_FA.get_path(include_classname=False) + ): + from vllm.v1.attention.backends.rocm_aiter_fa import ( + AiterFlashAttentionMetadata, + ) + + rocm_types.append(AiterFlashAttentionMetadata) + + # TRITON_MLA backend support for MLA models (e.g., DeepSeek) + from vllm.model_executor.layers.attention.mla_attention import ( + MLACommonMetadata, + ) + + rocm_types.append(MLACommonMetadata) + + # FlexAttention backend support + from vllm.v1.attention.backends.flex_attention import FlexAttentionMetadata + + rocm_types.append(FlexAttentionMetadata) + + self.allowed_attn_types = tuple(rocm_types) + + # Parse the speculative token tree. + spec_token_tree = self.speculative_config.speculative_token_tree + assert spec_token_tree is not None + self.tree_choices: list[tuple[int, ...]] = ast.literal_eval(spec_token_tree) + tree_depth = len(self.tree_choices[-1]) + # Precompute per-level properties of the tree. + num_drafts_per_level = [0] * tree_depth + for node in self.tree_choices: + num_drafts_per_level[len(node) - 1] += 1 + self.cu_drafts_per_level = [num_drafts_per_level[0]] + self.child_drafts_per_level = [num_drafts_per_level[0]] + for level in range(1, tree_depth): + self.cu_drafts_per_level.append( + self.cu_drafts_per_level[-1] + num_drafts_per_level[level] + ) + self.child_drafts_per_level.append( + num_drafts_per_level[level] // num_drafts_per_level[level - 1] + ) + # Precompute draft position offsets in flattened tree. + self.tree_draft_pos_offsets = torch.arange( + 1, len(self.tree_choices) + 1, device=device, dtype=torch.int32 + ).repeat(self.max_batch_size, 1) + + def _raise_if_padded_drafter_batch_disabled(self): + if self.speculative_config.disable_padded_drafter_batch: + raise NotImplementedError( + "Speculative Decoding with draft models or parallel drafting only " + "supports padded drafter batch. Please unset " + "disable_padded_drafter_batch in the speculative_config." + ) + + def _raise_if_multimodal(self): + if self.supports_mm_inputs: + raise NotImplementedError( + "Speculative Decoding with draft models or parallel drafting " + "does not support multimodal models yet" + ) + + def _raise_if_mrope(self): + if self.draft_model_config.uses_mrope: + raise NotImplementedError( + "Speculative Decoding with draft models or parallel drafting " + "does not support M-RoPE yet" + ) + + def _init_parallel_drafting_params(self): + # For parallel drafting, we need the token ID to use for masked slots + # And for EAGLE + parallel drafting, we need the hidden state tensor to use + # for those masked slots. + + model_hf_config = self.draft_model_config.hf_config + # DFlash stores mask_token_id in dflash_config + dflash_config = getattr(model_hf_config, "dflash_config", None) + if dflash_config and "mask_token_id" in dflash_config: + self.parallel_drafting_token_id = dflash_config["mask_token_id"] + elif hasattr(model_hf_config, "pard_token"): + self.parallel_drafting_token_id = model_hf_config.pard_token + elif hasattr(model_hf_config, "ptd_token_id"): + self.parallel_drafting_token_id = model_hf_config.ptd_token_id + else: + raise ValueError( + "For parallel drafting, the draft model config must have " + "`pard_token`, `ptd_token_id`, or " + "`dflash_config.mask_token_id` specified in its config.json." + ) + + if self.pass_hidden_states_to_model: + self.parallel_drafting_hidden_state_tensor = torch.empty( + self.hidden_size, dtype=self.dtype, device=self.device + ) + + def _get_positions(self, num_tokens: int): + if self.uses_mrope: + return self.mrope_positions[:, :num_tokens] + if self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: + return self.xdrope_positions[:, :num_tokens] + return self.positions[:num_tokens] + + def _set_positions(self, num_tokens: int, positions: torch.Tensor): + if self.uses_mrope: + self.mrope_positions[:, :num_tokens] = positions + elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: + self.xdrope_positions[:, :num_tokens] = positions + else: + # Convert M-RoPE positions if target model uses M-RoPE + # but draft doesn't, For text inputs, all M-RoPE + # dimensions are identical + if self.vllm_config.model_config.uses_mrope: + positions = positions[0] + self.positions[:num_tokens] = positions + + def _get_slot_mapping( + self, + num_tokens: int, + slot_mapping: torch.Tensor | None = None, + ) -> dict[str, torch.Tensor]: + """Return slot_mapping dict for EAGLE layers. + + If slot_mapping is provided, copies it into the buffer first. + """ + if slot_mapping is not None: + num_actual = slot_mapping.shape[0] + self._slot_mapping_buffer[:num_actual].copy_(slot_mapping) + if num_tokens > num_actual: + self._slot_mapping_buffer[num_actual:num_tokens].fill_(PADDING_SLOT_ID) + + view = self._slot_mapping_buffer[:num_tokens] + return {name: view for name in self._draft_attn_layer_names} + + def initialize_cudagraph_keys(self, cudagraph_mode: CUDAGraphMode) -> None: + """Initialize cudagraph dispatcher keys for eagle. + + Eagle only supports PIECEWISE cudagraphs (via mixed_mode). + This should be called after adjust_cudagraph_sizes_for_spec_decode. + """ + if ( + not self.speculative_config.enforce_eager + and cudagraph_mode.mixed_mode() + in [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL] + ): + eagle_cudagraph_mode = CUDAGraphMode.PIECEWISE + else: + eagle_cudagraph_mode = CUDAGraphMode.NONE + + self.cudagraph_dispatcher.initialize_cudagraph_keys(eagle_cudagraph_mode) + + def _greedy_sample(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Greedy-sample draft tokens from hidden states.""" + if self.use_local_argmax_reduction: + return self.model.get_top_tokens(hidden_states) + return self.model.compute_logits(hidden_states).argmax(dim=-1) + + def propose( + self, + # [num_tokens] + target_token_ids: torch.Tensor, + # [num_tokens] or [3, num_tokens] when M-RoPE is enabled + target_positions: torch.Tensor, + # [num_tokens, hidden_size] + target_hidden_states: torch.Tensor, + # [batch_size] + next_token_ids: torch.Tensor, + token_indices_to_sample: torch.Tensor | None, + common_attn_metadata: CommonAttentionMetadata, + sampling_metadata: SamplingMetadata, + mm_embed_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None, + num_rejected_tokens_gpu: torch.Tensor | None = None, + slot_mappings: dict[str, torch.Tensor] + | list[dict[str, torch.Tensor]] + | None = None, + ) -> torch.Tensor: + batch_size = common_attn_metadata.batch_size() + + if self.method in ("eagle3", "dflash"): + assert isinstance( + self.model, + ( + Eagle3LlamaForCausalLM, + Eagle3DeepseekV2ForCausalLM, + DFlashQwen3ForCausalLM, + ), + ) + target_hidden_states = self.model.combine_hidden_states( + target_hidden_states + ) + assert target_hidden_states.shape[-1] == self.hidden_size + + num_tokens, token_indices_to_sample, common_attn_metadata = ( + self.set_inputs_first_pass( + target_token_ids=target_token_ids, + next_token_ids=next_token_ids, + target_positions=target_positions, + target_hidden_states=target_hidden_states, + token_indices_to_sample=token_indices_to_sample, + cad=common_attn_metadata, + num_rejected_tokens_gpu=num_rejected_tokens_gpu, + ) + ) + + per_group_attn_metadata, per_layer_attn_metadata = ( + self.build_per_group_and_layer_attn_metadata(common_attn_metadata) + ) + + cudagraph_runtime_mode, num_input_tokens, num_tokens_across_dp = ( + self._determine_batch_execution_and_padding(num_tokens) + ) + + model_kwargs, slot_mapping_size = self.build_model_inputs_first_pass( + num_tokens, num_input_tokens, mm_embed_inputs + ) + + with set_forward_context( + per_layer_attn_metadata, + self.vllm_config, + num_tokens=num_input_tokens, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=cudagraph_runtime_mode, + slot_mapping=self._get_slot_mapping( + slot_mapping_size, common_attn_metadata.slot_mapping + ), + ): + ret_hidden_states = self.model(**model_kwargs) + if not self.model_returns_tuple(): + last_hidden_states = ret_hidden_states + hidden_states = last_hidden_states + else: + last_hidden_states, hidden_states = ret_hidden_states + + sample_hidden_states = last_hidden_states[token_indices_to_sample] + + # Early exit if there is only one draft token to be generated. + if self.num_speculative_tokens == 1 or self.parallel_drafting: + draft_token_ids = self._greedy_sample(sample_hidden_states) + return draft_token_ids.view(-1, self.num_speculative_tokens) + + if self.uses_mrope: + positions = self.mrope_positions[:, token_indices_to_sample] + else: + positions = self.positions[token_indices_to_sample] + hidden_states = hidden_states[token_indices_to_sample] + + if any(isinstance(md, TreeAttentionMetadata) for md in per_group_attn_metadata): + # Draft using tree attention - requires full logits for top-k + logits = self.model.compute_logits(sample_hidden_states) + draft_token_ids_list = self.propose_tree( + batch_size=batch_size, + logits=logits, + positions=positions, + hidden_states=hidden_states, + common_attn_metadata=common_attn_metadata, + slot_mappings=slot_mappings, + ) + # [batch_size, num_tree_tokens] + return torch.cat(draft_token_ids_list, dim=1) + + draft_token_ids = self._greedy_sample(sample_hidden_states) + + if self.allowed_attn_types is not None: + for group_md in per_group_attn_metadata: + if not isinstance(group_md, self.allowed_attn_types): + raise ValueError( + f"Unsupported attention metadata type for speculative " + "decoding with num_speculative_tokens > 1: " + f"{type(group_md)}. Supported types are: " + f"{self.allowed_attn_types}" + ) + + # Generate the remaining draft tokens. + draft_token_ids_list = [draft_token_ids] + + cudagraph_runtime_mode, input_batch_size, batch_size_across_dp = ( + self._determine_batch_execution_and_padding(batch_size) + ) + + common_attn_metadata.num_actual_tokens = batch_size + common_attn_metadata.max_query_len = 1 + common_attn_metadata.query_start_loc = self.arange[: batch_size + 1] + common_attn_metadata.query_start_loc_cpu = torch.from_numpy( + self.token_arange_np[: batch_size + 1] + ).clone() + + # In padded drafter batch, we need to adjust the sequence lengths + # to remove the "padding" (i.e. rejected tokens). + # Only apply this adjustment when we have rejected tokens + # (i.e., not the first proposal). + if self.num_speculative_tokens > 1 and num_rejected_tokens_gpu is not None: + common_attn_metadata.seq_lens -= num_rejected_tokens_gpu + # Invalidate the CPU-side shadows to avoid H<>D sync. + common_attn_metadata._seq_lens_cpu = None + common_attn_metadata._num_computed_tokens_cpu = None + + block_size = self.block_size + assert block_size > 0, "block_size has not been initialized." + for token_index in range(self.num_speculative_tokens - 1): + # Update the inputs. + # cast to int32 is crucial when eagle model is compiled. + # tensor.argmax() returns int64 by default. + input_ids = draft_token_ids_list[-1].int() + # Use fused kernel for slot mapping and metadata updates. + # Write clamped positions directly into the positions buffer to + # avoid an extra D2D copy for the common (non-mrope) case. + positions_1d = positions[0] if self.uses_mrope else positions + if self.uses_mrope: + out_pos = self.mrope_positions[0, :batch_size] + elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: + out_pos = self.xdrope_positions[0, :batch_size] + else: + out_pos = self.positions[:batch_size] + eagle_step_update_slot_mapping_and_metadata( + positions_1d=positions_1d, + block_table_tensor=common_attn_metadata.block_table_tensor, + seq_lens=common_attn_metadata.seq_lens, + block_size=block_size, + max_model_len=self.max_model_len, + out_clamped_positions=out_pos, + out_slot_mapping=self._slot_mapping_buffer[:input_batch_size], + input_batch_size=input_batch_size, + ) + common_attn_metadata.slot_mapping = self._slot_mapping_buffer[:batch_size] + if self.uses_mrope: + self.mrope_positions[1:, :batch_size] = self.mrope_positions[ + 0, :batch_size + ] + positions = self.mrope_positions[:, :batch_size] + elif self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim > 0: + self.xdrope_positions[1:, :batch_size] = self.xdrope_positions[ + 0, :batch_size + ] + positions = self.xdrope_positions[0, :batch_size] + else: + positions = self.positions[:batch_size] + # Increment the maximum sequence length. We increment max_seq_len + # unconditionally even though some seq_lens may have been capped above, + # as max_seq_len serves as an upper bound for sequence lengths. + common_attn_metadata.max_seq_len = min( + common_attn_metadata.max_seq_len + 1, self.max_model_len + ) + + # Also update the CPU-side shadow; NOTE: this is hacky and should be + # removed in when common_attn_metadata.seq_lens_cpu is deprecated. + if common_attn_metadata._seq_lens_cpu is not None: + common_attn_metadata._seq_lens_cpu += 1 + if common_attn_metadata._num_computed_tokens_cpu is not None: + common_attn_metadata._num_computed_tokens_cpu += 1 + + # Rebuild attention metadata + _, per_layer_attn_metadata = self.build_per_group_and_layer_attn_metadata( + common_attn_metadata, draft_index=token_index + 1 + ) + + # copy inputs to buffer for cudagraph + self.input_ids[:batch_size] = input_ids + self.hidden_states[:batch_size] = hidden_states + if self.supports_mm_inputs: + self.inputs_embeds[:batch_size] = self.model.embed_input_ids(input_ids) + + input_ids = None + inputs_embeds = self.inputs_embeds[:input_batch_size] + else: + input_ids = self.input_ids[:input_batch_size] + inputs_embeds = None + + # Run the model. + model_kwargs = { + "input_ids": input_ids, + "positions": self._get_positions(input_batch_size), + "inputs_embeds": inputs_embeds, + } + if self.pass_hidden_states_to_model: + model_kwargs["hidden_states"] = self.hidden_states[:input_batch_size] + + with set_forward_context( + per_layer_attn_metadata, + self.vllm_config, + num_tokens=input_batch_size, + num_tokens_across_dp=batch_size_across_dp, + cudagraph_runtime_mode=cudagraph_runtime_mode, + slot_mapping=self._get_slot_mapping(input_batch_size), + ): + ret_hidden_states = self.model(**model_kwargs) + if not self.model_returns_tuple(): + last_hidden_states = ret_hidden_states + hidden_states = ret_hidden_states + else: + last_hidden_states, hidden_states = ret_hidden_states + + hidden_states = hidden_states[:batch_size] + draft_token_ids = self._greedy_sample(last_hidden_states[:batch_size]) + draft_token_ids_list.append(draft_token_ids) + + # [batch_size, num_speculative_tokens] + draft_token_ids = torch.stack(draft_token_ids_list, dim=1) + return draft_token_ids + + def set_inputs_first_pass( + self, + target_token_ids: torch.Tensor, + next_token_ids: torch.Tensor, + target_positions: torch.Tensor, + target_hidden_states: torch.Tensor, + token_indices_to_sample: torch.Tensor | None, + cad: CommonAttentionMetadata, + num_rejected_tokens_gpu: torch.Tensor | None, + ) -> tuple[int, torch.Tensor, CommonAttentionMetadata]: + if not self.needs_extra_input_slots: + # Default EAGLE pathway: no reshaping of input tensors needed. + # Simply rotate the input ids and leave the positions unchanged, + # Inserting the next token ids at the last slot in each request. + if token_indices_to_sample is None: + token_indices_to_sample = cad.query_start_loc[1:] - 1 + + num_tokens = target_token_ids.shape[0] + # Shift the input ids by one token. + # E.g., [a1, b1, b2, c1, c2, c3] -> [b1, b2, c1, c2, c3, c3] + self.input_ids[: num_tokens - 1] = target_token_ids[1:] + # Replace the last token with the next token. + # E.g., [b1, b2, c1, c2, c3, c3] -> [a2, b2, b3, c2, c3, c4] + self.input_ids[token_indices_to_sample] = next_token_ids + + # copy inputs to buffer for cudagraph + if self.uses_xdrope_dim > 0 and self.draft_uses_xdrope_dim == 0: + target_positions = target_positions[0] + self._set_positions(num_tokens, target_positions) + + self.hidden_states[:num_tokens] = target_hidden_states + + return num_tokens, token_indices_to_sample, cad + else: + assert self.is_rejected_token_mask is not None + assert self.is_masked_token_mask is not None + # 1. + # Call a custom triton kernel to copy input_ids and positions + # into the correct slots in the preallocated buffers self.input_ids, + # self.positions. + batch_size = cad.batch_size() + # Since we might have to copy a lot of data for prefills, we select the + # block size based on the max query length and limit to max 256 slots/block. + max_num_tokens_per_request = ( + cad.max_query_len + self.net_num_new_slots_per_request + ) + BLOCK_SIZE_TOKENS = min(256, next_power_of_2(max_num_tokens_per_request)) + num_blocks = ( + max_num_tokens_per_request + BLOCK_SIZE_TOKENS - 1 + ) // BLOCK_SIZE_TOKENS + total_num_input_tokens = target_token_ids.shape[0] + total_num_output_tokens = total_num_input_tokens + ( + self.net_num_new_slots_per_request * batch_size + ) + + token_indices_to_sample = torch.empty( + batch_size * self.extra_slots_per_request, + dtype=torch.int32, + device=self.device, + ) + + # Destination indices to write target_hidden_states into drafting buffer. + out_hidden_state_mapping = torch.empty( + total_num_input_tokens, dtype=torch.int32, device=self.device + ) + + # Kernel grid: one program per request (row) + grid = (batch_size, num_blocks) + query_start_loc = cad.query_start_loc + query_end_loc = cad.query_start_loc[1:] - 1 + if num_rejected_tokens_gpu is not None: + query_end_loc = query_end_loc - num_rejected_tokens_gpu + + copy_and_expand_eagle_inputs_kernel[grid]( + # (Padded) Inputs from the target model + target_token_ids_ptr=target_token_ids, + target_positions_ptr=target_positions, + next_token_ids_ptr=next_token_ids, # sampled tokens, one per request + # Outputs to the drafting buffers + out_input_ids_ptr=self.input_ids, + out_positions_ptr=self.positions, # Doesn't support mrope for now + out_is_rejected_token_mask_ptr=self.is_rejected_token_mask, + out_is_masked_token_mask_ptr=self.is_masked_token_mask, + out_new_token_indices_ptr=token_indices_to_sample, + out_hidden_state_mapping_ptr=out_hidden_state_mapping, + # Input metadata + query_start_loc_ptr=query_start_loc, + query_end_loc_ptr=query_end_loc, + padding_token_id=0, + parallel_drafting_token_id=self.parallel_drafting_token_id, + # Sizing info + # Note that we can deduce batch_size for free from the grid size + total_input_tokens=total_num_input_tokens, + num_padding_slots_per_request=self.extra_slots_per_request, + shift_input_ids=self.pass_hidden_states_to_model, + BLOCK_SIZE_TOKENS=BLOCK_SIZE_TOKENS, + ) + if self.pass_hidden_states_to_model: + assert self.parallel_drafting_hidden_state_tensor is not None + self.hidden_states[out_hidden_state_mapping] = target_hidden_states + # Use torch.where to avoid DtoH sync from boolean indexing + mask = self.is_masked_token_mask[:total_num_output_tokens] + torch.where( + mask.unsqueeze(1), + self.parallel_drafting_hidden_state_tensor, + self.hidden_states[:total_num_output_tokens], + out=self.hidden_states[:total_num_output_tokens], + ) + + # 2. + # Recompute the slot mapping based on the new positions and + # rejection mask. + assert self.block_size > 0, "block_size has not been initialized." + new_slot_mapping = compute_new_slot_mapping( + cad=cad, + new_positions=self.positions[:total_num_output_tokens], + is_rejected_token_mask=self.is_rejected_token_mask[ + :total_num_output_tokens + ], + block_size=self.block_size, + num_new_tokens=self.net_num_new_slots_per_request, + max_model_len=self.max_model_len, + ) + + # 3. Update the common attention metadata with the new (meta)data + new_cad = extend_all_queries_by_N( + cad, + N=self.net_num_new_slots_per_request, + arange=self.arange, + new_slot_mapping=new_slot_mapping, + ) + + return total_num_output_tokens, token_indices_to_sample, new_cad + + def build_model_inputs_first_pass( + self, + num_tokens: int, + num_input_tokens: int, + mm_embed_inputs: tuple[list[torch.Tensor], torch.Tensor] | None, + ) -> tuple[dict[str, Any], int]: + if self.supports_mm_inputs: + mm_embeds, is_mm_embed = mm_embed_inputs or (None, None) + + self.inputs_embeds[:num_tokens] = self.model.embed_input_ids( + self.input_ids[:num_tokens], + multimodal_embeddings=mm_embeds, + is_multimodal=is_mm_embed, + ) + + input_ids = None + inputs_embeds = self.inputs_embeds[:num_input_tokens] + else: + input_ids = self.input_ids[:num_input_tokens] + inputs_embeds = None + + model_kwargs = { + "input_ids": input_ids, + "positions": self._get_positions(num_input_tokens), + "inputs_embeds": inputs_embeds, + } + if self.pass_hidden_states_to_model: + model_kwargs["hidden_states"] = self.hidden_states[:num_input_tokens] + + return model_kwargs, num_input_tokens + + def build_per_group_and_layer_attn_metadata( + self, common_attn_metadata: CommonAttentionMetadata, draft_index: int = 0 + ) -> tuple[list[object], dict[str, object]]: + per_group_attn_metadata: list[object] = [] + per_layer_attn_metadata: dict[str, object] = {} + for attn_group in self.draft_attn_groups: + attn_metadata = attn_group.get_metadata_builder().build_for_drafting( + common_attn_metadata=common_attn_metadata, draft_index=draft_index + ) + per_group_attn_metadata.append(attn_metadata) + for layer_name in attn_group.layer_names: + per_layer_attn_metadata[layer_name] = attn_metadata + return per_group_attn_metadata, per_layer_attn_metadata + + def model_returns_tuple(self) -> bool: + return self.method not in ("mtp", "draft_model", "dflash") + + def prepare_next_token_ids_cpu( + self, + sampled_token_ids: list[list[int]], + requests: dict[str, CachedRequestState], + gpu_input_batch: InputBatch, + num_scheduled_tokens: dict[str, int], + ) -> torch.Tensor: + """ + This function is used to prepare the inputs for speculative decoding. + It calculates the next token ids for each request based on the sampled + token ids from the CPU. If a request has no sampled token ids (e.g., + during the initial decoding steps), it falls back to using the request + state to get the next token id. + """ + req_ids = gpu_input_batch.req_ids + next_token_ids: list[int] = [] + for i, token_ids in enumerate(sampled_token_ids): + if token_ids: + # Common case. + next_token_id = token_ids[-1] + else: + # Partial prefill (rare case). + # Get the next token id from the request state. + req_id = req_ids[i] + req_state = requests[req_id] + seq_len = req_state.num_computed_tokens + num_scheduled_tokens[req_id] + next_token_id = req_state.get_token_id(seq_len) + next_token_ids.append(next_token_id) + next_token_ids = torch.tensor( + next_token_ids, dtype=torch.int32, device=self.input_ids.device + ) + return next_token_ids + + def prepare_next_token_ids_padded( + self, + sampled_token_ids: torch.Tensor, + requests: dict[str, CachedRequestState], + gpu_input_batch: InputBatch, + discard_request_mask: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + """ + This function is used to prepare the inputs for speculative decoding. + It calculates the next token ids and the number of valid sampled tokens + for each request, considering the "discarded" requests whose next token + is not sampled and comes from `request.get_token_id()` instead. This is denoted + the "backup" token id. It also counts rejected tokens via `sampled_token_ids`. + """ + # Precompute get_token_id for when there is no valid next token + num_reqs = gpu_input_batch.num_reqs + seq_lens_list = (gpu_input_batch.num_tokens_no_spec[:num_reqs] - 1).tolist() + self.backup_next_token_ids.np[:num_reqs] = np.array( + [ + requests[gpu_input_batch.req_ids[i]].get_token_id(seq_lens_list[i]) + for i in range(num_reqs) + ], + dtype=np.int32, + ) + self.backup_next_token_ids.copy_to_gpu(num_reqs) + backup_tokens_gpu = self.backup_next_token_ids.gpu + + batch_size, num_tokens = sampled_token_ids.shape + device = sampled_token_ids.device + + assert discard_request_mask.dtype == torch.bool + assert backup_tokens_gpu.dtype == torch.int32 + + next_token_ids = torch.empty(batch_size, dtype=torch.int32, device=device) + valid_sampled_tokens_count = next_token_ids.new_empty(batch_size) + + # Kernel grid: one program per request (row) + grid = (batch_size,) + + # Find the next power of 2 for block sizes + BLOCK_SIZE_TOKENS = next_power_of_2(num_tokens) + eagle_prepare_next_token_padded_kernel[grid]( + sampled_token_ids, + discard_request_mask, + backup_tokens_gpu, + next_token_ids, + valid_sampled_tokens_count, + gpu_input_batch.vocab_size, + num_tokens, + batch_size, + sampled_token_ids.stride(0), + BLOCK_SIZE_TOKENS=BLOCK_SIZE_TOKENS, + ) + + return next_token_ids, valid_sampled_tokens_count + + def prepare_inputs_padded( + self, + common_attn_metadata: CommonAttentionMetadata, + spec_decode_metadata: SpecDecodeMetadata, + valid_sampled_tokens_count: torch.Tensor, + ) -> tuple[CommonAttentionMetadata, torch.Tensor, torch.Tensor]: + """ + This function is used to prepare the inputs for speculative decoding + It updates the common_attn_metadata for speculative decoding, + but does not consider the rejected tokens. Instead, all tokens + are included as inputs to the speculator, with the rejected tokens + used as padding and filtered out later by `token_indices_to_sample`. + No blocking CPU operations should be introduced in this function. + """ + num_reqs = common_attn_metadata.num_reqs + device = valid_sampled_tokens_count.device + + token_indices_to_sample = torch.empty( + (num_reqs,), dtype=torch.int32, device=device + ) + num_rejected_tokens_gpu = torch.empty( + (num_reqs,), dtype=torch.int32, device=device + ) + + grid = (num_reqs,) + eagle_prepare_inputs_padded_kernel[grid]( + spec_decode_metadata.cu_num_draft_tokens, + valid_sampled_tokens_count, + common_attn_metadata.query_start_loc, + token_indices_to_sample, + num_rejected_tokens_gpu, + num_reqs, + ) + + query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu + new_query_len_per_req = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + + total_num_tokens = query_start_loc_cpu[-1].item() + + spec_common_attn_metadata = CommonAttentionMetadata( + query_start_loc=common_attn_metadata.query_start_loc, + seq_lens=common_attn_metadata.seq_lens, + query_start_loc_cpu=query_start_loc_cpu, + _seq_lens_cpu=common_attn_metadata._seq_lens_cpu, + _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, + num_reqs=common_attn_metadata.num_reqs, + num_actual_tokens=total_num_tokens, + max_query_len=new_query_len_per_req.max().item(), + max_seq_len=common_attn_metadata.max_seq_len, + block_table_tensor=common_attn_metadata.block_table_tensor, + slot_mapping=common_attn_metadata.slot_mapping[:total_num_tokens], + causal=True, + dcp_local_seq_lens=common_attn_metadata.dcp_local_seq_lens, + ) + + return ( + spec_common_attn_metadata, + token_indices_to_sample, + num_rejected_tokens_gpu, + ) + + def propose_tree( + self, + batch_size: int, + # [num_tokens, vocab_size] + logits: torch.Tensor, + # [num_tokens] + positions: torch.Tensor, + # [num_tokens, hidden_size] + hidden_states: torch.Tensor, + common_attn_metadata: CommonAttentionMetadata, + slot_mappings: dict[str, torch.Tensor] + | list[dict[str, torch.Tensor]] + | None = None, + ) -> list[torch.Tensor]: + tree_attn_metadata_builder = self.draft_attn_groups[0].get_metadata_builder() + assert isinstance(tree_attn_metadata_builder, TreeAttentionMetadataBuilder) + + total_num_drafts = self.cu_drafts_per_level[0] + level_num_drafts = total_num_drafts + # Sample a draft token for each child at the tree root level. + num_children = self.child_drafts_per_level[0] + if num_children == 1: + draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1) + else: + draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view( + batch_size, -1 + ) + draft_token_ids_list = [draft_token_ids] + draft_hidden_states = hidden_states.view(batch_size, 1, -1) + + # Initialize empty tensors for concatenation with the level outputs. + tree_input_ids = torch.empty( + 0, device=self.input_ids.device, dtype=self.input_ids.dtype + ) + tree_positions = torch.empty( + 0, device=self.positions.device, dtype=self.positions.dtype + ) + tree_hidden_states = torch.empty( + 0, device=self.hidden_states.device, dtype=self.hidden_states.dtype + ) + # Precompute the draft token positions. + flattened_draft_positions = ( + positions.view(batch_size, -1) + self.tree_draft_pos_offsets[:batch_size, :] + ) + tree_depth = len(self.cu_drafts_per_level) + for level in range(tree_depth - 1): + # Get draft positions for RoPE. + draft_positions = positions + (level + 1) + exceeds_max_model_len = (positions + total_num_drafts) >= self.max_model_len + # Mask out the position ids that exceed the max model length. + # Otherwise, we may get out-of-range error in RoPE. + draft_positions = torch.where( + exceeds_max_model_len, + 0, + draft_positions, + ).view(batch_size, -1) + + if level_num_drafts > 1: + # Repeat the positions for each draft at this level. + draft_positions = draft_positions.repeat_interleave( + level_num_drafts, dim=1 + ) + + if num_children > 1: + # Repeat draft hidden states for each child. + draft_hidden_states = draft_hidden_states.repeat_interleave( + num_children, dim=1 + ) + + # Concatenate the draft tokens, positions, and hidden states. + tree_input_ids = torch.cat([tree_input_ids, draft_token_ids], dim=1) + tree_positions = torch.cat([tree_positions, draft_positions], dim=1) + tree_hidden_states = torch.cat( + [tree_hidden_states, draft_hidden_states], dim=1 + ) + + # Build new attention metadata for the next level of drafts. + # This is necessary to support tree attention. + query_len = total_num_drafts + common_attn_metadata = replace( + common_attn_metadata, + query_start_loc=query_len * self.arange[: batch_size + 1], + seq_lens=common_attn_metadata.seq_lens + level_num_drafts, + num_actual_tokens=batch_size * query_len, + max_query_len=query_len, + ) + attn_metadata = tree_attn_metadata_builder.build_for_drafting( + common_attn_metadata=common_attn_metadata, draft_index=level + 1 + ) + + # Apply new attention metadata to all draft layers. + per_layer_attn_metadata = {} + for attn_group in self.draft_attn_groups: + for layer_name in attn_group.layer_names: + per_layer_attn_metadata[layer_name] = attn_metadata + + # Consider max model length. + attn_metadata.max_seq_len = min( + attn_metadata.max_seq_len, self.max_model_len + ) + # For the requests that exceed the max model length, we set the + # sequence length to 1 to minimize their overheads in attention. + attn_metadata.seq_lens.masked_fill_(exceeds_max_model_len, 1) + + # Compute the slot mapping. + block_size = tree_attn_metadata_builder.kv_cache_spec.block_size + query_positions = flattened_draft_positions[:, level : level + query_len] + block_numbers = query_positions // block_size + block_ids = attn_metadata.block_table.gather(dim=1, index=block_numbers) + slot_mapping = block_ids * block_size + query_positions % block_size + # Mask out the slot mappings that exceed the max model length. + # Otherwise, the KV cache will be inadvertently updated with the + # padding tokens. + slot_mapping[exceeds_max_model_len] = PADDING_SLOT_ID + attn_metadata.slot_mapping = slot_mapping.view(-1) + + # Copy inputs to buffer for cudagraph. + num_tokens = attn_metadata.num_actual_tokens + input_ids = tree_input_ids.view(-1) + self.input_ids[:num_tokens] = input_ids + self.positions[:num_tokens] = tree_positions.view(-1) + self.hidden_states[:num_tokens] = tree_hidden_states.view(num_tokens, -1) + + cudagraph_runtime_mode, batch_desc = self.cudagraph_dispatcher.dispatch( + num_tokens + ) + num_input_tokens = batch_desc.num_tokens + # Run the model. + with set_forward_context( + per_layer_attn_metadata, + self.vllm_config, + num_tokens=num_input_tokens, + cudagraph_runtime_mode=cudagraph_runtime_mode, + slot_mapping=self._get_slot_mapping( + num_input_tokens, attn_metadata.slot_mapping + ), + ): + last_hidden_states, hidden_states = self.model( + input_ids=self.input_ids[:num_input_tokens], + positions=self.positions[:num_input_tokens], + hidden_states=self.hidden_states[:num_input_tokens], + inputs_embeds=None, + ) + + # Get the output hidden states for the draft tokens. + draft_hidden_states = hidden_states[:num_tokens].view( + batch_size, query_len, -1 + )[:, -level_num_drafts:] + draft_last_hidden_states = last_hidden_states[:num_tokens].view( + batch_size, query_len, -1 + )[:, -level_num_drafts:] + + # Get the output logits for the draft tokens. + logits = self.model.compute_logits( + draft_last_hidden_states.reshape(batch_size * level_num_drafts, -1) + ) + + # Sample a draft token for each child at the next tree level. + num_children = self.child_drafts_per_level[level + 1] + if num_children == 1: + draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1) + else: + draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view( + batch_size, -1 + ) + draft_token_ids_list.append(draft_token_ids) + + # Update the # drafts counters for the next tree level. + level_num_drafts = self.cu_drafts_per_level[level + 1] - total_num_drafts + total_num_drafts = self.cu_drafts_per_level[level + 1] + return draft_token_ids_list + + def prepare_inputs( + self, + common_attn_metadata: CommonAttentionMetadata, + sampled_token_ids: list[list[int]], + num_draft_tokens: list[int], + ) -> tuple[CommonAttentionMetadata, torch.Tensor]: + """ + This function is used to prepare the inputs for speculative decoding. + It updates to the common_attn_metadata to account for the rejected + tokens (and newly sampled tokens). It also returns the token indices + of the tokens that should be fed to the speculator. + """ + # E.g. + # common_attn_metadata.query_start_loc{_cpu}: + # [0, q1, q1 + q2, q1 + q2 + q3] + # common_attn_metadata.seq_lens{_cpu}: [s1, s2, s3] + # num_rejected_tokens: [n1, n2, n3] + # This function computes the intermediate values: + # num_tokens_per_req: [q1 - n1, q2 - n2, q3 - n3] + # And returns: + # common_attn_metadata.query_start_loc{_cpu}: + # [0, q1 - n1, q1 + q2 - n1 - n2, q1 + q2 + q3 - n1 - n2 - n3] + # common_attn_metadata.seq_lens{_cpu}: + # [s1 - n1 + 1, s2 - n2 + 1, s3 - n3 + 1] + # token_indices: [0, 1, ..., q1 - n1 - 1, + # q1, q1 + 1, ..., q1 + q2 - n2 - 1, + # q1 + q2, q1 + q2 + 1, ..., q1 + q2 + q3 - n3 - 1] + + num_rejected_tokens = [ + n + 1 - len(sampled_token_ids[i]) if n > 0 else 0 + for i, n in enumerate(num_draft_tokens) + ] + num_rejected_tokens = torch.tensor(num_rejected_tokens, dtype=torch.int32) + + device = common_attn_metadata.query_start_loc.device + query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu + new_seq_lens_cpu = common_attn_metadata.seq_lens_cpu - num_rejected_tokens + + # [0, q1, q1 + q2, q1 + q2 + q3] -> [q1, q2, q3] + new_query_len_per_req = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] + # [q1, q2, q3] -> [q1 - n1, q2 - n2, q3 - n3] + new_num_tokens_per_req = new_query_len_per_req - num_rejected_tokens + new_num_tokens_per_req_np = new_num_tokens_per_req.numpy() + + # [q1 - n1, q2 - n2, q3 - n3] -> + # [0, q1 - n1, q1 + q2 - n1 - n2, q1 + q2 + q3 - n1 - n2 - n3] + new_query_start_loc_cpu = torch.zeros( + query_start_loc_cpu.shape, + dtype=torch.int32, + pin_memory=is_pin_memory_available(), + ) + new_query_start_loc_np = new_query_start_loc_cpu.numpy() + np.cumsum(new_num_tokens_per_req_np, out=new_query_start_loc_np[1:]) + + total_num_tokens = new_query_start_loc_np[-1] + # Example assuming num_tokens_per_req_np = [2, 4, 3] + # this implies that `new_query_start_locs` is: + # [0, 2, 6, 9] -> + # [0, 0, 2, 2, 2, 2, 6, 6, 6] + # _r1_ ____r2____ ___r3__ + new_query_start_locs_expanded = np.repeat( + new_query_start_loc_np[:-1], new_num_tokens_per_req_np + ) + # [0, 1, 2, 3, 4, 5, 6, 7, 8] -> + # [0, 1, 0, 1, 2, 3, 0, 1, 2] + # _r1_ ____r2____ ___r3__ + token_offsets = ( + self.token_arange_np[:total_num_tokens] - new_query_start_locs_expanded + ) + + # Expand starting positions to match token pattern + # [0, q1, q1 + q2] -> + # [0, 0, q1, q1, q1, q1, q1 + q2, q1 + q2, q1 + q2] + # _r1_ _____r2_______ ___________r3____________ + old_query_start_locs_expanded = np.repeat( + query_start_loc_cpu[:-1].numpy(), new_num_tokens_per_req_np + ) + # Final token indices are: + # [0, 1, // req 1 + # q1 + 0, q1 + 1, q1 + 2, q1 + 3, // req 2 + # q1 + q2 + 0, q1 + q2 + 1, q1 + q2 + 2] // req 3 + token_indices_np = token_offsets + old_query_start_locs_expanded + token_indices = torch.from_numpy(token_indices_np).to(device, non_blocking=True) + + spec_common_attn_metadata = CommonAttentionMetadata( + query_start_loc=new_query_start_loc_cpu.to(device, non_blocking=True), + seq_lens=new_seq_lens_cpu.to(device, non_blocking=True), + query_start_loc_cpu=new_query_start_loc_cpu, + _seq_lens_cpu=new_seq_lens_cpu, + _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, + num_reqs=common_attn_metadata.num_reqs, + num_actual_tokens=total_num_tokens, + max_query_len=new_query_len_per_req.max().item(), + max_seq_len=new_seq_lens_cpu.max().item(), + block_table_tensor=common_attn_metadata.block_table_tensor, + slot_mapping=common_attn_metadata.slot_mapping[token_indices], + causal=True, + dcp_local_seq_lens=common_attn_metadata.dcp_local_seq_lens, + ) + + return spec_common_attn_metadata, token_indices + + def get_model_name(self, model: nn.Module) -> str: + if hasattr(model, "module"): # multi-GPU + model = model.module + return model.__class__.__name__ + + def _create_draft_vllm_config(self) -> VllmConfig: + """Return a VllmConfig with kernel-level overrides for the proposer. + Subclasses may override to apply additional config changes. + """ + spec_cfg = self.speculative_config + if spec_cfg.moe_backend is not None: + return replace( + self.vllm_config, + kernel_config=replace( + self.vllm_config.kernel_config, + moe_backend=spec_cfg.moe_backend, + ), + ) + return self.vllm_config + + def _get_model(self) -> nn.Module: + """ + Default method to call get_model(). Can be overridden by subclasses which + need to customize model loading. + """ + from vllm.compilation.backends import set_model_tag + + draft_vllm_config = self._create_draft_vllm_config() + with set_model_tag("eagle_head"): + model = get_model( + vllm_config=draft_vllm_config, + model_config=self.speculative_config.draft_model_config, + load_config=self.speculative_config.draft_load_config, + ) + return model + + def load_model(self, target_model: nn.Module) -> None: + target_attn_layer_names = set( + get_layers_from_vllm_config( + self.vllm_config, + AttentionLayerBase, # type: ignore[type-abstract] + ).keys() + ) + + self.model = self._get_model() + + # Find draft layers (attention layers added by draft model) + all_attn_layers = get_layers_from_vllm_config( + self.vllm_config, + AttentionLayerBase, # type: ignore[type-abstract] + ) + self._draft_attn_layer_names = ( + set(all_attn_layers.keys()) - target_attn_layer_names + ) + + if self.supports_mm_inputs: + # Even if the target model is multimodal, we can also use + # text-only draft models + try: + dummy_input_ids = torch.tensor([[1]], device=self.input_ids.device) + self.model.embed_input_ids(dummy_input_ids, multimodal_embeddings=None) + except (NotImplementedError, AttributeError, TypeError): + logger.warning( + "Draft model does not support multimodal inputs, " + "falling back to text-only mode" + ) + self.supports_mm_inputs = False + + if supports_multimodal(target_model): + # handle multimodality + assert hasattr(target_model, "config") + if self.get_model_name(target_model) in [ + "Exaone4_5_ForConditionalGeneration", + "GlmOcrForConditionalGeneration", + "HunYuanVLForConditionalGeneration", + "Qwen2_5_VLForConditionalGeneration", + "Qwen3_5ForConditionalGeneration", + "Qwen3_5MoeForConditionalGeneration", + "Qwen3VLForConditionalGeneration", + "Qwen3VLMoeForConditionalGeneration", + "Gemma4ForConditionalGeneration", + ]: + self.model.config.image_token_index = target_model.config.image_token_id + elif self.get_model_name(target_model) == "PixtralForConditionalGeneration": + self.model.config.image_token_index = ( + target_model.config.vision_config.image_token_id + ) + elif self.get_model_name(target_model) == "KimiK25ForConditionalGeneration": + self.model.config.image_token_index = ( + target_model.config.media_placeholder_token_id + ) + else: + self.model.config.image_token_index = ( + target_model.config.image_token_index + ) + target_language_model = cast( + SupportsMultiModal, target_model + ).get_language_model() + else: + target_language_model = target_model + + self._maybe_share_embeddings(target_language_model) + self._maybe_share_lm_head(target_language_model) + + if ( + self.parallel_drafting + and self.pass_hidden_states_to_model + and self.parallel_drafting_hidden_state_tensor is not None + ): + flat_mask = self.model.mask_hidden.view(-1) + if self.eagle3_use_aux_hidden_state: + # EAGLE3: mask_hidden stores all aux hidden states, + # project through combine_hidden_states + self.parallel_drafting_hidden_state_tensor.copy_( + self.model.combine_hidden_states(flat_mask) + ) + else: + self.parallel_drafting_hidden_state_tensor.copy_(flat_mask) + + def _maybe_share_embeddings(self, target_language_model: nn.Module) -> None: + """ + Some draft models may not have their own embedding layers, and some may + have a duplicate copy of the target model's embedding layers. In these cases, + we share the target model's embedding layers with the draft model to save + memory. + """ + if get_pp_group().world_size == 1: + inner_model = getattr(target_language_model, "model", None) + if inner_model is None: + raise AttributeError("Target model does not have 'model' attribute") + if hasattr(inner_model, "embed_tokens"): + target_embed_tokens = inner_model.embed_tokens + elif hasattr(inner_model, "embedding"): + target_embed_tokens = inner_model.embedding + else: + raise AttributeError( + "Target model does not have 'embed_tokens' or 'embedding' attribute" + ) + + share_embeddings = False + if hasattr(self.model, "has_own_embed_tokens"): + # EAGLE model + if not self.model.has_own_embed_tokens: + share_embeddings = True + logger.info( + "Detected EAGLE model without its own embed_tokens in the" + " checkpoint. Sharing target model embedding weights with the" + " draft model." + ) + elif ( + isinstance(target_embed_tokens.weight, torch.Tensor) + and isinstance(self.model.model.embed_tokens.weight, torch.Tensor) + # TODO: Offload to CPU for comparison to avoid extra GPU memory + # usage in CI testing environments with limited GPU memory + and torch.equal( + target_embed_tokens.weight.cpu(), + self.model.model.embed_tokens.weight.cpu(), + ) + ): + share_embeddings = True + logger.info( + "Detected EAGLE model with embed_tokens identical to the target" + " model. Sharing target model embedding weights with the draft" + " model." + ) + else: + logger.info( + "Detected EAGLE model with distinct embed_tokens weights. " + "Keeping separate embedding weights from the target model." + ) + else: + # MTP model + share_embeddings = True + logger.info( + "Detected MTP model. " + "Sharing target model embedding weights with the draft model." + ) + + if share_embeddings: + if hasattr(self.model.model, "embed_tokens"): + del self.model.model.embed_tokens + self.model.model.embed_tokens = target_embed_tokens + else: + logger.info( + "The draft model's vocab embedding will be loaded separately" + " from the target model." + ) + + def _maybe_share_lm_head(self, target_language_model: nn.Module) -> None: + """ + Some draft models may not have their own LM head, and some may have a + duplicate copy of the target model's LM head. In these cases, we share + the target model's LM head with the draft model to save memory. + """ + share_lm_head = False + if hasattr(self.model, "has_own_lm_head"): + # EAGLE model + if not self.model.has_own_lm_head: + share_lm_head = True + logger.info( + "Detected EAGLE model without its own lm_head in the checkpoint. " + "Sharing target model lm_head weights with the draft model." + ) + elif ( + hasattr(target_language_model, "lm_head") + and hasattr(target_language_model.lm_head, "weight") + and hasattr(self.model.lm_head, "weight") + and isinstance(target_language_model.lm_head.weight, torch.Tensor) + and isinstance(self.model.lm_head.weight, torch.Tensor) + # TODO: Offload to CPU for comparison to avoid extra GPU memory + # usage in CI testing environments with limited GPU memory + and torch.equal( + target_language_model.lm_head.weight.cpu(), + self.model.lm_head.weight.cpu(), + ) + ): + share_lm_head = True + logger.info( + "Detected EAGLE model with lm_head identical to the target model. " + "Sharing target model lm_head weights with the draft model." + ) + else: + logger.info( + "Detected EAGLE model with distinct lm_head weights. " + "Keeping separate lm_head weights from the target model." + ) + else: + # MTP model + share_lm_head = True + logger.info( + "Detected MTP model. " + "Sharing target model lm_head weights with the draft model." + ) + + if share_lm_head and hasattr(target_language_model, "lm_head"): + if hasattr(self.model, "lm_head"): + del self.model.lm_head + self.model.lm_head = target_language_model.lm_head + + # MTP models call compute_logits via shared_head.head (a + # ParallelLMHead inside each MTP layer), not self.model.lm_head. + # If the checkpoint omits a copy of the lm_head weights at the + # MTP layer path, shared_head.head stays uninitialised and + # produces NaN logits. Always share it explicitly. + inner = getattr(self.model, "model", None) + layers = getattr(inner, "layers", None) if inner else None + if layers is not None: + items = layers.values() if isinstance(layers, nn.ModuleDict) else layers + for layer in items: + sh = getattr(layer, "shared_head", None) + if sh is not None and hasattr(sh, "head"): + del sh.head + sh.head = target_language_model.lm_head + logger.info( + "Shared target model lm_head with MTP shared_head.head." + ) + + if self.use_local_argmax_reduction: + if not hasattr(self.model, "get_top_tokens"): + raise ValueError( + "use_local_argmax_reduction is enabled but draft model " + f"{self.model.__class__.__name__} does not implement " + "get_top_tokens()." + ) + # Warn if draft model has vocab remapping, which forces fallback + # to the full-logits path (negating the optimization). + if ( + hasattr(self.model, "draft_id_to_target_id") + and self.model.draft_id_to_target_id is not None + ): + logger.warning( + "use_local_argmax_reduction is enabled but draft model " + "uses draft_id_to_target_id vocab remapping. The " + "optimization will be bypassed (falling back to full " + "logits gather + argmax)." + ) + else: + logger.info( + "Using local argmax reduction for draft token generation " + "(communication: O(2*tp_size) vs O(vocab_size))." + ) + + @torch.inference_mode() + def dummy_run( + self, + num_tokens: int, + use_cudagraphs: bool = True, + is_graph_capturing: bool = False, + slot_mappings: dict[str, torch.Tensor] | None = None, + ) -> None: + # FIXME: when using tree-based specdec, adjust number of forward-passes + # according to the depth of the tree. + only_one_forward_pass = is_graph_capturing or self.parallel_drafting + for fwd_idx in range( + 1 if only_one_forward_pass else self.num_speculative_tokens + ): + if fwd_idx <= 1: + cudagraph_runtime_mode, num_input_tokens, num_tokens_across_dp = ( + self._determine_batch_execution_and_padding( + num_tokens, use_cudagraphs=use_cudagraphs + ) + ) + + # Make sure to use EAGLE's own buffer during cudagraph capture. + if ( + self._draft_attn_layer_names + and slot_mappings is not None + and next(iter(self._draft_attn_layer_names)) in slot_mappings + ): + slot_mapping_dict = self._get_slot_mapping(num_input_tokens) + else: + slot_mapping_dict = slot_mappings or {} + + with set_forward_context( + None, + self.vllm_config, + num_tokens=num_input_tokens, + num_tokens_across_dp=num_tokens_across_dp, + cudagraph_runtime_mode=cudagraph_runtime_mode, + slot_mapping=slot_mapping_dict, + ): + if self.supports_mm_inputs: + input_ids = None + inputs_embeds = self.inputs_embeds[:num_input_tokens] + else: + input_ids = self.input_ids[:num_input_tokens] + inputs_embeds = None + + kwargs = dict( + input_ids=input_ids, + positions=self._get_positions(num_input_tokens), + inputs_embeds=inputs_embeds, + ) + if self.pass_hidden_states_to_model: + kwargs["hidden_states"] = self.hidden_states[:num_input_tokens] + self.model(**kwargs) + + def _get_eagle3_use_aux_hidden_state_from_config(self) -> bool: + """ + Some eagle3 heads (e.g., nvidia/gpt-oss-120b-Eagle3-v2) do not use auxiliary + hidden states and directly uses the last layer output just like eagle1. + They might indicate this by setting "use_aux_hidden_state" to False + inside the "eagle_config" dict of their hf_config. + """ + if self.method != "eagle3": + return False + # Assume that eagle3 heads use aux hidden states by default + use_aux_hidden_state = True + eagle_config = getattr(self.draft_model_config.hf_config, "eagle_config", None) + if eagle_config is not None: + use_aux_hidden_state = eagle_config.get("use_aux_hidden_state", True) + return use_aux_hidden_state + + def validate_same_kv_cache_group(self, kv_cache_config: KVCacheConfig) -> None: + """ + Validate that all drafting layers belong to the same KVCacheGroup. + Need this assumption to ensure all drafting layers can use the + same AttentionMetadata. + May extend to multiple AttentionMetadata in the future. + """ + kv_cache_groups: dict[str, int] = {} + for id, kv_cache_group in enumerate(kv_cache_config.kv_cache_groups): + for layer_name in kv_cache_group.layer_names: + kv_cache_groups[layer_name] = id + assert ( + len( + set( + [ + kv_cache_groups[layer_name] + for layer_name in self._draft_attn_layer_names + ] + ) + ) + == 1 + ), "All drafting layers should belong to the same kv cache group" + + def initialize_attn_backend( + self, + kv_cache_config: KVCacheConfig, + kernel_block_sizes: list[int] | None = None, + ) -> None: + """ + Initialize AttentionGroups for draft layers using kv_cache_config. + Called from the model runner's initialize_metadata_builders. + """ + all_attn_layers = get_layers_from_vllm_config( + self.vllm_config, + AttentionLayerBase, # type: ignore[type-abstract] + ) + + # Find which kv_cache_group the draft layers belong to + self.validate_same_kv_cache_group(kv_cache_config) + kv_cache_spec = None + for gid, group in enumerate(kv_cache_config.kv_cache_groups): + if self._draft_attn_layer_names & set(group.layer_names): + self.kv_cache_gid = gid + kv_cache_spec = group.kv_cache_spec + break + + attention_groups: dict[tuple[str, str], AttentionGroup] = {} + if kv_cache_spec is not None: + for layer_name in self._draft_attn_layer_names: + attn_backend = all_attn_layers[layer_name].get_attn_backend() + backend_key = attn_backend.full_cls_name() + if backend_key not in attention_groups: + layer_kv_cache_spec = kv_cache_spec + if isinstance(layer_kv_cache_spec, UniformTypeKVCacheSpecs): + layer_kv_cache_spec = layer_kv_cache_spec.kv_cache_specs[ + layer_name + ] + + kernel_block_size = ( + kernel_block_sizes[self.kv_cache_gid] + if kernel_block_sizes is not None + and self.kv_cache_gid < len(kernel_block_sizes) + else None + ) + attn_group = AttentionGroup( + backend=attn_backend, + layer_names=[layer_name], + kv_cache_spec=layer_kv_cache_spec, + kv_cache_group_id=self.kv_cache_gid, + ) + attn_group.create_metadata_builders( + self.vllm_config, + self.device, + kernel_block_size=kernel_block_size, + ) + attention_groups[backend_key] = attn_group + else: + attention_groups[backend_key].layer_names.append(layer_name) + + self.draft_attn_groups = list(attention_groups.values()) + self.block_size = ( + self.draft_attn_groups[0].get_metadata_builder().kv_cache_spec.block_size + ) + logger.debug("Using block size %d for drafting layers", self.block_size) + + def _determine_batch_execution_and_padding( + self, + num_tokens: int, + use_cudagraphs: bool = True, + ) -> tuple[CUDAGraphMode, int, torch.Tensor | None]: + cudagraph_mode, batch_desc = self.cudagraph_dispatcher.dispatch( + num_tokens, + valid_modes=({CUDAGraphMode.NONE} if not use_cudagraphs else None), + ) + num_tokens_padded = batch_desc.num_tokens + + # Extra coordination when running data-parallel since we need to + # coordinate across ranks + # TODO(Flechman): support DBO ubatching + should_ubatch, num_tokens_across_dp = False, None + if self.vllm_config.parallel_config.data_parallel_size > 1: + should_ubatch, num_tokens_across_dp, synced_cudagraph_mode = ( + coordinate_batch_across_dp( + num_tokens_unpadded=num_tokens, + parallel_config=self.vllm_config.parallel_config, + allow_microbatching=False, + num_tokens_padded=num_tokens_padded, + cudagraph_mode=cudagraph_mode.value, + ) + ) + assert not should_ubatch, "DBO ubatching not implemented for EAGLE" + + # Extract DP-synced values + if num_tokens_across_dp is not None: + dp_rank = self.dp_rank + num_tokens_padded = int(num_tokens_across_dp[dp_rank].item()) + # Re-dispatch with DP padding so we have the correct + # batch_descriptor + cudagraph_mode, batch_desc = self.cudagraph_dispatcher.dispatch( + num_tokens_padded, + valid_modes={CUDAGraphMode(synced_cudagraph_mode)}, + ) + # Assert to make sure the agreed upon token count is correct + # otherwise num_tokens_across_dp will no-longer be valid + assert batch_desc.num_tokens == num_tokens_padded + num_tokens_across_dp[dp_rank] = num_tokens_padded + + return cudagraph_mode, num_tokens_padded, num_tokens_across_dp + + +# NOTE(woosuk): Currently, the below code is not used and we always use argmax +# to sample the draft tokens. We will use this after we find a way to manage +# the draft prob tensor. +# Refer to https://github.com/vllm-project/vllm/pull/16899 for the details. +# FIXME(woosuk): The logic here is duplicated with the main sampling code. +# We should refactor this to reuse the same sampling implementation. +def compute_probs_and_sample_next_token( + logits: torch.Tensor, + sampling_metadata: SamplingMetadata, +) -> tuple[torch.Tensor, torch.Tensor]: + if sampling_metadata.all_greedy: + # For greedy requests, draft_probs is not used in rejection sampling. + # Therefore, we can just return the logits. + probs = logits + next_token_ids = logits.argmax(dim=-1) + return next_token_ids, probs + + assert sampling_metadata.temperature is not None + + # Use epsilon comparison to detect greedy sampling (temperature ~ 0.0) + # consistent with sampler.py's _SAMPLING_EPS threshold + temperature = sampling_metadata.temperature + # Avoid division by zero if there are greedy requests. + if not sampling_metadata.all_random: + is_greedy = temperature < _SAMPLING_EPS + temperature = torch.where(is_greedy, 1.0, temperature) + logits.div_(temperature.view(-1, 1)) + probs = logits.softmax(dim=-1, dtype=torch.float32) + + # NOTE(woosuk): Currently, we ignore most of the sampling parameters in + # generating the draft tokens. We only use the temperature. While this + # could degrade the acceptance rate, it does not affect the distribution + # of the generated tokens after rejection sampling. + + # TODO(woosuk): Consider seeds. + q = torch.empty_like(probs) + q.exponential_() + # NOTE(woosuk): We shouldn't use `probs.div_(q)` because the draft_probs + # will be used later for rejection sampling. + next_token_ids = probs.div(q).argmax(dim=-1).view(-1) + if not sampling_metadata.all_random: + greedy_token_ids = probs.argmax(dim=-1) + next_token_ids = torch.where(is_greedy, greedy_token_ids, next_token_ids) + return next_token_ids, probs diff --git a/vllm/v1/worker/cpu_model_runner.py b/vllm/v1/worker/cpu_model_runner.py index d7c52d58a5e..61fe44d251b 100644 --- a/vllm/v1/worker/cpu_model_runner.py +++ b/vllm/v1/worker/cpu_model_runner.py @@ -65,16 +65,16 @@ class CPUModelRunner(GPUModelRunner): # Speculative decoding fallbacks import vllm.v1.sample.rejection_sampler - import vllm.v1.spec_decode.eagle + import vllm.v1.spec_decode.llm_base_proposer import vllm.v1.spec_decode.utils - vllm.v1.spec_decode.eagle.eagle_prepare_inputs_padded_kernel = ( + vllm.v1.spec_decode.llm_base_proposer.eagle_prepare_inputs_padded_kernel = ( cpu_tl.eagle_prepare_inputs_padded_kernel ) - vllm.v1.spec_decode.eagle.eagle_prepare_next_token_padded_kernel = ( + vllm.v1.spec_decode.llm_base_proposer.eagle_prepare_next_token_padded_kernel = ( cpu_tl.eagle_prepare_next_token_padded_kernel ) - vllm.v1.spec_decode.eagle.copy_and_expand_eagle_inputs_kernel = ( + vllm.v1.spec_decode.llm_base_proposer.copy_and_expand_eagle_inputs_kernel = ( cpu_tl.copy_and_expand_eagle_inputs_kernel ) vllm.v1.spec_decode.utils.eagle_step_slot_mapping_metadata_kernel = ( From ff2c2bd80a200813e1b0d5821a2064f0abb90100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sophie=20du=20Cou=C3=A9dic?= Date: Fri, 24 Apr 2026 00:48:29 +0200 Subject: [PATCH 081/153] [Docs]Add documentation for bench serve visualization arguments (#40539) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Sophie du Couédic Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../vllm_bench_serve_dataset_stats.png | Bin 0 -> 110228 bytes .../vllm_bench_serve_timeline.html | 3888 +++++++++++++++++ docs/benchmarking/cli.md | 32 + pyproject.toml | 3 +- vllm/benchmarks/serve.py | 29 +- 5 files changed, 3943 insertions(+), 9 deletions(-) create mode 100644 docs/assets/contributing/vllm_bench_serve_dataset_stats.png create mode 100644 docs/assets/contributing/vllm_bench_serve_timeline.html diff --git a/docs/assets/contributing/vllm_bench_serve_dataset_stats.png b/docs/assets/contributing/vllm_bench_serve_dataset_stats.png new file mode 100644 index 0000000000000000000000000000000000000000..72c19d3d7c0725e18a79f0d6d3c4d019bec2d4f5 GIT binary patch literal 110228 zcmeEv2UJz(wl!XpXrhQFv4M!?SP_WQ1QG0dP{ayKwb7(0RXT}^#sYe1(osZIno^~! zM3LSEq>J>9^v*xmL6W?id*8e7fA5X)kAJ+A5vA<2&)(nuzP09>bFR-{4;|dg`5pIn z92^{+%ze9#aB$3A!NKu$`pj?elQ0><)A&o$WcLY^qlTI$=BJG`IOI;7T+lN#(bGAz z%uK_`SjW&nSa9P{f}1xi(>5`=U@R#lr2qR7f`&#~La&rfb8wb#FYG&M%)znnd-{Kx zmS4dUj%gem%w5}$UGi?LwDNW{^^qAb`RS+GYj{sd-SW48>FIG}S=hC#o!SpHWIa=| zj_>0a6_@vl4`tklb_-+bBp7(yt1dbqwlO_Pi^#g(vM+tQX_|MDfr zk9+1${dn!IWq>ah53j*VC?LDQrwqnZq(Z}OWTitGpS8XmgNYGF2j}EU7OXxVC|2ebXbEn~b zsb+yFuKI4PIWH%gD~iA7$Qu?b*Nb%MthQ_o&GOy41s9Pc5L>}YcV5%@$jd;sBuusZ z+3S8STzg!juN8}7)tNLfqb&C6)AOMv)^a_Kxw$vy^A`l}ckgX33`*Bu&(ANDcS$5P zCMKpPS|!=qVtlwHMKi}+!lLf(UZXej73&^4AIykN?hCaZDX%`*m~(q#;gkLDf~A^X z`rOGAk8ywZjB~e&r(5}Ldh+DS(7UuHQkO0~4Gxa{{FcYA?(F>LdUth$%_XLyv&E_| z)#^oNi#3;otMebnGP6E*H0WTqVsf`@Vt<(~XQo)0cEASZm<_p7JqLU(8azgOgJiOs z@&kTRIm_C z>Mf|S9ceE$Em7?Wk{Mg)We~5!qo`)vyP#}F($#)>!)g9wUE1H+bE*U#^ogi6E3X$% z8Ip6Dy_m(I$rEW*pCJ}DTS;N}?%gqX7S>~}>bYl~wz?G_+#--}6`L^HAf8hCwg~Gu z%1~Zr%mjlRey%>lwG#^g<>#r_Udv|y5 z(o%=xJD>M1;PAJ=T4ylvM2!b)uB~A%5Kwx5C&w1+u*A~$W0r?-PjtAAz~yf^Y3h~8 z{=BAhYfDXHPfli6O=@KR9-ctuScAoqCfWx#*IY>){B$$hF~#y0hJK4-@{s1iE$vaW zmr8A}j5p1+pElj7fVoDaqau#8Jl9i{e~tHr9p7+ryR@{l=tUPPNR0IsMr!9jSnVW( zU~uf%v2$%7p09J29SWZq4sVOd6fD&axE+B{tatVsJr@?&;JIfc)OOCt7Uea*c-*1m z-fwse6F2&ewP{6S#6&A@e2(-R(=&)~e#B*yRXs62&|DO(n5x@7@XA?ctXW>7<#E8K zQ!niu%0IW`>_g)NDSmHmE)=)O5>>Bdcx=*;?Z-^QDq2&SV4nNzbm}IZf`=hW(N9a` zv4G=mE!`^Vr5|%L-lUo3>FGI}S5jmKpY;WrSQ}L1dw2Tv#2-^oXpkANl`+cOK4TR# z?8T=WQUevny81l>DYNW4qf68>p9J}Ow#SqUZnf;)hvAKUvq1UWZ`bEZ^u0gYgGno{ zmoQLerIqKuol}O3TSPl_;Js&Tlgijqm9YdfnzeU!&S9#ms$!AzsC6tLCGgI16n=V;<@fAqxG6HuFo4H_mya5SJ#T_mTCFfaO+17>t+v6jH`HU zGH%H7zVP~6dYJ5xC04dJ^%_?$!k>F=wXIcBi(Sptg_vg6tCS7mG~&(LJ;%pJ8iUox z&THoSCgt>Cb&ThhYWew?S6Qr4O)$&uD^d586DxgKs?R0W^^(>qvuY%&FEq9)zIeDe zzTqB%uTqH$h2)45HLE~*Kk4TDfW3QeExNm3C`qKRuTRb~1p^TD;K3Q6*sBvG8^2Hb zI_c^}K&*c($FuG0rajxf8S7V;Dcd?+XjiJ?E~-as?a8tAuChW&y+=|bCWibb*7dcO zM!GzFpJX*~#A)eP+3DZR*8VV&zQ%7uUuSj7=@Cr7BjIXE+_oKM(YiVrF6-nRlPtZ0 z!{VRTEW&eHd;Iuu(U5Ypa{c|Y7Hn9HIIuG?*=EFzfm6A))m2e#=uYI3_3fA)fsDYU{<7fwc#TXq(f9G1Nj`{rC;9`0Y@S3}4|U@a zS0@a&1oPxHW(nu2=&IV-*fe`pk7}lVc6PDsZ#(%kBt&n}0&{d-5+X|Lk{+Jw!96>( zMMXt*c{`f|oD9077l|9F1YAro?@TU^a96X`Nm45*3QO$H;EL)@<&vp*Ha7UVwnQnm zq4RY$U+d>IKB2yj%7oK48X6h_1I3=k7n|;1b33oA%TrubTpZTtD~ZA3@kzx>mvvV$ zVy12sM@l&2E~r{8*+D;K$||Z^XapE5Z!ONX8}BJ-@G?}BJuYPzbrs>UH5x7%pYg>yCsxI8>|_;7xc(ulC%&;cFf^V@ILZ8+_6BpOAOYdz0~ zJi-Kg=2`Vi)!$w^?3z36)b!`l+piefv?_C!OCgE{9?@x^_0V^#mD~7dzKQ3(w36F; z+9Gq^WCM>hx5risa&n72#*_?KPbt7^PToPXJiN(y4l5Q^rC~a1zn?KOwU183~tc&6fD%-HuRW?vgyfVt=Veb=poAaUdt>+h54;T6wTvF{wnHYEYDKkPNQ#N<&fJ*a& zn}X}dKF*Li(OD~MI;W_pC@>={U>;=&FKAeEA`fdG+N((lIcVN%`iR&vUPZSDUMAYwDzi$PVp+)9c3Nmy8z z%R`Y%T^|cD&pdfk=yriWU(^^@&@~Z=*txUUZ@k&B_~c3sjzjBJjk85LC!B0D1!6-6 zY}E%}FXSGc#jg;eeWFUKWJ$2qO2qSvJDq9_?M9T@4;s(0;J06Y;2Hn9|NT*U{ium+ zoGYyFjLFPSx;oG#AfoP7_1X&gUCDjf-(G(6Y3!GohDm)2IyVJk)~;ID2W(ig<3jc< z&uP=Y5gGRDDiVD0@fQRutW{Zgsn5Ft`V3RX%yd4!{25W`@*K&>`NN_=WcM^=>qgm* z4u0k=*Nd$*le67aLwA73N>hy|RM}>^m&n0@aI^P^_lkN?00krz#}^wcV8cJ17Q3+$ zJAI>SK$y@PQM<9ek`kqXf$U$72P?Q9+}igZACFkuj6|goPz(IDpB~YXSc6K>**9G< zjcQxAC{$o_$vKu*yg83CxV>oO1H@WPo6zXdk?INCbxfU!u>nJ)6t31(ky-r+Cl1@P zJ&R2*R);Fb9$^(=ZNyeZ+vI6(XBFJrc(&Lgt50VEFp;Q7+eEASgi=dkko-Dk^KU=M zBb-`kFIaNnl={JY5;?Zll#dB7KwDNtYII+@_vc}kD)3~qa^)9$^ zKeo%RcL5%i^!0<&PF;?QFr>9prb)=Cw|@B{Lr}oCBbSZ zV+6@n%sE@yo>_nT!B0CX8G%ru_LeYfe$*%hvebbfA2TK7 z?Xos8+^Avq2WA1rWi&HobiZ?Ja*t0ZgV~Vl=lA4jkeJo2;tQ#q;)EzMm2*yN`4rWO z0WX+Q+Mmy_a1MDgd#UbFcl}Vedx|&Tc-J*TR|W~~F-tD~Ht%+f7s|7i)>h+}(`KAY z<2BdoK9OYEm$CfX5*A~qveUMtPK-c=qeR2K)g`>vU9XtjxrgQC?!j>DB0GmwY{98i#+XB`^|C zvSAB%vP5)rwA|7Ffsl@Aqe&Fhn~|Rs!c;tX3|`!K4PNh7wWB;=IS>DF>_crVwr&3W z`JC#=@HvkU2Ou6eF_=^yjFzX2DLap}KTlgKC#a6XEA>q^4)EkPN8PfY1t7WJNqSh@ zxtyS_oM1JO0CzH8^;8izY0B&Etd?PRad06$bE-DB4pay$s$ATyX4xFL+9`jjO@E}H z9BT=}FrjtSg#&f?_;n_?3@YO_2D@tbhBl*;?i>lT!f=)swN3zLJUX_1)yb5UjhJ#& zntE0=@J)Q?^HND3PV%sRwg>FOA={V=qs-ZhH$6-@<+utUErX5jVsfH*_2y-(k@8@zoHjd^ehc1J!nR zlyn&BtG)DCz`ASpF10u}2kAr8Ywm6(oBM;?rHgtl^NlVOO6=Ovfkdxj{rOECDyE|+ zPkNkAb*T5hJSU?B1i^q$=0IVrVfE_5e!lndEhmgCtH=9eXLE}*t#nenB3fPD+9R%s zh3O$zB`&fv&3QzAmAT(5JlqulM!1`5b$nhy}iwRC; zpu*LZ4J;V3twc1vT09IGD-o%gok799{`yMCcW%lC9&yHv4T>ql$LC1Y-J$bXjI_tj z;*DGv+}W`aIkB%Q=+UFqg^%PiC&oJ`8g2>ayjyD3b~ndoOCu`NbgDS+@4Ob)<7V5M z+R4Ykk*8)oTFYlRm>jodsaQQ$)P7cPTd6RR{Mtl8C5NilGeG)qppN~~A2kH$!cE~x z{YsJLp>bhx3H(t}Kn&p}!!b_{ql;?8Tl-XuZH)WoEZO`M5di>DcaRJb+WK|+f0*NC zm~6pnFueJJd$_mRnM&p%+@J?w>neI|Ig+g*s7&kZ08fNZzrM17cLG4xm^hWyOez<- z5+~Nk45eF7j8y0HO$?@OP*Zy#!e-_VQsUvl;$8W-$s@}K`@B-Q|KAae{{;fzs{Dgwc{;J`~n ztlWs@o%8bN>B48<-fBx4sItIv;nt5$TWZCyF!U`hI;JCcc?MTb%EU-YBVyqJmcygt z`z28JIwuZ%`*Q#y<4WG(_;^XGl~V;a0B?B8jDL3GY%NKge?~VcoK=V7+OpadXzFBZ z@vjK027}{8it4}(c+0cWTHE)=ywcW}Qa2Vi8fJmDxhJR^Z~gHiW`EyNMMZb2E0`?x z@osmE{!*=Kb3aN=C!@3vv+5?(x~{r$BB~}231dy?R*)SoH20acRWDm4y@D^=w7yr? zR<)e(sw$hLfw}u-=FgN4GDjKGA4FQl3Xr_%mZd5vY z)H&C>L72&!C6xFqUF-TdmWa;d_WkO^1v8d{1sFVqI^!OtgwhBNiP5f9x@67RB&-3R zu($z~2}?}!w(Eu(-qdyc^@lZLaiheNbYo^cG6n$|7Htkl$4KwA9Y{dbeVCt2kz#D1 zdLr4hMIdpnVa-I7K$8H6$u3&)pRW9H@SXqm%MoYZ+#pVd;7{t;?|+b&x=R;n(H!Vr zfPCT5uv7WNrP|xVfMcU0DHB$yYc@7yd%5G0N+_zD9%r2ctVP;W(@l{FqGGbZ*0$Ri z)||M4R8n8(EPs%ta^cn2#20c4Mi49dxi+ow^(>(SEN`P&N=NyXoA! zJ5nQUk)^;L#ggu?F3-@y*_kZ9(T_7EhMNLPzH=q^&`De{0^Rx(1i4LX&_y&7-f&d|^1j|>RHT938LW8($6t&twq?$JgRK3-HR8__2j*mzMhtWodO|2{J6g4__?TkF9?> zohvKDm0uY7fY{B6muWLNIHd0p9Y(3>2(scD-%BgNARe_H@9_)oF3$r(&-E39S3%fuzoQK&kyK>sp@bcMFvDo)_EF zXchwJCJr@t4HdX2zrUdP2H8NIW&%;Yrd1iLhT9GAOIXgboW;R=6L4GX{9v4JNmyWOcuI1*jiRA3Ndq zpL5Yq{vseO3xdOw<8w+hR6(kTqpDG}YA@2Rq^go`i|2eR-tdI#&4WIYq8mnVd&iWL zt%oFY2RgHTwod&3Qki5Rb8~2GuEu@aDTfpQAa1-9`>fm zj9*MjQ)$oO58tqA9bEu!$&LuMV@Hoh7Dve2b~^_meT4V)=@}$40?W$E3V=No*t`}( zxQ>Z5VE=j_OvOM{4I&mH4ON$Q1=YursnmLc`v3Gv%oyj+F~otezLDodRRqc2@EpK~{Yyo*BVuolQZW?;$VHDfM}e4KvFVa2D6&K(O8|7-SuO;Q3J?q{+f0)Ibm>3vNx$*V z<5-)uM*8!ws9aY5u=>C|P|pQ`1Eic#Y6T7Fw^&%S?pf*_-PTRUSnUmf983hEAPOp= zEZ-q?xzSvRYk3;%nYg<;@BU7jhV>sWbL9{^6|c4yCd`HkKxu=%+VSJAtzij*l;NPp z5cJ%j6txzGBU_w7f@|>94I6ry8(eDZ)`1TD-bB00ulI6CSJNI zR%ue$h7e8<+Lu%W+|FR$*(@({Az(4}o|2StQK(bdbQkkh?x};yk$qfI)fE+DUG;c- zH4PM(Nd?7hqG5|u1~DtnAP4T7k6o9-MdRLb1_sk zWN|l44XqE)P7`A4LDtd4NUL_%LgwMCoHk?09g3vDm+}NX9;v6;LVnPp2x~WZ&F&p8 z$P4L2IAx@=GXiCZh?RG7a>F1A8w7}k>ek43Km)_7o6RG(!Fjb@2CsQVe$&BUgGn!o z11gHdJb1L%Y3Y;ja8f+FQE#iqVZNh8Db>48!7!UjUyi2h>tzGEE>3WBtQ9_+=E!~o zV7^y7uaP%sHG(p1uOJ3^?2Xe>4x~8~YbwFP!Rxu@KOrF>*bR~=+H^{P-j?>!@xLrf zA83pb^j}$u5E6-^j&vt|RT9j3-|>ehvX*YOtbvNJc-6OTxpv*jEDByg|s58=tfEu&zCvM`eOa&7w~Z}*pjOdwFSCv)$uMfsmZ z0TSu8ZLea?_ABZW!zGJ1p4kNnvP5Yd;vS=V@U6HJN$w~e_c!L~`hB{X#_Rt4Y+B5j zz{z)VJgZq-zE>FVX6fx6Gnq!8eqEzU)i`j3XOOSuQ_l5YKEj?;5%*b)g)PWrVdjg4 zO_?0THfW4!J2a@+^c7L5APmR^5icr#Sf}t&J%Vg5PKpesctrqPOWNFJ#zceR4KOtay5Ab0SWlEJartbaP zvHtB>s>eFa*@anG4>Dc%P4&?S`KohB2EHj6zZT*|4t^W~7QO;HKk<7;C<7lRCrgVX zP+H!OgK#VwXWfwHLE0bi>_CnDx>?nG5xtqqU{{2X2YQAnhU_BozDQ zAHbiv>@^W=n5eubqt6#9Wh#c8xJy+(h0;P$t3-t|s}h7?Yq+br?Y_AnG@6P>s#{g!*6C_`bv498p#E zfgMB(6x9Kur5I~`OpXvUachahCV)_nE#?MGZF=(w+`_Vv?y~M^0g#B8wD6aYS!<}V zc-IHS>FmT(DV1Tu{j;=7@NI7G*ogaewQb!DN`L^+s9KD zPuaKY=Wkc&y__@j)}py0ZvRck=SKB6Dy&P*l}sp0`0J0Q{J<1`iPx#Pfst7Gj=x#Ybh(%z)v_d!gL!mjwwdBUE{@fYx#W2rlu5!e=R7RFR%ZK zt!nZV{{w!l|Bp^Awy!g>=b%CIP{R_`24r`lbp-%Gazz$VkVK1>Y5m&-C`=QkpOPO9 zMcx9kp%B3$$0Eop0j;nHvQs09j!e{x&RAn&w7v+|?iuMowxT5!P*3^fb3D%FpY#L! ziHq%1jWQy8@R}R~HTs->ijUIeZnr|X(SU*-dO`qK;KJpk zNmnAaFKvnO0l}2f?r5aLBOt;u1rs(fjR4*?A=2m{raA2*z;^iAr$hp)u4WS4L?sY; zJD6^&`%^wmWe&SWD6$8-t7>_`fEH?xMd=iF57s)7tF5^@#cq$&QXc^CC^HWnn7a;x?6k(FX7CERSn? zaC4JoZ`15WKP`jb;vMKOFG%t$#VaoCBt5B6PCRtr1fgQ*OsyPmkDsT{(5yX7 z(O;}=C~BPoYsCAaqE$Yv!YRhN$4t+ciWPR}@E;XF)q2i3KFd!=1_h^pq}tx+4c>>P z{|P2Tf^~Iu6;Qd&j!fGwldWqnujw(w8o|V_*KIFq9d$!xU?3_UjzjX0fb0VeVuT=2 zE>v%C3Y9P~=<2#roN2!?c^Vdo=tleLbKd>(^(-TlYeXxlb=tJHv}}ScM8zb3cN`u} z@s{g90ZgoR4^KY2Ltnq$vLw$FU@+38xd0{C8jrCv!yy%`ihSaun(pma414E&QfM_l zKgwypV~UEgO_zA(tZamy8#IKD-iz!$SC!#x!7tMgBIOa=-D7K9%x%7NAM0S~Cc?bi zwusq{kG4Uypn_+o|HsSR@cyt82b_BF+h^0hOigX#oKYC|peiOgZKzq^&JzV6m$moT zvlcKcIxo^qK3o3f@DpGoGf85Tf*S;+k2I}<{+*SNB95)ECkrDBtW{#uCYt>rla5lJ zhr`3e*3J9XGceoOpaEd?0G?B1d9KeEaWWHtsf{Y`#*%C#zK`lHjI$KR3HA&+lWHru zC-I893hz)#(o)U2Qc~x{k`&WYMRDP={tbw|>CogSkiBmC;O0f8kj2QO|KF9xZ(|<=c{Kt;mM|ziM z5<>`N{F^FEtBqw>3fQ@yh6wl8dAI@!y@1= zAnp%V2w!QNVK(77N3zKuldiEPI6UY2yp`SM)#EYw(m~!vvV`r{_*riX*mFArmN*u; zwuYmA!^bGz7R!7)c~Xw1QJ- z%;ij?CCG}%rV)tW*^mP5&6z=>_D=jBAvbo3-RS3&FD{%WtDc+C@Av);_a1@*LY2i} z)0@N;f|B0PT4MI$^z9h+q&8??I)UF|9@5MIY|Dl1R}cav5fbzOr^_xhUdG3W(ZV3& z0=79M7RaSc7Nn=x^P!XPwBX=c9+GE@LH?DMX@^n?QTe0)^es8zI}f2;U+bltH$z z9zs9PUK)f5hP$q@{CXa+dctb=TyF_sf`O0_rQfZKdJ$V?8C)|`pXnYH7AD9a-zyuQ zG5?8Pjn$d=2tUQsV-Be5=OVDC)LdK~qMt4CU~JPru8E1+oErm{mqWWVV?2L0Nl}QX z3pSj%gX%_{icoNwu5`)zs?ampFTD#H=E8S4^2-K!I<>Kl4-Ceo& zkyfrRlZBH7m5#Y%jWb&NPB@@66N$6#^Z6d3Q$~9z^g&FF7!gOn&)uI|X9*`J@0V0G z)6d_iSiD9k$5K>P;r&Bshfn|v@;_&&TG@0y6Z?m?)26*04s>BI&JigqUst!HN+hEn zSb5YO0!&tDOxb!SO-xpu@6BQ6KaQ#vQm2l z%#jF_D(Wb;yYvRU@}`--TlN}iyfcG{BGyBgZc8x@of$BQ*aWl(9q@p!qM}i{M0P%M zEWNg+9dp}$g^aG0R3gq7J&kq?Kk+Auw!zjmg}||MgO_-MDGJ>2ofOLgyAqF(g_Jt-Azc{7x4hqW<4hI0j$|_ zpC2bLn!|+SKT(i=2MfLxa4JrTMkMW0PNl^IWKn)mUZKLXmVWJ)e z;Jynkcq2$4wFN?I6>Z62a&stg!^H}6Kw6^f<#c1T-bA12!w}`!PU3!^hq_^LtBJ!W zg)RuWiDdo$3gg^F*!M^)-DFhvZWJtoZdCQUAa54(bUmcGEEu6cizl3ewl(xVNZJ)7 zQe2-rt;Mp%>>B0N<@cnCx9h5B0!+Dr(pyh9aU;}bb&yQ5p#r)PC?Tybokv?-xl zgSgJ*5hFN|>yTQqDHD+KH9r}(;a<>%1=>RZU`mMwIFG*+^5J$Nl4$auR6bme9!wW4Q7P1} zMivnS*RTKPJl4RuaesE#1dqeG`Vk_&+SLt~o;^A3-v2L;MCm8vW> zTE5Zq(L3Tk)YI5H6cQxbc&qz#RfdxUzkEX|RYqrX*kw}d7A+8?5U%uU&a(~*Nblg+ zZCUJy3*%_51p}6TcrULj1QfD8+|Q*ShonR#u6!LY(l#VPYC4h0jB&CvZ4*kYfb{y{PM1T< z$_Z*|I{K(&Naq2|>zrk~(MF+z)pAf($s{X~U-4W{$UHWy&Y2u;grQ7XO(l>iHLJd@<56-ta6hfnlplq~1T~6>U2bf1 z`DgTXXy52t&LVKB!iZc_=#~N8J75dv_an&Y`DW;#lOv-l;EPbuMHgV$~V@ji_YKCvHW2VMwKd_Pu|{p0;*UBSHr<~>6bm(W-^OTM z>mRN5qj9R44A6I&X}&~U8aysrUOsu2#HBah^~*tSh-y%?!V@1(g#H5=hOVv~s8 zXT(j6j~JuHi)?tXZt3Z|aN+ve6-sB!D{fru*r{b&cnVQqw>yFLsG|h0QqOYh>gpP! zUKER~+`Uls{amHRqUVntlpd-(;3={$)qyh;h9`Ga30jEi=-Y}znR4{#(L$vloP(qN z$Vr@gfX+nfJ{xOM3! z;U#cUc@aN$6Rd*n&_AN&O`7o*dga}Bt>>LbZ=%xTKSLkTyG=GD<_-GiUT7{75JAU-i+|$r)<39c}s~eo(k2czP@2Tkz(&!ZR zPr8=9&J;@Vwi}7H%T!R-cZFJ32SBwCz5q*-SNty*p2c7G-N$eq7*U3D%#)pw{wZJ+ zgrV+m@}-a`?#9wNIJC*Xm;FYm`Ac^-fzJk_@}^GnIH>7H@0MR9`JYI-P1)*Z%v znWTeJ*Jsmxf<1F~kz~Y?akOUpl_FStjL_du=fGp|4s?&_EWsDcXhEZ;%X25!RuSrO zcnkfPdcy-{@iq=g4z?<_R!CsMt`mjSy+GWyJo!bqalz|^_0mM2%G zSpk?E!BXEyeit-|JHxF9e6ey%$BVP%Al#sq_@dNijl8e7s#)nOSWpfC`H@GY0TGk` z;|98PHw6>+wXanR{%W3rOyNZ|_n>^<&teb^%Rx5_`4FL;?l%J$NR2kFriuTJ`2M9Z zk*Z2N^4UY+e~E?wMtp-8H4bDvBwiy&W@40hEb0sXb^7Gu4{qF*$q0cp6uoz3?13NOZ1wt-vjh|QrC|>F>t%cg0<~IzrZSp9>#Pi zY3k%sdU3IqXqr#z6k$KE)JZ23E61V4*OzA!BSLYT+Aqja)dSAb0vuraP2a+_VvYbG&I)>75n}p$aty@jr{vyXh!1K1h8c zoe9-s-+)2Iy6*F&dlr7cZV1eeK-r;kFd$gKkZ60B%H3}K;qdRBC9{8g0{(Q4arRJBz-^MEk0A_n=Vn;2GAy(&{`2 z4~%rEA6{rv)o_**V@`~Z@w5wD4|XxqK_<&|58~w8HkQS+*R=KKtJU*rsQY&K4&OTv zx-7nb+?APs}_NLou&a!d<*PVmXqnD2T4G#vrsq$aqwYy;u* z?;q1_CBclqnUhol?3(O8EC4Tw_NYQ`UeqE}jOkf25HO2dpU(#4tZg*ZvSk%oYr+&O zL$PJVe-~rQ^Qm$!E2eq!l2viZm&S)0y=+WgdBWx>UpLg~H}S4E8`2y(DybiZcmk+4 z=cBSI>N6;OdrH0i3!SZe`hDM zleNTlu!cOT$!NjRg1Sr(oI(vcI_pSBBwHkzbohn`A^YC#tem%; zNiBqAyoCy+Yvu)+;OVOUbcCe6TgygjB&NYzYT0z3bW@M5R%Wb+@MFb|0gM=-Vi2=R zL_82)h%ST~Vf1_@Sqr{hLClu~x&eCNra$-Q#{6spMeLq~K987PL_2Jokyo(Q0pfZC zjI(_+sthqZFbq4n1^70e`E7>u0vux^a&^W$vQU$eUk)?Q5>18MFMmVK&2BFGB$ErF z`DZ^Ml(KDpb?@#jmM}VpMwJ&i34$RMmB^29wU|m0-rab{n5p8y3ZWSk+^C@eId+33 z<zP?sz_rqjoD~1L&r4#)7~BA$m6UQf_oTq zoY^>>)mb5Y+(*gG*3DeaBYv9l8n5>nXk5hN!^S|8H~tlVN@>~k3&@w6NR{N7AYb4) zl0Y#c)Y-77Tmd_%;C6Sr;~t=^pb_-~+Ok9Yj6rZ1Ve=nNw7jLmtl~H2;3E&-f^ZVi z%YqS^XhER2AIn({jf(88T@3iI)5%1Fye((rMhJmWL?8oPEAK9{uR3QcRjJ%v{vo1R;yo4g$?V| zyvZN?hgsHdz$>!&1<#Ix9p-GO%;4jU(6pb(*=Nzo*9c&HYP!OuxitV=>xS>(;$D99 zxWaNatRws2VIdLm)Xw6;`)uc#^)h5q7$=Xi&Ov^`S`PuMW ze9#{9%%o(=X#Ur@489kHDx0*w<5~eGcO!|fgvM~EDe3w^Oq+lZrTRad-OWXj(_}7U z%p#CA@J}SDmG`$4@l&X~#!U>A7{lj=^hfL>;fnGmu=JZ(=V{|5bl9l%cH!6CX59ay zUVMFZRwTesa;MpTw+-pI%^OR}`_%L$GE?360kusJTDKBA%}{#vT8jRF1ME4S*>T+^ z=u1{!3Nt_jwr|QdAkz_h+awTNihun?G{?K*ewLto9HJpjgkheumH2Ps2qkcfg)s6` z#-6%Wc3wW@i;=NzJIj{|m$Mf(L8!-0T(aoNI)Vs}?Pu15_8n?bOz{M+uD2U+w4;*r z!8IBW$(fU7q&(sls}7q0QD>qD9g{=bJ;9OzNV=X{2-%4bMlN!uDWB2Cd;O>9pQYRF zE{qGDBZW|=VwAp0!1NM+c53>UaSYsz{@^b!y?cM+GkzS`c0hY@UscS52Plo6O0ZOi*fe_fm(Rw5HMDP-StW{zP1v%<+qyfQwyvUf zmIYg-aMHZ(?6PN@Ma%oAmrYw-4hQV1$BrfX21$Lz)Lv_tUBfQOcd)TVS-1i`8e$kB zYWJY-k*Wo$w2XMKu%Bv_jQ;n#_OY;gO(Qjv2LGG)x{H0g0!ws8>qm3Q$^ z`|__j054R7?8i<5WZ=3f@?rO$5`lG$yb zG7LR#)ZY2m1>S@MI^Z0{MqE*)`XtDZzXU~};9s8sKo6NhjHoE4DH%16KXYoX@%f@q zUrE|#{+}n3OHj^kz6+mz`_)5CUkRx`hBp2J!C*{M2ZMZcOCcOkBNpGthqO)THvQ|8 zC4fbaG3;wL;YqB5Mc1ntJ!sAn3+cpO6SS3ET{RGSr5IsIZMUg$x(sJ0AU*-+EW6P( zJ0$U?>4<{94C-%!{#SrF=6OnWVuXY|sS<5%{aUk2W30p|+x zLpn)y@1s^&Yz1=;NZK&HhK`!^WzOska@*X5OlM4xfi@B&5sPvdHED7ePk<5F53;En zt%A>2Is7>~$sjt>ej;dl-%6@%&#&^3 z&|^BJq7!PL=&>9mXI1BtfkGs{=pqJ zB0Q6BL^FyCzCEq#*gR4oD%Eu z8F`w(>>2$wZ*Ltjanb{Exf&=RsI+u0w`)O&JyiRjsZY-4+*$|>Lw&F_{ygw+TAn`NZ=u^)+ViK=Y(|q6PII{TG*9VY zYpZ+6`bnL(wDkrMQTeAn+$#s4+@Icm4;@AM*YpY?OqZ^&rPJ#NpqC7PZ+k9j^L~08 zXx*V#T()_2@w9yfb_Shzyps;&nIxf`7D;#W&ZGWV6e`csO{b(N>wuW$(H_&{{()y?Jmfb5!7oJ$6*g~?r4v1zLW z*l4|gr#RN$RL5zO`_&QP-lZ-7uXZ`{>i)gnAO5aDCgLBu5=Y;0HJ*t5FVJ&zQ>_}8 zzg@A!^!;Jou?j>~Qi4Ip*iZ*&BgL5MTvB%cMx$@9C-j4teRu4uujYBu{u5uAE9$I# z4r1}oHpE$>&)5D*%3r9iey^rawID7+_oic9gtW!Q;^5|p2dmLGO_rRSj*1bgCYnTifyX9j9com#fefNSzkod5l<7ia)RBYkO8 zFu@U1Sd`JeVpDPFG-&6B6unWIv!7 zGax|4Fm1!&;74W+Mkz2hkK1iN8EKNI|8NT}=&!LyJ*We&7&>i-TsM67C*oe%BjgtZexAZZjk z=!|b0p*9y19ccei6pAubG-5Xcw(3Id>DWK4VAC6b*KHn?ol;^a5F3<m}`jnIrufzORhO1g1Mu)l=?cG2zpnwzAS(WbpJ8S@oY?iR|+B;`qB-uQb@Y4I7a5O2Nyuy(a73MHTp zRf-nGS+xx(2d+5v?}d_3K5S;gI)@T1lx=yA;-54Mx7J`QP$p~j&Y4U?b<~`IHA@@W z=(367R!*Nsb)42y$!I+< z9g9ZSCp^zgBbcg$+)$V-;0+HrVVtv;?;rG&L?#<6sH^*-AY2gk_7A@EZ*ePWBwt!K zfmdhI24Om8Uf@v~>6Vq8%bB1=^6R>b%w->khK?;;72}CsPXKihL@%ah$~|d*bEavK zpnzmUfh+$et>T;M85z`*fZhP=+9_14-XG%idrJ%YTP9sxqlwkt(Qhl~em}_)oe-j( zw^?;4t#ysY{wy3^o{4MAn>T4-47sKsQ@Wb=ss+Low(4)&^Cz27Ss)(pwuBYzQux_Y znv+zPFpa{-PmDU-Ho;mqtre+{_|sTC6q zVC4$+?9=AKN{zR1X!9eH>0-m24_!Uj&&QcTeM6}cw7U#81e0ZvpBbGLM%diN1-pc4 zVE4F894BnrLVB?ZG&7u7r^pzmvhSbV*!{z)Z1lBoNe`Y1sMi(r+G5_J)cPMdrP(dn z+pN~G?^W*^HL*~;b(t+r`)v9DV1LU0Gh5Nqv%nDkH}7$uA$GkB+hEn9{q-IAQ8Jd1 zeEZ3eYi{>ySIjB0&T8D z&D4-RX(o(fkB3}TQsiKWI`Dk*?Htd3UWv3Sxgp6xRCteqD&H?Csz+0Ch5t! z;(8Y%t|qpP!e$$CG+KLFuW|ADGN=O5b^kIh9K0Fd%-eP8pQy#}J!#$2YumK~EHZ6Q zXn+S+;={v)NY z54-%=k>v|^aHPx2lN7tYWd%KbW;Zwwk%Vl-DOR0Lr^6k_pT$_)yziBFQ~w%x4BF@9 zP02#dOE%~zZYz6mHY`M5&Aab>e8^)aiet2}Rz9e8P5AfCf;2p9*YIGcnHcPb!$?Pe zUUA(pDP1r&Lw_DXClFmAUiOdl=Wf699|cJ|a@)0t?O)CN4}v5e$bQUs7_=8Ds}6gm z<-(a*{cMz4xyUVnz0|28SN}M1aP%^n38Y}p)r#)UY?AP8)u70;&S5_r{S#s^1=3z( zOcv3DP>^!qDA=U;;Ti2ALpwxKQwZub?Cr4$b!n(2S%B!3m+zuK_cYPv0INI#>c*u= z03}N1>8sq&Vf&Rh*w<-YVQ*LssOpPlY^}}U*jo@jwGt%b;Xk~^yG4u~(a~d*C&dPg zCGZ%2`G!Ms4m%^zxBp}RpvJgef5#T78|No#2em*_aPC{?lDA{V?rn1q@7lNP;mK`3 zt@Zz#n$^y@mNz@*J-aY(Zo-O%yXFPXyzy8uP<-CpUGsKH9&u6I(b6zF{xQ}nvTgN4 zJ+sKnkv=I47Z2kGZ|v}sKGb2Z$r1j1IDu*fdd+=zYh%W{cU@)un}mf|F`Jv48wGSD zOV2$&kyuQf_a8YKE7%8OPHa4P?%di{tKR6A@87rY4uf~|g^!6EXxn{q{rb%CXwqLU zP5+e=GQB=>j*NHZn1%FDR988UW*!gj^y#Oc3$2{@WkGW!)&st2vC(!qoIDfw;`u3&nKDP2iU(-u%=L*6%Q?D-W@26G^{m8!L&hHmM-zPfk z`cfs*cN`x@e;*z8_g_Z#tG}JSwQ19)(Sn$4P3JdfKJPDQ|Cau^x|=@k>pOSv-gj~N zF(qoNal?V#yLJr*>Q5bF|9S?Rp_S!f zrdtN~&p7xP{~XA7-+r#VEd6D+_(eC)TD*9%ER%_`d38yOX2c7+ZT4$g!1k!!Ze!&P z^~%HQp~;42pWSWj24j*=$*@1-_c>wDA>22v{G-M!{(P}b5$*9~p&qNs_UL@hzq~Hu zB!=YntmaGfY=55*)81-w9^Z8-ef0{JeVf_mO3XGFD*SzPaqM=dfByk5pSno)m0r4C zRL#5Xg7Dk5Yu4;FOHjq~k=8HQj7&fCd@xFmeF*!H7kPKD+kdZ-oALe?ub1h0yL;$u z3BEJ8dh3<5=mWTM{d(6)t3QtB_N-D(0PI(Iis-N$mq^F|y`0_~XS8$7-cgalgyY%T>&c&N{?R1#^`KZ%JfwzlFjg1(8H~QJEx1W0j9oOI# zvRkI6`FBrH9mE5cnHC*aT(^ArcC^eEp)W$@{QI?>ypqR2(-ooQ%uVn?laZAQ{$&U= z-0XgUmVUF6oApRoITN{AS;uzl_~l%5>toS9HOp7=XiD;G9>2V2;gdb3`*=BT9#-0S zDLqE(d5cS^w@HhOFD~}#mB9|qAZ$3Jc6=ZT$i&iQ4WxdxoD}>vH3sx8S5>n{!>jmY4hr19DqzU;jQF{8Ph`_z(&2 zd}%~rahPf_oay`$5)#$jUSj&)FDgDiyfQd=$bC-2-iBG$pGCgc6VOAQroP$L>^DrWe;P;vd4?gx;Q(pOsEX&?KSa> z(Wfjn*tC&!Iu83It)PNf51X2_Wd6Fkdp5*s-9?cyiKrA#BcdIbgC%f9r+UTJmePPly(;X{V3j7&XLJ z&ciUinZn!^pQvn%?)-Q#jjo=aPvO2|Vvq;U3b9vI7az_N6~A>b@pY;`;+C12TSKx$ z)UL@~;UWn3y9nDw9SfA5AFZ9gFoMYzFh38hh1|)o-6n%aufoJlk>_5NqnblH66#CT5|J+n|n*2#(!!OpNsQ{TK zb?@d~y>jI-j8ZLCNeSnRLz+7crmn*NNYwjKH9v zg~9CHcgR!a&aX4ELxR!J0CeR0uo!8>oMqecJDgbC4ou*}VtsOt`H}`xqEwIEFp~P= z`|lad8FQAbRbb~re~~!<@8`kW76cAT8Jn28e*gXVQP}JfF>yaWpaAsnakv#Zmu&7z zEZ3=c@jR^_CJsgH{An`Sb=uH8>BK?r%{8MFSy=~GF&Jz4LvCeU(OegGMLlQUQn%2X z+@da_db?etHPg9|L~strZ?)2PYhe^0+F^amap~5lU0q!(^w$h5y5D=oTQ-HYWTm>D ztz~OZU{sCd`msd;1x_2kn|iUnO*3@%aHrKlM)D692mfV8uNjT`H%@eqLyHn#~ z-CK+&)@H#wVlOV|mFMV+owa`6av|O=X2)Q}i#BUl3RO!|N>5K8a;uKgaZamuT(j!8 zv^1-Pn`-m;_`E~0NWpyv)e$4m!!dv-wdc}^0J^n7p)YR(Ez4YB8#1 z&#l5=N1Qp1jaJXd{2%Jx11idF+Zv^9?KZS+7F57M5EVgGkc?Ri5CjE9l8T^$#3D#m z)HWeVDWH-JpdtbyN)9GK1PVoh5=2QQS#pLqSJC!4|9N+e`|cb6|HgPd&b@Z4RM_9% zYtJ>;TytedJ_}W_$&&T=5RuxRF*@Ac%TYW2+F|dY)b@zQCUg1t9u!IL&c5(qDL(#n zWf_Zy%U-=6uDWtt>wWD4LA&0l=nFG_lz6V2i3}fXe4&*&`Z?#v+`f>_Kff$~HF&#o zYrkiXkH9tIZ98@hCDWz#^Ph=8e8K|q_B-(fuL&a(yedrpxij&1me?4sxi}f(jbfxc zrim4X0qy0_)yN14#bpObT_T}n@@S&Z_Vb_9g@ASa?+{6(55C~hiPYYP*tYw`iO1;7 zjCN=)U7@68cxBOsQrvbURsRNJcqWujx^}?^6{YVWH{LguL_IevSctV(|M1~M3UVxe z7V9p|MmGBS`Hi-T{(L+9`b(=9V`~z0ZQV%w{^N{3{~uz~{|12n3}fT(&c$c=*U2(} z!&J+l0@2G^IDxbHWL32If+ffQp4e3Gm6#?jB(!D4iWMHm=?)-RrdBcekKiz|KiItE ze{t?4E?g3+>l?{&FE@JqC{EdO#_20vbI!dS-~uLaPF%Qod!mEqv9pg$pUk!tQczH1 zR`VaYYq><$*5mFPS+3dwlH}do1LbsUpF57+CF9MpA>H6Zu6Ozex_qGk>Tw-^K$))_C$QY!LsCRj&Gk)ov;tsz2c11 zyn45|^0CY|AFr}+F2A38@v3@`$4@ZBbLzJAKNwyk-1K4N*D?a`B|B!^e`_{<^}hl8 z-`ws!;pAyGe?H=cL~*LY`DOv02{Sx3`~N|(JYE3>3m;wh!P>QTXW=`E53R;;-K0Dw z3!YOlezk?|v2FG&Z5Rf&^UOTi@A^tbwYUD*_(SFX|F2$~|92*W|FIAMGY5S<+QxIr z3weW(&}&J{iW4$=;`1|~35dHXc_deK_4M0jOcXQ@38RM^j;O_1`*Qo0exX{wdDYW4&O zJN(R5l-Pk^wOE{^H{Xjhv_Jk~oAL7j%nU~!G2wd2LQ#t;o^jr++RGjBW@i1*lqUMJ z6pc5JjZI&w>w58o;LpE*eL{xoA=#h*g6Ly34Lx= z)zs`OcU%V^B-b(ssocBVT)!;71+mZnI;f+~IcntQ?%`rG29(}uE{QO~iE)|OO3Tn|5cXvXKHmS%=8G38^@#Z#guWF`NjF_@t==GBCpjk zm2ll}ALiG8Vx|?%_u9t@33Kk~8ECvRlGl8o>rI>|_lbrHXc|8ctUz1Yyjiop#rK}- z>geIc6+n8rtgEoa^^$b>J6g~HeZY0 zP5C5~c#YN>dr;z@H(UGJd2eB1;hikZG`Z^O%H0{u=NK|@Dz}+Otx0?CxlL-EGs4@A zb0{lp2E2Us7$J6KG&FZ**(!=eB9WasWy*?PzIoy42h&HrMuz{PtuhdMzfKyb7qsdW z`r}`{+sSflQ3GEks&F!1zRDNQTjk&lTez2@&D98S=(f4Z#m{PzJe*WZqy}pFf zYMpkalGL4+bNAWy#@>D!AHTk@;0{tBqZF0(n{_Qlb24w&Ug!1n35y-5am#gA6lk62 zGflSrTN~=EUOn?iuXKyd&$tg z_qOlir>p!ouXMiMsQ)LyMX&uo-6B8pXzu6we@7K^f0Sp5ET72{9Lp9*Bou)sUH=I_ zH)mTH^trh!GUGp=c)DL)*>c?{8>N$hQrq$5J&p%8-|5Fwf9at#al*eo%c*@w{v+Cm z3>`E>S`2I#31&^MH+Z&X-eP*ypz+JYt<-2ujmwW1Bsme`;ou@;&ez})nl#iibMil- z1XO9slJaGFqn6r?>P^Adgd!fOevlHW@Ch4eIVrz4sy)b2Vy3|iqXfG8b9%lSW-QE{ zR_`K~_3y8+;lH@-i(c{OtOHkAQGHAAuBE}lB*C=iabg&#-(ZGF4sWf~Qo|V$^KgDu zr;i`VzK)@%-rrg`p2F5}DZJ8`JkR(^rmy5DgaJ4AbPi5lm}GKA9;Ilp?eVerZBNO8 zPB%FxZ?BF)M1>z4{$>%fEL?OQFIr`;@5t)dn)}V=heXWz7uh{FhY;j8MjrYXW2adi zXnJ?misk3~e*Gn*uDMxh!xxUO+sjeq3`84P*|$3}h)Jm~|H+NNSl;SC*LWmBKm4jc1`-CYpdYRbKae{?$6C607;RI6pk8r%75Ju&ti-%1(VmrXdO9a(Xv zaFt>u2;%Su5l!h2Tmp|R;$au^mbv~zGbHUEN2n}!?^`N4<4ZR(R2A0hKYn6e`-VF> zqXP#R?Z&vSKev7^mpZid;`8xyZgU50?Z!yrm=C-8*Gc-IQ~(2`8WTKzsFbRe=jr7Yp`tdfdjW7 z6nO-zFtKIJHbdxG0;N|3K=44S1@}FdoSJDe{^s$NNdjlLRwS08u6c z92$VU|KbM^&Yp#uOAlIocd}q7v@7e)8`(0jdOP7gQ-b2*GvIU2zy0zj@*Ex~GV0oq zO6`w3d=4r89n|L@!RXnl**dv?KBEY&E^bxQ+^;ip;{_@1zg*3nCOYdIMz;iTbH`ej zoKEC+gO4@Gzy6QEzy1BjF+YN zG+FX=d8pF*=CMRhQ18@By$18_gVRckYz;4PV|9fQV`^#d=!WgeUoywWLXc(9#d+6~ zq3z<5f}7N!T!QhXG84=MJb_eo6Z^x*?XW@lCM1e zyjgB&l+x`r7o?K%#4f75>N+ljF!4U0HKH_Myw&tTp+{*%SeL8H{$9!b6?qnws*lpU z?xc4qRNrt;O@DPeJ#C&@W!eWLt^Oz_QZ6{R=i7+GZ3rgH21<=VD5&1Z@aH| zMaa6jD_xiQvT(!io>#(Z$K0JqzKLI!%I;5{lVJlHyE$Z_E@UXF- zGDTORvsc>@GTKhwGiOG3TRBLm_kdhj<^i)+hr#)|zll{)Wmz81hL>Q0s6JvxU(UjoPGF|a= zhI?xG*p*qT%d;c9O}tfAvLBBBBDQ9YMuhS0EBMi*yZ7#mR88-nDfQb)wcGArzKMfD zUSmcqy1KXi*F*Ue=G4wV$qtFF&)O&T7f!=cQPJ%z`-c1lzReJfi7+7jk!w$ou`I7k z_Zc`<+jePVbKkkM<1AMWpTCE zsbYj>zS~gCY?ViP@%M!r;W^Nq^d(MFO6tQ=_i>8g^!dMo$3QA#0Op5?GB9%>Ofkku z+`9Z(WZ~d=jF~V)YMg-Wbb2Yg3|_k1!7YF6ZLN5I==E=jJsA1I60X7!EakTGI`(AY zH>BxA@uwO*%}hbTnDdK0_OsA`Izz6yOpH;RedQy{8vfU>heODhZ~UL3Y#<(G;f!uH zw={ylE<+O?ZSv=q6}pH7rJ#Fs;tp>cK_})XTAhSTJmOQSzR6#p2GMi0T9;n?7Roahav@8+Vmr79&QwbD2v z4PD2N(3abve-vH1P36y{=!>E@Mt=phn4Z#@i4l;zAA=-e5Umt<(CNGq7Z7+dP|G(l z5%UftlpMW4sm^Ffj3Z&$N)-;Q`0}8F`)}TCtwJO4AoLTRk8@-Pa^Q-(H&R;!6amTy zlGRVW{n~|>FIVuK;XDXouC-#RiMLxjNMRS)ITb%Sv;`%gN9Zs#|8VzQs9Pt)x1LiF z^dLN3*nhRYE)oMBJG-Z(2ee*3gq6CFAEFc>eY|KZS{WZ9OsgIa^vZ^lQdhbU zzY6TlZKCp?*C|+(KGwo)^}Wx{boU<<{TKF3{HB>&k=lP0OVPR$FlG95>G_wlghfPdqVM?Dr%(HZa);3cE`C~R z+Mg(%*!$$d`Ra(awzfK^x~tz@Yw9ih@u8Mb7hG7AOR%P~o6k)ZUAFAiGxL&n$j;P#4udA zDJ6%z-K^oJejl&7okgU5GD#l-CI}#2_?2En?h9Cz(VupU~L;47X&4ni^Ao6b}xc4x{tnt)7=0 zY}*+I9p{&wYpF*tV~$()+dlp|gBT$)|DmEuohw>h5-#IO z1{W-^YrN!ZUovayWT-N3;0rHcG&~6Nu+v95WRFnpW5d+a8R@BOf+fY?N2u`%Ee6>LQ)sPH$LTDSUh(Xc#nPRVfYy zdne7Ed6L(OzTM)TYf;`}M#Tg43guE4>$b+xNwF|1-OhrU(33p77?Eo96&mAUA;kGM z3*^8;@&ntH*U;Y2udbu*3d*awJJ;e?w9_-Xj%AX0!>hU+0p~I|;Ao{_ zy7|OC`>EL{K_qNC29eyrXq@!nH z=6d-UqCgI&1EjkazKNT_rP3@!-eKyz{+D{_o0P$ovYquAwJxSy8OvJED~}U&1ESBN zY(5k8Jsb*;_w`4i(teeSL(^WNTugQ;GOW%MnZU7L<&2vUMfVBIzZ~u#~4WNS5=t2v(&P8QE ze%Kh|v4KP_!+BP_UG~xf6)jfylP7EZC5??{o9tcaj{dqqh*Jqyl*Aev&#tYLT(rTe z7^QhCMFDEa(Cy~A>(+V(1kbkxJ^By9i_xMy{;0410<0G46mY*rBI&v0?Olw}7>$x` zBgXF(!?NoULO|+ugwZ5s%dHL8Z3*eni-;*J8xelNDGbRQaD8&b4NzWn*Df8HIXFP| zsu0LrJ)nyv-J57-I0D_`Af&H`rIugP^x!e+S5s4Ck3LDY!D`zK*lW$5KffVCNwaoO zG3~Hn%~cy`1h+eaaGsO?s36Z!R(LHH+=gu7#-*V5lxw#+EmKlv3Ze$3KR@TnhZ!;+;g=|va zI%D8I+&u`)90v@OFy#}fhHrPZFvEYPjyC)RN}gNnL%)s@;!gs)0-U_H z3=D#?F_o-7e(Y#yf63~=C_DxWEqQ+i*5htL@%IrP(?(n$O%;i;Fc@A-j<4p`8R@rV zEFWQSSfI;+EqU^iozmQ%@?X8CXKGHEk$FvZ12*018(Y6F(U|q=g8sxm4=;Fm?$h?h zdEOm=&YtD<9{nXE3i3 zt&rwJy{+;%>>gtyPU$~hO?!GE8gnxquIpbRwSCm4<&zYn2+bpRu#DSjDMv?#=o(i; zzvWq=m111*mY<$%(*BVBa7ho}y;ll0Ur)g-;5L%Kp0=tAQ-Ly8`*T2d*cWJFlVJdW zN+bTS1b5in*|YDWlT!oARNH{n?|OFs`<_b2p8E6f?9*pt5!tO~oxET=amL|A5ynrl z6u=+ZY$|_hdf6&-#BqVidb4~!X^YbQYQxPESc!CRDS2Fj&dnlMvAaWhb#94TL4)+t!PaFq$c5trxV9( zgZrR8;bLM^uqB9CLE@}~vs#98Uy|YON&>>vAReBb%YK`63^&?uxX~^rZTq@3V3s0o zG>@0_xi=d4k)15W%UWA^1YF@rAU19|`uEM@1tCNj#xg$!06C74RtsfqZs)lmYrJsf ziVjkDxa1l)qQ~tcY;&HWdd={@7K;h7Ti)g87o+cc&sTY5dqr_3JD#8Ttc_`JW%$IX z`j1{6>c_m)7nbQJpTW!+yyG?Vo~rFDwK6SD&``{F^+5Sz&*3|mQ@H}s#@J&Z0$P*^ zh;1my!NM#Ccj#{r(Q6K3ptVrsv9c{%u|UK>hJ=6oP>AdvzsRp!y;kb}@>=JVzUC(@mTK77GW@j>%Q`hu^>71iRl$moGDo>Il{++tqhQ z=OoR~N81oW%Z}DB>G~Lr`X1Vt8r{F7R=U4>^~xSCy6^bzA$~Fr2fMW*(?J8nb+ie1 zh)_m>?gAOc(K8l4ra)WQ-|+cw$1L1H`(CLNvrXz|_?XO|H!lJv94XPUI#B__jQ~*r z-I?6|#++>F>L*ssT?@m8v!(G~nWKYID2x_k>gVk54mPsx zgU>LV=q|jicEQy8J=6`Rar-OZA;Ru)b4wF$L;^*@R2J6<+7@d%Yv1nH)YNp6P}9^5 zq?Pb-8kGCemBlw;gfA$c?;9Rr-Rn^k;o#|+>0_q7K)SLdAlG^<56&?LEbU7>9-O!@Y~3^CciZ@u!i z^xjhkU${=nUUho28;!^1v?2Z&m}&EO>bufs%JTQ0~SW?2R#yv#icATqJYC zMqkCERGd)+&&%(%p_^EApmDkwG-#-*lq0Q->oPrWurgz~Qw?Q?C@dLC92^7BY{qR4 zF7ZA#F$Jk2bZC*VCr;dCue_XKs}62PIw%WF6i10bD*9||E6zN!Y2EgSJhYiSJMnof zLmN2;ZmZGV*sX|0>e+l8&H*bQQy)gg^|20#cYM*tnMMkZE$>AF4QkSi{5F_)E06Y@ zFhUn|&(Ez|r2vf0=t=t0@%Bj0092ZmhH#xm{P{RM&{g%0*VOICo^P8sxh?o zNCUGlBbm^H?ZLwn%FPc|sz@Hj1+wK2?md-ev>Yxooha^$mf#~j-#0p+mXE(W-VN^zdx*~V0NbWF(W z!~wz8SD7Z5+dyy1*F@t$KIoK76bAasl^Vwj;tlU=W$a6$?&Aloed zCg^RsKV)_-p9f6>PWSq+*!IR?tMHCu&paqCqN!!DCuk1r*|Qpcnk zbzR+nyo5`3ILjCMH#cs{zhf`-+pHUi<}+&M0vTO?u}xg)$t=ie#`BY%5fKqL;pO;h z=h`V#rrbw&F543W7yn7~DTtDSF<1pD{7yT)UO@kpV$8+7?RNy-&+3U*Rq9Bdy-Uoc z@%71-O0a^P^iP>0g`c1~;tiPNBL!@N1Y-TMt227nuSth(7<)!LCB1}%YYVY^4Vob8 zl6LMqB9sfISP+N=^0B%Bwd5_JEHDhf`IMhT3)a>dbArAudd)Y$s+E6)6de9@QNSv` zXcN71@xler0zk-yqe~nPf;&JizMVxI3VRuOH}guD+Bw)YD@|9l0rByL&u%>n%_rG8Yjp*{_55k?+iIJb=c9wHz?(#zY0E=8F zQWkTnosdY$;1Y(9nvAHh@EdMY8>{A=TGckJEhZtsra?BMa&n(3vj?F7553=gTj$mj z9P5}gCe$!<|2TYo38>Myy>i);S$UHU-pEYY!G#)=4@0Xc1*NqVWqY8G@2mQ+ybt(b zhA=nUcHq73@Gt(=z7N#XfnmvpJAdo8Z8w2LiNuJ99hO@K#+4_dQ9Yg(1E}sk?kY2+ zJR;AIzC48P9FD+Z2Bk$-4#*Qf$C;dmVa-vb#Fpw1NKBbFZ5_b!eeB%^@<-Y5t1psw zFZCZ+z^Bv$->$>7(o@I5bVY)3iaQW}-yqqp0fT7P69a*do*OS=Ql}66(ttKNreGH? z<&QXeZobt(b%)9mEG=+PsjI7d(>VsEW0cW4jIokLhY(dm^v`+_{4~F_y2WW&6J?Bx zmoFm`MA~a`TOM~?7u*DHV!O!4_te(Za5)x=Y`*pJ@$m&fIJ>YZc5U5yHgfkJFLcQ1 z=i^CrT0Rgf6P$VCf@D9h)cF^i|5&zW*0srkZ&s*o80W+eE=!no(rd++LsHp%KP&4* zwVvGce^iV6KNHver>N=Q3cdKK1K{Y@8wT9!2N&P?5J4#bfvkl8iT zf@Ac)LQyZKl1yyb7)WJbs0Jr z`76(hbK{*VGxGs?moorB%TYMceZ2n;4D2Zfxuxu$66U(E6V?~8l(FEl(gw|Wy2-w+ zl7_w7=_MfdjNTu&8LX}@He*%On>TMx-bv(sr0Yg7pd+;vNvNmJTlo;QCnn|boUI(mJG0yee+1kUdGcT`wAUOr;g*FleV%y-XYV z^q5$WNXGWR1iwqN^p|ITACgXjeLf-M(Eg=Lj{->wK=Sn)HweffybvEwigo=(SVR`;Lm;f7W(8xv zq~xL27SPK?ZDaL0;tG($7@|e>l8=ursv|vceO2BuqTR3(X|R;f?I>h9>9FlffxGkEIdks)P3lu} z+~VkZI+0oPW(rt7jro_i!D;D&w-no<+O6wKOy;PI&g1=BGC{lWWIq6NlB7Jb72dtZ z_^aqyEk$!L!7=|8d$uD2mf-q~#nKf-`$%oa__>crOrlW$ScNhHGNs+BBbOMmQX@q_!i%F!eY$KDs_V*>Mg2~un1kP5 zpU*oScZ{|ROqWsh3rA$8G64E5LvoH>Sh45+Tl_gro8S;DIJl*BJGQI{1CYSBaYx&z z%WHh^`{;sMiB=iJ{ztmex7sjgm1vT4#20D3LmlC)bqfnwj0le2)XIz_IwQAF)Euk%&|Q%MD_^ zyO9xnk9-h{q5zAQ3-8i|CshXGZM|jrdfySU>FNgQyh2V+4>Z*WWP{zfFDx)P`~Wz+ zcLA$cF`CL^!+->1P*X}xca;(q&0dvLWR)0&jt&5j*x=KDns~$4;UUH=?M5GJ2ttCx z7blZZyipVGNtqL$ZUdM!fqQd`S%d|{l-6is*I+xKx^w4y+&qtADA0^$eh=`t7q;t( zmh7Mnn%L^QK~ocQB;K6&EyFL$p{NW2lgolhge5<(cv%+j|Io z_u*spc2y3y*>&bg6o(`k=n=7Xo&74Af)9AACiqS1XlSsVIj@~pX=#x)12W_?D0ksewbwCoQTGbeUa4TQo&Rs(*tZs}~0|XhhvL*t$BX>eJFe zLQ9!53PgK>giAm+#|j_@vh#2dW3L@i9Jy7Hn=1vW2QV@GKbmzjK+ZSdq1Z%1@{B&fF_xFc=bf#?JNg zgmMY5@GG8tpX>%c`aaGAKqE{p44Y61Tfl`{**D^IP>*%VJ{)(0GL(oG__uirGtn=7 z1I8$?lNIstVTA`$o_v{d-2VishNFkAL?N3k16F!HD<1?(aNftFBJTpTb*P|AA-A*n ztcxswET&HTa=VI_^dou6-nkaqm~YN`W~Qb$v5kdtPgEonBv(p3=sSYO{p~CmkA>xR zb`><9pT4NUTZt_HB|0wQjq|v<0hIXb*!4aGBe-@Ht-31ZQl-W&+XgS1BQ`b1_HDW* zvEaD1@ig$@ziE5G3uZej64hoZ{YdoL=VLl7@-6r*B)vHX&0&ab2>8P4Y^y|tvXPFQ z#rGjQpvn#`w&0iE{p+t=49816PIi=C@kiv z0)z}iZKC*yZwH-t_XI)z8rYVX2~)A3>AZ3iAT-iMA;zsWhc<`yYydZk?D803!XRFy z2VNFQk3GMg;}vFLL+(S$5u7LB>qq0>ki&}$D7$^Ktboy?qx19a$BsRO(cN)IOinK< zzGxZWjapBLZ;<_y1JRRGYPrgWnQ8p7laRvHIOiRR*3HVwVz6?O%^5q@)$KFxzx+mB zBkgDcu(etEWE zoI+JqLL8UtKrK~K*{cJtgElJy_0e|)lIhn#Q*;lis;gfY_VwNI7=$WUZ!g%_h%u~8 zeMgiUqHxaTK|>Ku6c?J<`Z<0FOlq_sAy8_|flcpiT+V=&yTHBK>yN>r?;&zWB6D#B zYSmxZ^aDHhpdc9qqA>-qW8J>QjiCPZS)7yGj@NvzPPYB!Jhop|gn`xP&!6f3GIH(g?qk#cwO5ouwltCu2ZXKFkDI*8ZY;_y|n%KmReEirhl;E5|H1;A!EGX z3k$;IjV*y=#81Lqo*1x1Mx=f3wBtbq4G9#WSZvXGCu{a+SXV12I&hQ?U%u>6-l5qn z&}>J{90Ugn2p0nxUvgTZwtUbFzmSk<~J59s&m@u zH?xetgMkjs{Zsp;$ut(He>1*e52`KZIEA!>pxeI#h+zPprxa4e`uXT}ydC}z@K-&Z zw|(o@xsg|fGlUCAk}~f;o?$;A5QIyv8QY;46lDqS?!+ksf`lUhl`&CkPe$v>sVQ?C zJ|$RJ8__)s-_}SfR}kpLA5)6b)AD%_Dx)iZByiE>%Lwd5N+3{AYfe*8ktD7ZT0|X? ziHT)n^;uG5tEa2(G3M{QyxPFMrvzzokjLo2(R9d7Trg;|@||pNB{k|(Gt}%+>W~vn9bflEi z)BUS8f=wct!E0=jyE8`vo^=TXRyHdN1RY& z)VZw#Zmal#=3>TCV*x+3q5*U?6N*5zQ%gg@Mi+3=1CWfEESR_+;SJS`>{`RVKf!v& zg~!KE%O4SOgq%#ckyyj!OXXXtQWN)&{-T1=qxWmRFdqij0dEN!lt0MY>iOGNZ?EhSbfv#DQ1kq-+?u| zGI7%co|zN*+oyv5yN_$6Vi4KM+Xw?jkd(<^6;B~4Jcbi|>Egu?xJ#N0+9t1o+kn{|#7R4Xm=0aYl zNJMB1e+h>}bQ7wGC9t!ue>w_x=WD{Kd*|pIN<4wqqX}OWY1>k4(;srA3=^woUAtBS zSOpBE2Ey83Dm&Q=%5XHYFp7M=xkv|CeoAYAx3rX~$onZVj`At*vzjp5{;Qy=1uJ4Xv=jIitU zMKVbx@YuhAhra)@`zKV<;8%JV@~$Lt6bQ;t$rEOfHdL3}dwlA}Jk<4&uB=p4G{EEg z2$=r8H)0sModYn!iq)%kVQj+;YbBI$I$wFy`kuy+|!x9>nTr9d3D24&c}h8PY20{Rv(QzW_p z0#ma2h!t6-`&g{h!wuzo!E4ir;9g*I0cpOQ&psgI-S3ccA`ja|EGMQ5Y{LWHLCBx- zFZhK+7i(S=&S0lbRV_y-i~;Mr8Ax^?>Qz(&62QX%hgk~Iwc$Gqe7Ukv>hM|OHet`^ z#O@P7Px66zEBA^y8dG(8wL#KuBs9-)N5$gH=JDObM9S|7bA+eJF|y0*DCSpy0s`dm zqD+r|0zbbS(E(^wG#%4aDY>_Y*YqPu$JS;aLj{=fAP|A}@Kv7cyqMac;h+2+*|jzz zm`=KjL&W|2=E!;E?Z00|o);;IfrrjR+?evh39=Lm%zMC%8%VVWDyCD{HtH|WP4niD z;8Eg|Ystg}MXR30F5j2Z%wF6X0T@VoSlY6DFRba$d!Ph_hVI?)T6K!rgH3>;u=nu5 z^+<$+C-Mlu;5ZIFJ!Dve&7C)|UVIS_034V%L!kGb=mSXn@FXbgk|+j{DjKO#a<>pR zBHcHThZjHATSw^*a(Q!*)&Q!zTTB)S2tUvGaM?EU5{aJ zqEz(ONr8SJp8Wp7!H+3>1@)f733Wa4{D8Dmn*rekoY4~C@H-$mL6NIcA&l}_NK*9i z)%Jnby_B{XRfAHJF_3PZ|>-k|Lbcw4y+i>)nCSXwTqEy*CC2wm~TADEp&JkDgYZf>c)*()T9t z&>x_qExEJhcL61ry{v8wA2^2mzX{0dPeH|pV9mf!ZNnCHf$FIn#n1*)8od%( zE->E*7|`AaH@QkmN(;+U1{MK%`K1!B-gn6lMD|UzDX4b=0r?n=>Zv48^do$tw@|k? z;#7mJAjodh)rKit;Ddg#+sFnEkVqJ{ng^tZGI5<0^3fQTumewzTzs@Q@-7gXcF+dv zQaC9_(KU)o+^PaNKMDBc9zjOPp#~9OkjywWqDaQf`0JYiZh;IEeF(_eZKcbSk_~|I zx!=TG9Fc68V_P=it^{b;#J{Qmz82Ft2TeA=-{HoUh0;aXG9(^Gta9g;z#CiU%mE6H zMLI4S(t-MC5UZUTLfOx+M9OUNoePx2l*hEQixosmKd$@4HbK$kW;3LjbJ%5Xa} zl}Zk42UB$QJd&{Ez_vA-<67f2Zv8e&ux`Ai==>D)Z+m+ixqaD+e#gEgU2!ftYBE~+-5=LpC zstuq*vKd02QJgv2BZ*tPlH;zQ zb~5g-8HG%=lhcTNw zz*jm%IW5iFrg~uK&I>U3zKdiClP&i^I>dAG9IUTkcwvR`+M6^C%nVUoK_ZHoNA6Tq zJUMhG)N|BXL{zj1E~%^RM_Af;58t8Vi4_j;C}C*B6jG^;i6LWZX`Z}ZSinFN%m!IO z%p0loD1Z@5gLy=kyAg%7Q6EGQyxqo3b?5V;Pk*_ON#lx(yRirkI4U8;{YB_x-rsH43dCkaa6fTHiWTb z&(NhoeGsaus&=mC4-#va@uq-tBN5&pWopmmLER73tW;q%^HeoMMeu3kso>zdhJ3@JK)UQZ#&Q(lzuOb8 zQ1=J%roJu)C&5t^;Xa2g$>E#w4v#WXZ$g&NpWpgzH79gA-AI_SwI0R&Xv0)ps*xxU z|Jb6^^C!mz@X>y_-?j=>WNP847{mbjeCIxQlFy+c9SK|lXmC{+h)$ETJ&Y%WlyOQ} z@5p8nxY4DJIqFHHBki8oF7Pb*+Ci>gGoJt)kq6gtU^T*CKsYN+yh-ngtS7&^iW_W~ zJf!DF_!U{7B;yvcA~0sH9HUJvZz=jga4QVXF4P z4>SncfJm^jX>|(Ip~%w*WEliu{~T%zPxuC2ItP=fjTiMGSfa`XHY_!2*xK5rz-+x6 z*ZoRpeWTN|4u60blQ08&)6uo>>yK}*Lrf{xK>5EJZlA^28jrxm%UAFE1kIB>@{x@4 zf-ZU6nzPjGfy77FxEkBhok|_$&}R81+(g?anWZ4LLQ&l6+Dj1C zH4AWlUf(BKpB_}}fUi~n7BWd*jP=vGvw2b#rj??FIg=7#)ByAQwSylfci)l)wGCx`sw?(iMU_R4jS4FE$46NyQ<&j}~MwJt(7p zha4P<-IUjd<7C_Ufg%_g?OMAuG#o=;>*?uHgCn6~@K&6T0Jd_`d#y1yj37`Tz75EK zFO&@wA92mYRHn*g03}=0qw0o=ea0(KR1##H`Y4lllNFB2>Dg7sTrs{DjM+<>8VDv-XILL_z&C{rGST^J`xp~;s*eA>(uTRe{0!blu zIqNPS!gt^!JN+??-Hn8x0Ex901r21pIIax?S(V9MH&I+bs%nHf%hdw2JrF~!N7WI5 zS-jAL@CH(29i*Z}<6t#ux1a`vMulSN^Yk8$C?;y23(1iSmUmDE(t1h|99o?`v-n6< z8&crjP1&5U1Amt;J`i}S<48SHfVXv^cS1ch&^c@aRiukPpL_$L_3inHjV$OtYT_x0 z*58MAC44R_L_=V`9C`Qy76${2*h;#gbEVlV)KSQS50we3DM0ME7m$k*I^y%p3PW4P zn=k0KAeOP#Lu{Qm;B?3ho1{FM`QWgL2~|FPO##)HFsPsDap~~l+aW{JtIu7z%n$kt zNLd>>Wic{w5_8ihQCHuKl&TxNRIrmLm1nIgWg#t<%ef6ihoD!48c>z;4mO7(G5Te| zQd4%6*n`;>`wByqbfOPP<2!>TCR08aXp;S5cha8lU{}z6Wa1AY@1zPMM(0k2bi;LR zc5tFGHi0gd%pYe5;C-HX7|-ou^tsM`iQ0g-@U^m77M|+@82pIu!F4kSV(nok9EJky zJHsa2a-XlS)-T<;7DO#IgGi{LOk^7n0%XY4hiK?J5(TJNu_}U%jKzF>cTnyW3&zqP=_HO4|-Kdf`Sy{AJl~x{{`?bNE>gwxB;oLO{X}UE2 zST7d*RRaQmRnHJwoP=+JcCL4Bu}l2+qf<*T4xl3k!tRNWLPK{ z(*R!vD>@gMj$jnu5!6%i(&W~zEgEQFYx?eF@U;djCTyqK9szh-rtA+%wkA}dNgTvP z0o%{}Kr4G|)oo}m)U@T;`qcU>#LR?}arV9^`^6iOiP=X5Di}$jlFJ#E!GdkwvBQ-= zC@r{MQ7ZceWU~r+NG`K2hQ56YguFR0>8!~Kv?LWEc!nPRLnMS~gK-p%2^kM< z1E8S0h1zX$v4T?zv(ExRfOZwT3)bPX@iKoGNQkZdkS3%?vOD;zYp8P*a{IGs(qP zj5REJ2ijF;4AQJ1Qd&thv;J8{XJttQ?#k#0JJL(E6`+6A>W zlR|d^Y7gyy$hslDwsLUyzhKi@$Jm3&g$BJS-2x1ss{YjKXyPpeE$SLXqKI`#anz}5 z>3VDHVg<+iXOdtl1efCaFNWI_l|?RG-BXl4vTG8zP1QR9U6+0~I#e9kwl(Ut_NL3}_M`>Euy7W5*SsMhSGXmNXj!7(}7- z-NfM2*0X4Be3!f!6Mj7nO6&bEJh3Q~;Dp;iBW?!WFx!FN7K^%$PTRs&xW(Bv zz-GQ3Paj3hP%CaWee_ZqTH}#e>Lzun~EGaRi z>5IlzF^%A?4pTd%2L-^BAw3|+KSS70&@~x+Q{`ao&J?i|HF?SWIjxmlN{Qg&g+kKx z4kIvF#f5gQIA&0XL`NF*>pGC04n}>n9)L7v;Crd)jpNq^@Sa>!1vVG0eooo1FkE}i zQSmj{T_?()<^Hi)xzY`@Hzj%$vz9|CmX8*>Chr_Hmwt>+r)Ao?7AQ#yADWphNL{N4 z5msQ1OoVS@u~SPzKqPe;xPUM%04Vd8I$G$D3V^~jEjF+50Md+*+L{%%Q+Xzko!qHq zxz@8?pyolJA6CHX{>Ou`Cs^fO0`1#E6;&? z-o%vUnxX5FoUCQ#gSKf;7_DS(JgiM{P{(U24s!v@X#b2@aQAa8|`=G z3%E(eAvahWLd1aQC|Hqi{iWsxvp8yBp(&BCSA^#X2(-#*O8C^UYpJJOsNOSikgK-+ z2;MWW9$WL`#fw>kYu9CV9RWqufNmJum`n^@uY5Q6s0x3V1ET@NqEl4hty|_G?DP*` zoSbzr-oCLYYv;Qz#ti35U4H2V zo)?#8tSTQuBBsWoE}wIM{gvXMjFKa+fK?2V^s9w0rVapMx)eZ8iWFrJxWRXT6^F5SmYH^~lq;06c7 z9oJ1N^ypDi${}O!-QR$a za1CG`+7sNy^>ON89H}qMx7jMC5i>A2q69Qvb@Wz_g7(}4xglvJaobZt&Z<4tIK@7a z%Ecs!LiW#Q>Vy1vAl#TaHewxWN^9wxm+H_g%AhhR^&c6u-I!x$X4Zs?#Xy^y);veS zKp27{%Z1qjrp2u)Y?b%PA(a*#Et8v+ha)6zYjd?C#pftM2l2j;nOqmfZMYNTDFW06 zYTC}%-avt3J1RBN`svV}96b>sJa;*}rcgE)eO;~I=OHkZDzpO@B&8ZP`ZJ2KAEXgO ziIkzXJL*=%le0jlsYto;4t{GyPzjnOAJXw0e>DnO$hPuPD78zV&L-^zwD$+_^}&Eu zy_3sMg-^S40+l-qA&O)fsGKLJ{3yTq_xE=Mo55}G2k7Or(yNAq@ecL1t5`eph((j` z_hbpQ6(S7XC>?x0=K$2@n$abdg;GrfF6v06RgX|oNAo$-0Q`?iAB-Y2yxM~RSeB@@ z7Z{N(xkUlxm7{Wy^`N5>lz&}ZJ|dJo6tWh0YB-TW8Dpca$QY9Ssm}|3rK=g_ukdX) z|J#vkSFhfNibahTKb(xD;gaHz0Tr0rj6185B(vBWkH{X|9l%^3*Li5gie1#C1&?d8 zgxzOgdk(dy>6V|LZp-G2O~7UkI&A&g{5jQK1j|sCkX>Gsm7JjJbx(!LaM1ZR$EhRD zLi*dNJ!(L|r~jg8X)Ke3PaD6TWLl}5{Be*fIsovH^s0s$FN{Scjb#sVgi@bExmM)z zLMz$_PGC88pri9`=4bxKkcji5vItUUYO5o;4DD3hWJalRcf`7qu##lZ$|^=RLD6>y zkWz?4r@ZKw8<*v|Eg4~zx%{+$i5mi+mwa8a%|q6QfjHjPvd5$O(!C_x`8$H^aKd(D zO{r`{^#9gDml%|8mdHilK1;Zs(yh$V*r6# zgn_Cj*jeWQ=8++Vcf6%wL5q>c3aEHgCW}KH1!9E^JStk*xrP9-pbsw>YiNPMD_-zK z)@gRAe%phEbq?bEPIeG??gfVHJ-Pf+-L0jD?=M=m-QHvcU0ZIvPl-!!;qN(H-+u!b zDyCzNF@VR=t$Rz>_Aa(E5ge#qQRr|q`xF>nX}o#g!`*i_Iw0Ihqx{{|_;CjSZxKWS z%NkoI#v$27{0yYY$BOM9Vq%(_H^m-z8S49VML#UlW!QEv&urc#KX!DB7Mp9nd#*M* zc?9uQ8nKdr;iTzxY0Xu0a)M=6ty)#6FvL_tk7iQZ66S9^6#gV~3v*MbW7z^Kb`t+U zF#Z_9#1L#7(L503Jb3u=eV(IQ_JLWM}tYk&-|8<90@W&9vz_ z?MMwswbT7(j%E_=qq#>U8XGVAec4?pOH*RIYkgI?>R1SYYur+{KtG0Exr3B&lLoa@ z#Q1X;vpSXf_<>KaU%y_*I{6IcP-@HtU#ElTK}BW|Kyq`$rT#IM3amL9g}2LjF4j7h=iV!GpM~KsR*qy{Nqe~T4K9--*a>T zkuDmCYvI}>J45U3sw3l|rdVLY9{LFWKelI`xVUgM?R+!F_5+;NbO$f;_ zZ*0QNuuA6evuDpH2+Yv*y%nfxCSe}8|2pC&asf>wHh!Qt_h>3bV3tzACy)MH8*0SK z2!`7djdD5ZLa>>sJ)am#YRnMCP@mb2u5F9fT^bC3k87n7YUMJdY2Vz|e#6l;{@%)u zY9Umw@`c4&s>iEtrn4P4!`LhMjKSrnJUmVP&;^?sp)IazS?c z{NT$82qFg5PK|4AIeMww%r`S`sd#2|jZIsWcEQJ|6eB_JfW$n8TzU|BF`Z7#aY<5K z@iOes#mbLix}aF5z!G6Ah4DL$)&(*=5%(p#-=lolnJQ=(+t(nVfSZD6YK}hB6$Knn zEs;1Dv^+#$gRg&QqL&j~8rJn0tSp3gTS(7HkCVK>G3?8SEQi3Zz&OZbTe+21gKdPV zkMT?-f3%m5cz+;KQs*QO+rHPf>Yb@lOEMj@ZccNx_yfV`0W3idkUX4*Zh$9`M4rG3 zM~BFFh*iWo6S}m)9CF5Z`+Ck{Lpo)ce}%SN(%wC1`c^RGuvQ2~Fh^rfqqb% zaf&2WyF}iW&FoS0ckS%I@QqaEht8ig@>O7-s5S=s64D-||K#AN(0cxnKc-C4g5(*! z5722GgQf%?ryr>Ng0||9(dt;V;2aa2rmV({GWfC>;u@o8hOF0HNZy(A{oTnK}I8liJmKLZ@36?D|2`&q%Jz#&~^5tvYnaw%ZprPUb zSPV6NFT|ec`lFK{*UW%M7E^`GH}`AUa(a_tz0`!=un#-X9_$Em#M}YF-b&=9LX9oF zEFb67nK}=5Z-TNVtgB~sb6G5P7EDbKK`fTY9JWIdPHm+Ub9fly>%A1raS3;`D&MzCl6u(@vPoVAme<+Mt(pK7gNDZd?1Z)VUJ9D!88h6*iu-(my?m>z zXylg!&@xr_A9s=82L#MUn9)6>gz4s1I#9bm$PoA4mPTvuAg^ zWrb9PQFemJ`T53bJ8_Bj)U+u6sb$uGZtN%_I*7DWkp z`2(08=qHTkm;)%vsPOVWLn%(>XSEo zm@Dd&=+La|t9c*om}j`niw^hh-VN?}4p=K!9kh!50O9EX_%|RZkeh2AP@_OUg53%x zt{qhtX_wM5&f`Q99x>G+!fg-skq24-hoN7)vo&Z*A| zNM(=4gga2KO2cwRY52!$i8gaT`>@?fbN#;n;k$BB3F2bCvIf14lq4RH@XOEYb4SMbD#-|S6Kem{KJwL0PCYL z5toGchDHK35Dd6d_RK=c(bbf422vnai#A9un&xWn0U%3Vv}D`WVu2P^G$1KaDUlV9 z!x)JOF41F^*0CK@Epl0))?sB0Df}CbS2BT)@87>GRyedYLt+UG7O=T56}itsG*gGi zzdVC*AouG$s#;S|M~*X+dX$;O?73kr?SU-QRTB{uqaoMPQIaPg1;^GVyTH^E^Aaqg z5CjakWut&=sr=eC&9bp2q>64Rif9BFj4G)7uoWZ7KY;IPw(Cn0Vy{7V#L&j&8DhMC z{&p>ntz<_3UG$edqW&%9INW8)kX1rLh|*OwTDAw$a+hTMfQbfmv_aEL=bq_o+j|d3 zk}a8u;}VH8s4&L_zDtkk*+VVu+)MJJ6l@!jkPr)!fwBkjo_naf4p$mSFH_bwfb@t| z^5(KRD^CHlG?T;&ZBfw1%86{q8Fq!{40v*XsJp`AoaU8Xu7igFf2oP_3cs8-8ZT`J z4v(OlY9Tm#T5kAg>qYBw(nc#Y8hCjdKgHbx1h$7&1ll;t2F$%s>ZRlxWkq|}dUGo> zXu-*HK*u1d`N8(tc-*x1{y=CF&{~_@KI6+kXnzJ-HMJQ+kGUHMkggccAY^|ecI6s9 zHiCL{8c=`x_U(IWg?bR`q6*4Of}?>-ECd|O8c%qAF!5DtP~1nNWZXSXD2nVoaiYa} zsF5!|JUqNBVA3h=0*E>%5_!r}H_B86HAwz;@#=79n=)gD(`YJcZ$z34Hlu0%Iopof zfAB73R7HMbdJVX>90UB0P-it7pZ$*D%M}ndG)QD}Gy+>C4?GjA&fv{P))Z1fx~z&p zwEJgK5kZd{uBk{jU>#IohSGhbZG{iDIoav=A^qZm$Di;`z_-G>x;oi1fRjSfF|!9z zy?c|Lo$_o|F6hA6-Ins>y-tW{g3YQ_@1s(joT!gSo#1OPyJx)f=bHg^whZHA+aBM; z_2Z+yrO*ny;6xA@f`JMisIc$CjcCaJc3C3jEx)YIiwGm6z{fv5xf4&Bf0tW0#f#q6cr$RFa?Ma`=kueBrnVPQ(ml|fI!e0!FZSL%F2}rYA8*E( zF^qL4OJjH0Dxo3~S;lge656E1D5)e$5n9Gz3}d^fL=j5+qEu3fLA28%Eo8J&5~Y+< z-{Ux|nYr)#`Tm~!{=S~)zhAHCxtY{;o#%OdKJVptAII^zkq9PeCdEoy%8WS%z@&V~ zK-_)Rx?R_l*=H7Hf*jgn<>ubLTeYzH8i))`^00)d11)tN%B<3L^TP{w`BQ ziKvJ}3ibx>)}O#s_6)7HBo0+zjV7}S&Ovb`hj<4WKV z6O1HCt|})_6ugp{y==wq>v-Qrq^KY`o>8X+va&yBrLqbN+IZc3I<=v9PHk~n{1tKs z`P;xw^qI9Wg;Fz}d(-;i+L7Q%fBKNzp@%w?6(5fV1XfC_Gxmrss4vGG+ut2`-3;P2 z;VEJUXe)>=k$C|JA4%P}1A&KsK35)#RbsH>xc++)7$xlAsm4dHpeq=m09oBY zU#1RnPcWYcu`hD$2;u0TXL{c8&g(Ese-Ec#+418SL(F{RkAvU8$5xqLfEzWT^aeBp z%pv8FhhO6J2Q)>y*EdP6Tgh9c;-H|Z0md#ll;wbff$+dNeJ(U>x>uGDLqDQHE(wYS zi%- zL4rd466Y7#RI(^%AmqxW&I}$nn1c8%J~Eibr|V!ZYJx1l;7T?hAY;<3IYUnR+;23Y zW=b#tb$`SjByGDYcOn|g8Kk=z$FCwtfeIUkXVIGs5WN^pG(F%rPa%Z5MFny~lyv#H z8V5|=Wfc`yVW-lB;2Dg1)l3(hET#?|GeFo0TX9+RPFx|SBeUkWvbBPynw|c0X)qMX zBIAQYvDA_!&$fsRa3E)B`|xiW>(=!8nN_FA2#qB>$>x)gXe@5=R@(z>j2L- z?%XNmYq;yLbBgAlu*5@0X%dK3XbqVln3-7{kuUhHVbW$88gTgY`IZa*_8S8z0w*&M zqAHwPdcXAPf_W5e0BWP_hAa*fPsZuM_ck^D?Nr_8&LrXEpuB6CniDU_htjaq^Z5d3 zWV&Rnudv~HeYfQPW&_fT5BI~)A=^eG1@FeN2}a#gZ)H2p)%j4mzDCd5-~8v1#Qfgq zUH{p+yZ=N~=CQ23Cg7u737C;0&$@8oo%THdHRe0$9WXfhm>~CdV5h6To8)=O&>ky( zczX$BabVUh3EwMV?uh?OPWb8b9}_?SYNNE_(1=^V@9NnvMK*HUx}Sc&+FL$JCIS|& z99!+0Gik2I#@_@ZjseYkr8hqRwb#1x>zW(T54*fc?LY}&NC9D8)p_61c%9pZJqM@A zb-&JU2G7@retsk(B?N8l_#4ir=lM@h5iYma-lW6R-*%sZDSYc+P8|NHJO6h)S%eqj z|NP65tOB_H=Re=RJ#-wcT0Z+??vZ-nP8}CHU>QCjB3Rg(la-WmOP_wGFciub#sLO$ z4EbGm-tRoH4M$1m4MPJx4_v8*`Kr6$2XfPJmI7xBAZjN>$V^%Ytb!`j;}FaSWCu0NY$~s|->I%%g>ylwMgY=V;}fTdgc>Q-8X72{ralwa#pnB< zZY{v-JB{;`u^NYXqY0p(A}A;bM)8}1G?sK9t8{ucI$M20DG(Ob0D$4ij;8@>Ti|Gb ztvKlc|E!I`o(n&yi6M=y8$kVZ48-(R!2w}8FV5B!-db>&-o6`<@pv(y9DO8?8Gu}A zD1!~TGw1^-kd{bToCwVO^nSI)95z7fTt7J=T`WVKdYAPRmw`q+^3f#pvB07jq+tfy zus3R`R2?EDw-%vuSjp935(;&RBsrnNYGZ zOnTPP5bh$Z{%zCre>(84_we8J*1yZ&<(|@VB?9;<&O;lQZJ-1y)gn3%z=lH^=*H(Z zxhn(kkeHOxZy;}rDH2zRFCX*?j|bM!{f6rxH<-s5k=9GKT)YBc859f;(PhWLt?I{~ zZbR*HLth+5pyWXcsEKHp*!Fb9^L_MmsVz-5rNhbExHnGetyW3m*@O51q8= z{6;6VL_9B`6!f>N)x6N#vL8IpCm#Fk?eUU=b(R`GM4nG(gMeY=tC)@V`_b?Z#WFio zvM?*yKJ_0p&zQzVC{;|VsYaLRV)T2w*g?J&B<9YSmD+Z(JHmW%4FI)RIH;NUF^N*` zU{hgT*irIJH-VQJt6+TGD4aP^<1^3MNJ6FB_Jm{i*PaJ?$Izj^Y?n$^JfzCutUbY_ z?peeWn4b3s*&{70OCKm57V$3G&!2yV?v}9-9DD7&y^CwJLsFLaYK^+AocL1PIQw&1*757IYi?vY9pA0~ETWI^I>N~JF7!t+ zCHn~ksZ=mp&f%Gj1})8TH?V-|w$MkmfMFgy_YJx=i3qzqIO=hI#+tJv&lNS@n=>Bi zcB|Jhkl|x4BGn+GZkHelPjLL9xkFl7TCXJu5v&lPjMA>R&)W)IKN+}B-y6muK8Q!2 z)>%MX&H*5rQt-~xLfgau-7Bclum_oi72Eo_^v>BQea8&M@Lo!)Ivyaaep~zv*JVaF zG5I+lE#VQ40~8V1Yb*#b<1rk~LbGupb7pa9S|YHMQ43LYE4fzI)O{2@t-mgIpc77?@K-~SF9cgCBx4VTIj8(qE!%*BXhdF|JVJomK`JoZlb}d!jmEC3 zs&nnw({J`m-`Aj##H=BOpbdgr>X=`KOGNPyh+4-DHB9Zg?K;ooHglGApgJC> z0iaxwMdn#|)Ao8#v1C#IxymF4JHt0+1)P!2LwY+I9iw=0E0&cVp5+9wOJj(so>2f2 zr7BRv==Hbv4Y$vj4z}{j#if%2yd`J{D^ZYZ0~gazu>>8fu>|IcULq%S7qUg1Y$E6I zR?%cQ&Met>2lHGr{9E|Y29$jj*!jzW$gPHH5{iE_R0Wwz*}BfJ??C{y3KV-d+t8tL z*z6sfAFr|SQMP@iw$>&_51PZ()0@zHr1y7izC&^V5?RLA6TMHM&DK}l1aCzCc71(4 z&JG@nu%ZD7to~-5^VEsn+ZNta*iot$0F+>!xc>5}Pm>leUMy{j)V1a(Im1GTvh9#P ztH7yt0Y$?LtffuoY+_S!xP%wzdgk-8a3>y^;Z4V{se=8pdfmEpk9OjgHbI@e3}^;# z27_!Bl--%A$BipyVH4ApdcyKEC(y8M0u-!Gco%F9JmZm0SehXjlc!G2)>3(T8eQQ$ z+{|)Za8%ap8b4q*bhAtzhkgzMz!6c< zcM|O+7rzpG!als=03XLRJVXz;St^O^Npu4GYmW}vz3iJ}4An1{tY z4xW{wGRiA%Eb6#u%$X_>H?#o&N5iPnFELRK@p%0))ygv^n57K^HYPK}=%YnL2ee91 zpS3$-lw}XEwaT5zttN|0EdzW!u&bv*>Ymx(l+(IE#V7 zT@+w%3rzX4F;G22k|Q7zT{Nl{8U*GJ(2M)vO5DVspJ$V6v*0e!kEj&csP zO9X$R+{@y8DmI%t#=7O`^V5J0dEz930@ZhSzg-R2XLD_kr0W6pSO6tu#8E<4I=rIa zr#Cy3HGwJVw_zzrg?MxOjhHi2996jKoL)68322Un+6F%I&pyazs=oC7SrHhxw_V!g z_~-lj$hM!i1AtiN)}!Bp4Zn==;8uW+CU{2%wCZEIWY{zxPVNuvToJxwn~&k{`B{^1 z&gV8!4Ds2WE%3_ZBEk--GQ%B)+4Xa6EiSDG7p0uLT2hWle+B?(cL>p^S)6y;#1($n z0mYI8Q_8Aqx!Rh*s14daZ~q0P8D-#MJ79%7!<9f|r_;9e7Pq!=b{Kc7}>)Mixb1GA@*_=@LxHCpm>M<(? zk&5>e`qm+M%18k3Qc`)Uj5JP8Evu4oGpo$k;WjvEW5T0DDQWTE4v*L59~>~a7=8&# zAbMn+ogs%2VLcBBl}Kn+-O4ABo~9eV-f%!Ey-&PLiH-;pY_~;$zA^W zT+kF6WNnbh(m>!m51djFqo0K_4ZQ(gTYAm3F+vY*TIIY#3C=i zfY%-{&pmHMS!}&~|{R28HZp@_)eTG)9!?fg3|DMhBqEv3kB zTL2dpVgp5|$XM$RD@JjM{I6@-fzjouy(44;vtBf@8_+0qeG?vO2vULPljQ5R;<~sg z{if?0+(!&FTv?YEyUS;MJ|fR)rY8`|GqG_`#5^_%S=GU+2MtvPET=O;xLsr3j>ab%n`z(^rj^Qa?4S@b>IKoUuArES! zzVS$}QSkOO2a21%H}u921AfxWvA&=k@~Cm9Rj&V%0zuunUfQ9-N%PJ3J9cO0f0p+( zz*R?y8B$7q)719zc8R0^g$7S!R8i?gV*^$JFU^4Z z)Egn*51+C0rz1O+F&cep*Ha@Y1d|gBjB??9t%D6RVmgv>2^JLfAFO*L-plmX!i7CP zpWGVpok2sW_d;2d>(e%7-`M0IE+7_C^Y2{477?}Vws?c>qmD||{UdZpDH>(^u^p|+ z9U+5$*afz616U9=buRqz$Q0QkWR9~UR+hJGULts)Ig?;5n}d?j0))pn$Q8f@moZx) z^SGbyDO`WP@3ed#;ip+qQ6cRNf8i`d z`;({%DPXvd2$I?o?_Do`)tsBcsD<58i83=4bwL@za>%8+-;}bJVF6*KBqyS<(yJtwucgMc zyI$ocQ z=g(9W`{c(X4Dc*K0J?aCT{3~m_T=t<^O~Qxp`;TQ2-@+d096wlOR_P}Jqa4- zocm}b;zn8kIfk=QCMuu$vQZtgSXVtZw$J=VMtM>MoQ~Q*xOS+Nf~XOFZ}3_(>}pLX zCno^g-e_k1v2;{l92%F@ph_DlRdj^*D5zPVYGe2V$(y$t9=n&S{iaw%@7JiNN?V#^ zQjc{GFf`|FG$Q&ft#&B-sf1e1YXSSpK~#L;lWredeNOkEws+_xKwZZru262LX-ig zM10ay4C*`NpUpGd{`+P<729Wt-wdBMEy8*Bls%v7R$3ZI*^Tmv+}!?qNn^3P#pRhM zc3n|XZb#xl$K-0I55X?mldFe(2x;FI(fx<#*&C(~@9NHl8oyDMQeLleA+9`WK|q1= zh{$iiP?wE;TDtz{z8)=FrqvhFwU{G>W#;Is_Tak+su{W8MwfmKMD2l|dFa3);DgVY!&TJDT$Ke6Pfep_@I zaUmu%20c)SP2PRme2lRQNU{anEto%eSY{bnhqv`zN8e zP8v`VHR$W^XDgSTS(WeDGkw#;S3OnXN}+ZNgTuLfhA3okr{`+VnZlaor7PBxJ=}>l zVJS}rTLBPTpQx?QF>DM0kLVRmM3+mcJ<8~i{B1o;k1mDNm+j7UllYuO!{GRV z|5CIoe?Z&aLYug!Ip{)pbTUd`&6o{`z;U^PU&e!=d?~@$dn$BAe+sW7-0K?l*f9qC++SE>A$E%2w~6&RaB( z3l-UjjSGg|aiOTIq@CfV5vtsT)-W5d>IKiT@`>s#VwRO^fPvU&qli2OAIio#`6onc zw_IJ!9K|XDoTZI$OEP^F2e`nss;oT#j?)&9VL=$YH5LUiZx8&W-R(_r$>#9V6v_%z z3wvv}!`d#5<^C&FAn3P~LV@IqCCuS)ol8dp>B%^OrYfCD1j|H`iHNBai6}np9gfdB zfoqxp3c3}nXmCb-WjUiJpCnjfV<7kC3>{uz0`yM=7E6I^FxoGd@#dpNwV0M|cbwy?)LJ3rlWReqLM%qdJ=J zbEQi=6P$y!*I9U5RRFFbs^ziE8%b8h!h8FYnc!&fm{J}}DiL9nQwVTEfZ4DRodqQC ziy!zIvb#{Ny_TDk?ApQB)csfsS~)Yu4cyH+JpGAcVEu(#P_o8@hG9ow^NU5{m`4bS z7CaYXWCpXd&>BWTN+_Ld7?W|i^DtpV+vi}nXiKiymbS0Gk5Tw6>rfn}Vg@3JTuk^q zPymhIojG`q&AU)$0Bf>Ghgh&|L}`A9nV3 zmES3CMvjw&D)8xs_ZlNg1(jRCmDp*ze5x@uDDuAqg4D2idv~#^0p%wq?J77-r(t0b zJQg8m#4;57E)uL1S=iESS=@@U?34)0EX2+mJdV9pIvOTpyK~J$sn_^?Fc_=8(!vf= z0xS2CL9Zkp=`TJK0JdsH+45YYED6VnEXJ9W+XyHSkctO}Cbw_&5kyjFu>|WZuBdFP zOZgr2Fed&H=soi1X*j#f_UvDQ9Hd9`FR2G4+y~gNK;MU z;r2<}c`1Jsvpr|S$9>Imj78)5-PJ*CuL&gC0rP6RkAdln@j{TQrW>Y#tk zqwdG;&YH82-s9c^8~JcH)tG}Nwhh?En8(yK0o;#wj?^P`}f znjQ+kR25?qw&TF0;!_HyDtA{@ZlBYFk<|*1Jh1R_&}aD`mv*imUsn3`e8xsyrW3_uX6A3&_VLK_gR6{M0{|zd?+TqaBR|skY}| zsSEWH(4~pca~Lmf1u|@h;?|&b^Ff%h^MhqRi}~pnX@^^%edUMc!#cPD(-8kvydAF0 znM5HyZk)RfdbUtU5j{f1?5Pf`2Mk3JO2CwIXXnJO2LvTk%dMzjINV`xAh%;AamC=i z-?PnO88W*jG9jvswWN-FHV_=g3$fbK!aD+hYI0%1Ce&@v82 zcVauX1(}U=6FW3{Rp=YS6=^u6g^>b*y7%ZTSR@W!yX7R?SYV<*##r|oI&Cz&5#+LH zEY>RQ{cJ-2`GfNz9zD7W^Es4g&qfejxO zM_Q~Ike+6Pw3w#9*2W#@DNIL2k%5m`0@Qy><1E%LP)(en82!w~W2uCfJFVw|FkL~J zr3$a9Tp)E*&fI$!(X$RiBX=I!R#~@XQ$*qslz_c66-JvA`j`= zE+(V9!?@0TmJwi{xwE6kXQj%~;E#?r;65mE+<@8>CwuK? z4H}9R$e`Ykd!hYWQKOKVuv&RWwjH92fJ|b_af+Puoa1;^juqZr{ief`SO}jrOmC~uux8*Y&Xw1U_?DU$0<;A9 zIFJ;&MFhiWgao>9Jb-*C5jX=}by$5+{11BR$=^oKosH@4gKU?kXjaZZ$)5Kfk*qLD zcIwor5w&CWr4gmJOUi*{1XXCXUjTGAzGe6Krw^`&zKW3Ay4D>=O&?1y=Un$o)iAMr z)OH}gCeAf$iOr32q|DaGJM!Z!H-8A3XD>98Y%BYBc9;AkD5zC$NJmm?TE+Pu^Lse{ zg@dL0hK7c|SSj`WmdFJ$w!y1aa3r&Qa_iGNJ4AMKWUv^Mem$@KyS?Raqr1?PTe1k% z8@RdAyICcCC)2At*!@TC1Ip+u-Uj+@YzNa)kscJJ?io*7D*Bk87=c3SX|s-4oz=5rna&! zA221*OPAK0(7o*UP6W2Ii3KG=BDija*4p_l0Jm)oftDhL%Y(Xa>G>RVblOm$3GV># z^})WLeXJ5ho2Rk6f%MIs1qiSV8_(6hc0ZxFKGH*m1P~}jMoLRh=RCA{{B79?^iGIO zF(xU0^_V?Z7L*R}X|?3?pPjpup~Igw_CIv@e|`c|I@5%wG~b8VbOpy(AgZ2&1ORem zzC*ghZ=kp9W$eKY@SODq6V-Gy59fI}4Y5TG9__FNxGcO*%@u9!Fm;!`D3JhpC*9r7S{;l3fHk#6U?$m8&+QG-KdprS4I1X+9hGN51=d@z&WNAsSC%y5uJj7F#5X#?}n=MC5RQelaG5 z6i$>wAel+%0Djm6z>UJ=na3M|C|0a+{g}Jzz6q+q7Vv*KUo}Vc4BK3XveyMg@I*2F zOD(fdC;NaIArGEi;LuYdu$i+shlh=DIr2g}CSV>RoA73a?#gFI|IX2U#`ODsa8Xlv zx??rQ0r4g1*PLJxsD$<#E>bfYLW!FdDbNeG`PdsEkvxr46@n~Ic&0eK1|wM_Li}}g zPnz5GWE??>9{tr$VDIR8DM4o}o1hHqiRrl6MbH=d02tk|cm?-ob=a5R3b3{kM!8!e zZ+}Su-9PE*%%Z>_RrVuiU74B2&RqW?*%b)W7L3dwQPhW(nSy*GN=2^J^N#@NjdZQ0 zi1I%XW6X5rs&lo_(i^?`C@j%O>-Y6a|0tizqanu|FieGh!-PomfR&47(Y05r?n69Y z)q)ft5pn+EnDL?s?XCVbpb<9lRrsCxi*m8l11B?mqm;f0aQ=G^P33A2zb&$ks6bN7 zkg*Oui7qf;a0n5zq$31M!6bVCy;3yrAm^H966RlIhlIr%#cX5&AR^ouoU!)2s%A7^ z(4cr8SUfvw9#6q8FuGEPq40?%Q#?wllKV;-EiW|GyzTq}%)_cp&} zZ|*gH;}Rwv7C$IhmSZw$ouFHN5Nhqf1zpLGxRU?A@e3)pv$|;X5dzhKj&hW4TwKA2@pAzlsyGVhn1Hfki=N`mf-<*P z5^B<3!b<(cws29j9 zk=@96FBtZ?J8?n~b3bfrRu1(;`LR>m<9;!pJ4$N3^s1Q_sm|@iQf|_^t2>e&a)X!H zX);C%tv?EZD13}ncwiai9YDKLg&{*4C8*#a6e+mxzOv=$6|b2VDZO1^?A_21QRUiP zksM+g5UOTBBy)whRlnyN;ZD(?n%|dMV{tV|^LnzJrq+c~m{woIsp@`R@En^44QE!D z5;Isd!JRvm-5q;BJRkOrG@z;g3a|pTR`Bo-{F`J4y5CD-2zh)KvL8Ln$p}Jm#Xt55po2`OF-1IAqe3=GGv~M(M5&F*AzKuhZ9V8`HBGZ zQxp22LfKmpu7YvVp0Y;e4!fh@+%VgG_H^iziCG4w&5dobT7z|q)m^g;;mB!Zqe=viQXA12`ZFF*pOKNF|uc~X8yQg(l{=@F{%LA(7f-NUS90=+C zkD$4`bMothZ)=fBS$vLfQb%bnmW)|+Q1_ZmQcESVxATa~vJ3XZbb$qn)#f#oiT5u2P%^7SX5eKq_9O43bmBPo^n!L9l` z6-)1w@p4Tm^)Hr+*<1g$gEHvCGcH62|s z!6kn{RPGxRfTe;nwDlxMTJn7|Y6rEE?FWuCOV zAufUM)#?E$I}@rlONzY16~GXEk))S4{kNiXD$p@=cqZOqqK{(%?5 z%#P4y$BKruKN8UhUGq2(_T#I-3+-sH>(GW>wpD^IesA+zLZDczp>MY%A=QU%L`k2( z{y5#C1k96(px8d$8On93Z-aGtLWBb+R)XVjim3~hv{m)UJbs`rb1H9lci_@;JkHSC zL0#h;kYjZg>=z*vI7gwH%5&_-93wTXbpNc9r9%33m$d&;mHz+#@c;CD?0;|a_V2v1 zWq0Eg`rTi{M=7a?x1PW?z4Pn;eHYMj> z(#X#hD~m3!S=aIUW(6F?7lcGC>T}AoY-bSK7gGiAbCm-hd?w{)D|A;liy)gtH)#5^ zN9}#PK%$#)#}L=dUVxlq4)8&i_Cmqshqw-xcT^41(P|L~PJ zwrxl`?~tis(xdl=;~P5MCydwfbZV>alROei@}B8x7VX{ukP8Cywz6-PZbvNk8<78NG!BwrcB&PrehV*GA_Xc?jSfjAP;3{;fnVDl}f1uPin z-;7k7g}OV9Wukc&##=tHfwDl~$m%=_6RUEBA)&)z^9Py=ik3B31rWRu6>Sgw5s~%>L_9oj(NkxXTokF&jS8u-Ezy9Y@ZKIi6 zK{yK;u@(?3jLgPkE@R~K{_-fj*N!v6{T}X+w-SmZGfo~7F=Vr>KRUl1k6c1l8j5Ox zrwh-Y31V93^B%Ji`J*xXx`Gtt{`h+^=*hkPR(sYMq@-mMd8q2Y8oq5l_+dVz>tNz3 z*R~j+huiKS3r8JdO{KF)8%}}bEDi&a4UaU;|Ck9#Rs_BHbi?02h(SdtpzA0>NkXdt zN)e%4M*s`8^m!z;5DZ>KBx-{=-zrMQMagA4LE6`V+QFYJq>U0k2-(OB%!OFN7=SV8 zK#z9L+sU0$*GXyD)5kW-MY$tNr9EPhEm%RFnK3Q#zwXzLo4!O_>G)*BFB#|OQm(Xl zK{j(-^|wNjSnsZfG=^<8LG11$O%*_ABWAu1a7=XSgmY%Vt(<_1d0J+v6i?JTd+o(N z{QCU4;{|{vGz!8h4@w@$J%YJ|Y>_(P@x4uj7GeoN3WGU2x3|6+y-31?>HpSxakCl@ zL;ZA<;Qw+7pRM~y|Fx+8c(qMlB?Xr8)F@Xr-rU&-CeH2#1CN5mOn|U*7ooG(yW5XPzVJmR+(t5!=w_1{7iLD7XxUkxnBL1AYq5XvK#bY?5$8RkHz9 zbKT;kT><+6yHCm8?lykXx#;qF$)@v7Dq{M$xd%>gT(s!StSL@$(l^h-&`c5gPs5)3stO=?Dm!FMGPW)jjtr`vA}lE1X9=8{ zVysUdc^V>|&bwX*{8iC*6vU1*zPA+YpaeI*N2#X3eNGii7lK!8YZ60 zNieiyE;(bI$Z0W0tlryx{~92M_@^wKRU}n?AtlBHe@fkQ{d=8ictTW&A--Hu)~vGM z1Z@F`=yIR;0AV;Af!UyTI=nsJ66BXIjRPFQ7O3B>=Wef)JK7lGyz&cxqEsmLg2kt{{27zUig?BY_&9v}ORXWJ1dDhW2yDzA&^(Mw{#0uVQd7N8 z7HdI$>Fz5qdvF2-5Hqh7zRlP}Z%sH}0lcus!y3AZ&{N10Z{|d>>JEuo()lp)cd5wg z0YC6he*Le}hW^7>zOvm$uo94xno(WZ;8CLT^8MM69_ZiAm;MiOnf}9z`ro2d{f}RU z<%tFRA%Uoby+szpRwv-f6@bpX5W}x}QYm9KnGNvewma;0&w+#uw_JO)gxmq5u_Ruy+ zm>9O%Mg@a>ZU$XZ(1GC2pO9Lg|7iWl%xO3ypah&eG6{&o6e)BbMuwLVvnAu)k{KZu z6J;`pPynuYgWXHyf9t}@D%mD~$5^HlC_0BRg(#mq$g8i^fnkXi*(GL@E z+Dm@fZ}=Hi zJnajOA%o3tI^6JIoqh}te?<<)A?`QUXva>tM5}~_gmJhfn zKzH*%Uk!PWQywqIBc<@HZr7Tt>wo}Q6%VO9{SZvBI>U;z=<76mG)Kqm)%NPzmK?JM zGM=X+@*IYkZ(q}|zoocZ(T6Lz(WnEO{l7h8^ zYMc|sKMiq@c~;+Cx%+2@VP_|dm)rTvzMKPnl$t`VtzLilln{6%BHQ7fimCdCwx_QC z*HhD;UfsCHWnD?c$2&KJO}qZkZ~7GVJ+rIM1wy&V-KRvvgsDmJPlj!Wn8L+8vo%sP zE?pOwDJ!grx0R2sp0qXRZr5XY>jTi|j&-3R9VAhV!*>4;u)4&noT|EG-1-*2JjPQ| z0IESJPAs}VRetN*;f7s1NXolaK)yR6f0rj9J5L=C*z0N zn;qN?OEBHzQ6Jf1HWC2qBoJ{{RA%{{<0zQomw`)%pTr?Qj~%_b-bKfDM&Rg$5@XPK zi8I;3Z_XT|PEs;B>xN8I5grlYwe?*;56EKMg8hJ_hi|_-kaT5C5xf4$utGVwmJd#< zosxXdJ_^?bH5>>y-uMM@MxARCK|`bfQ4DcJE+D7Z0Cb!{5b&H`^?#dmIb>09OA@*; zNx}(~a1W>?;gX`68Z|6sOuC*s3VE(N2~E>P+)RO8W=q9XS1-7G_+g0^qD*1?N`V`n zk0NHvuQ4srt0c>>r7+vY102F`8CuHmQ5TS^El{AugQy;o;e@s$7sGhUizsAe`K^Yj z5)}xP^Kt|R*+^+)9Bp3IPG4|QW`MdgNW$a59#PDe zFc(VQ;n*>(SL8AoBkb@#7z-#9hXe~Ir2H7m)+lbC_{?p~>exsxDT2KWE)ba4)bPXzr~=uv@%&TMnb3WAmT13T-Q zn=_Fh)G*)g?0GagUc>@L*R>j@vbAJT6-z!&Hb;UGz!;w9jp3Vaq5nk?U<$!2P=wbV z$)0lCmhk41JITL|MdTF$ga`nWm}~&|b|Qaba#l5JK}cWa-)?IKp%G~$Of<06t5nQf zQ)Gk}d3d7p2{q#1b0vO7P=Ha7%lQiUX$@$eMPhnqup!d)iM#F$ns<7lzC&kS-;L0O z9-^j6OitRA?aR^kP?}wdu)Pcohis+1%{ul%eq$3O<*oHp{=ZwHh=ZBgUpjbQSRti< z3Aqepv55B$U6dTR8?`@u?^#k%_nZZk;e@V-`WW_xqExgxF?XjUn~NnlZ3_kYy#}1a z^(oBW!#u3IQa+O|sWz=Id zi7a);h2ZH(g^JK$h&vmbrfe^ziKq)K9~*8wdf06~<)m0S>ZLKuAqgz_Jv&RWgi88Vtss4^3+gyGSi$%Vcvz~KQ&K~QCN<|VLWi+f$fx) z4CWPClG{Uayg6%w z0z8X_5oYC=FF9iZpjvSn8ZC~jWivUS=itqp-$%zFS2Sdb(dmuHR40&nPDwYWz2CJB zXP8LpWnd=02 zRcaHwE^Vt+IgmUWpSRBO?tlS8X#+;!dX^oVimCZ&FwJ+lT#=Ci^||MzugG zz@hNCkUZCaY=r!lRN^81QvQ;i^{KSKCLeEWTY$ zVM(l`MvIK37!yMN{EGa%`iB7U6q&}KVAa=lT(5k2A8fa;Z ziEP&!MSkZn_E4lDpj)ukd5+OsFz|%y-ck2#F08=PISuCMf_Vuqy{gz>NZI)P{J_-< z_S;rt(4&ndO?`suE5B3JMWTwR!ck?KFjf>Oeh7o1wTfLg=VluiyPWSUTGlXihc`oU-;^*!7s#ugL?q zRT$%A3I^ov47Cql3ww#}@UsD-p30cvjcEo_LpYZJ$^6i5CnhNnEvdt;;AAgK`Cn`_NK|hYTo!0e}ZWSxBI;rI& z41f)YjDrDI4g~QmZDtt$Ia5-@QvCyzm^93!+QQ11{-}MPq0_A~dTYYRfS$|)nf}w- z`(dC5Qli*bmo=+=2u4)$B>iv|7AcS<64(Q;c@fP5m_j*m$IgK-1m9Pk9@2cUr8~$W z1aBuDZUH($v?$7^gopAL=4I{Kul$UHdB=fr+(qa>vQXlAfWP7J>qZRX+NVQ!xt}W@%jcr0Y&_W8bM|`YcM|e*I*Wgy9kB5_t5f(; znF_!zn#c*%%LjP`^VwAI1AsMOLVhzvufzpe_z0TtMo&*qnl{BF>O!4xE^n1Pe&gY` zPY~H<(LX`MhU_Wese)`*oC8MrYsiggq+xxTRju!_z;EZE6Ts2DWd01tl$jMEUGm^S zKsgIVb1Wt=e0psWV={qPzYH=}POR^J;BbNpQE1%zS?0Zjh+-j0xY#2_llajGj;p?V ztk)5E>)&ZQivQO63LJ$g|0s?#h5!SOtxD+SVZYNZ(u@@~wI8Mjbq zL}}}Re4TuEI%L0u^>>q9-HdcC_Vwzo_I5t$v$Xxd>3M$)x)1uIIUtG7&G8USD?wQaE&}XXyeME{ z3G6?eC+q`12Z~-{7T?KHO8aV{x6{zd166Em{&%ej9Ei`&jxSEV) zt69r1Aaii`z#p2Q*11rqMAStUnv+#%#EZvwMIfot1qk^hw7kO0X+vw*xxow%lk5gs z11Y<#VrtJIL{W-4!+4lf--A`Rdr0LuA?80s%?E!X|JZcApOnXQYI$f1v2>)92x?!= z5ppbYLMCmPwtiUtucH?tM35%%SQt6q8bodZxfrx`xO{B3EWnrLo%iHcRMi(DB=8b4 z7|kC9x;onjIu9KEzTnp=ysleEDN7=@14#6pyr$v!KEqgDvnf70L7##OcosQ#R5OwB z2-^g`l)kB_*jDCp<>p`IvfpGxeuJ1BW2l2|xwj;l~It2O3NKSmbwz;JcVW zxbIW}LJ7VoK)_;52IJp#zQPmZ59-8$96C`9WXT?}c&ft|(9<-rHWFC?{ z&+Lm^M}CL|=u6KJ+KCXEq5Q@p1VPyCAVe=zi}D3lumzWtBYtZkqY^RT-3uuXkIz7c zl)q4z=5VTJ9i)L@;n(xxykAEQBt3+cGaW)GAhEhn-2=5{auUI%fa9qz^ai1a|JlVEnsV$9sqs#^ZoBa}w+D*;k>v4S$({ z(93ZSowCF4^~OteMr@(2fN6{E2o=m}c=un*^8S*7&Wl$ z>-G#wf?qdssE8FjhoZ{9`uxm9sdUG{bcN(h$OX#=-G>FA9CBiqkfYshT)`67o>O~` zPm@I)b%CrRrJYICCx5-;V?>7rT0%t(DU3pir^s9(Z&khRkN&xcPX5QN5v>rL)-2NF z<91+uC?p$j+DKbr2BZFiHc@nf0+B48P!GD`DrgEcAmO!6Fvf3g-bN4@fQWO=b@Vm< zpr&-@e;orx1qm?`Cy3$ffyt22N#iC0m7(YvRM9#y|Fl6y6DE{@z6*)~?h2!JJR-vv zXV7bvqd26hW2OB8;Zn6xojpGXncP_sfQ0Y!uGj z8>r<_`H0B%{HJ=bD`zQ9mMeL4An>}Pn?(l_54Ais=0vsV0Yqb<^n5Ou7?$pbj~wxA z3E95!+$_G1&1QfSJX(zdzIc>S^xmih^*6f|lC;BES=Tp+zBZY?n=5cGub-I(qn6P+IrrlsVYI09uVAziNp-_?H zH1`uA&gE@$NX6bgk3D4ebKjW{p#PC=w87|x`j=CCKfOG7kJe-&F)8Jg5K7kql{9tv zn0tZYlbD3wD??HsxQ)=f0DVljbQ7{CfewxvLcf3Mc@+ziMzK%-vGR{mX$s6(tW9eH z#@)P)PL@~j*^(F`_;fj=dQdGtvB(ah>`i1}ppaT!*64ovw0!P$;DWF~gFb;`AWcvi zqQP(19TtauF;NT%WOa};2o3HZJ-eu^jpWD?Oea&INP+#XjJMXzLJ?CYCNi`us{pd7 zC1FWVEF0K(G@=_HJuzeJ#3TS7Ds09>~yZc9;zQ8Gyfa{<^5s3xfgXskWeD=Y^ zZSjU064;6bU8*IRNn}sNROGJ8y+geNyH{{Ure^$#Ju`oIJ1-qKbky=W62=a_?qAJvVGgKP|g<*I^i9QfSLGuh^=!XUx@}O?9p9#$x8+%s# zSd^#y#_Bvor)G7$6H1h}IZ{Oesisj z^!SVW@inkFFL+Z835lMy`7I*3e1rG+%(Z9Xm2$nP33g%un1A3AJXRd>mEpc6nqM!K zuKo}*;jwSjLSy?6VeOX7K7@|(u^w}MUAl$1AiZXy^3Xqd@!TBPy+~2l?sDymQm4AS zJJJT1wU8j8hO}yXZK0k{q>9Zyw-6`_h#aem&5`D|regD+MddS-!Jp(2@-_+2D}h~4 z|4hyiYJLcllG3!Zxg6&CWT#=@$Dc{bv2f+2ef*aL+2Fw$owTUX_o-}P*H^6ZNd6fS zbrxGqnlNb8q8x+VvOY~V%!r2|(c*z_j;Q)^ShKmr45O87Jp&VWJj-c#x|z)2xZ)d^ zI_T)%wChjE^-^tdrM%^{Zl3zH$rs6O-!VL}{LiuXWd$-!k`{h)$-gV3tAEJ)O;Wyk z-VD`3-i!ph^gK%b<9IVsQ|)cW9l#~_$$s31Vb0# zQWb&h91)w&m}gbUen`YFhZUHHGgUs6D~nRW6EuA^55-sKv<(qh_gX1{f)-$#C(&Q) z4&Ql|s!#NTE!fH4sDGm_eIk@XejxlSt5BKCzfKi`$LfhAGb2N@eb|6l6@d9=cTd`Y z68lv8e2H3~eMs^omyh|%f9bG1zyE}nEL%Ao)dPbKT?z#~HD*iR>@<+kwXxCd5vQ+g zFH7s#5q0~^@5>>_q_&zKZ)VVfijlSL$#Q#B5+4sWY7Tm$v-6yBse!8r<$y#n9^jt-uZ)FMyW)hPv6u#C0fN>~xQFU!evcV|r3LFJBpvicaz*IJ8abm9plHa_ z^NWs2{adVPPeOX`A6ad6ySZURdV%#V=Y?zL91w2H0RXyS z%|J^MHpIk-KzCvvit}W~3l@w}I;+wHi-I`1TEwiXBDzWjfY<#YLxxzO{g6IxX{gUB z8&;`DoSLD-uFF5WTZ+|9U1HHT9R*bfs1}9{`#3hizcH(T=U`#k{e4)Cy-yQA{C|9`$uc`l#2A`ia?LY-viCW6EsV`IV6$s z8=U%d^@LRE0P(OLQGowt1%;MEe&X>#lLD2#uf1_MDcU-+pYI=UZ}gCA)I{Tg27g`) z6vwB*Q&VR*mWz>!#wS?TrqaEWhEOcfYrkXl-}eb&O%zOH(I07)rQ~0Gv_s3QP~W z&eE@y%E<2Gk^zFH05MDCAtV0|6a~V}92Jo{+VSvrEcD!NT-JVlD3VYtJ!NcN8qU$~ zj!#c^?usaMSM-y#LwemKnAhaFv1TD9k*+r)@Pl1W(a(Uy3Ph=J&~1``TzZG^jJhb3 z{8Ih`iNsb{?eK=au|ji(P&xsY*ybGl52Z_T;r}xBVBvsou2*e4~DG$(*aiK6r1B^q&haB zP$o@KDDYs65!O85Hq>-JBvk^Xil`78_8Y!gJqMCbMD`BLBldc%X9Zg3EfqvNhWuUetliA<3qn*{QJ=ToDd+1~GEm0JW1pZ+Z%8Hp^hGz*a5OKh3n$?%5 za_A&z+aT%=RNjPOmWgZgLGUnVj`7#PP5J(@i=MSjg{8HYS1Bj_DFok@sy+ftU>4d5 zMF{!4AvGseMxy)mSc)mpxU3=2-r4-i2=>vmFSE-e1AgVolcU)VDz8Uw*#;cy3JPl& zd5)(crXnn{WP-R{Wp2>4jI>7^&(CxMd5SEXSzEIUMAy&Z0Qbf?E$TEMUKnz}4aYK>A7NkMb&{{)%v+t7P&WKNAaN>uj}# z<FJw8IC>GI8 zsbDlECQi;mos}r0!vgD_KI1Hz645-jz>xGD7DiHM>=Az77ev3vCq0K4pCCnAmCqej zg24ADGR5pg2E85pw;tOGp!hV9*7F_tAm`8319Fd4zgtVu$ zb)AhT%X5Co|BUIvB^N+E@FxZs|x_uRbRJJSZXR0{O=sl{x z@G`ysfgP#J5P*>`ok~zW6Bs~YXHwmard`^|7Wtg`OfGVyH{1`phn_m$?ho}{5Qs}Q zMXw1~Pw>@7mk$Z06$DaIG#`$*xkN$HdHlqP!#8l+?O$H}MR0J#bGO=J z(|v#W`5MB&MjI%jJ-~h4SD0TI(P12)a)T4q|5=JN(X}eUFZ9#9I zAG_kbnkq-Ac=$;TSNqgrRfoL)#5ps9sr=^rNg-3WoPFWmHcsb8~i0k%OUQxjE;mRJM78jc}DWayKjPVS8QmTnpC?^diTw`OIHfk z{tNw1CM>Tp2?ybA&~$cg$u3wJ9IV;V@%&#ju0VKDInIb88g=V%@VCb|HDAF;2tam7 z+jTsl^p3Xd4#BSq{ZVh1y;4c~u}gB7pHsS=;yCGLQPjqxp{xURaEK7d`d{w?V}&)o5bnp8WVW= zSa4oavuY)^Ih+ZIS}+qkPb=IjnqL^2cd0l(tn8YK=4lBUJA%Uu)K;x1N(>&w&GN=8 z<7@uYih&cP99CmC>3fA}cWWaGfC*v>!Vkxy+sNN~weu%|ljj=^_yfCc1#~p-5h&`$ zH_`WDd)b#Uzw%l0D^D(_Dcl8%_Gni3xAc>Yz8=dMmBNmdU$-rtqbIO^I6f4!ZlydN%*n`uW~TuPn*yAx4ZOS>&b(=E3sY#6dm0 zL;cbf^wuXle35`o~W2iBuE zZC*)OBWo)W{3H(aXqTlYXI6SpO3H|-=d?)@Q?5qOMu-B$ZqZeqUgagZY0}@)1lj3NP%EKH|FxtE5 zze!~)GolmxhDL;xNd!va=D&u^96E>9uUbivM1Ud?zqr>JFU<{@(tjRv0U`y5SiQoj+p!rN0?2V7owtB32m-;r+1 zBC73aolA|2`$x8X^aD->pX!VVkAkyb9y8E%2Bs8Qc(RJ{FJn{bUWtRDoqn|!Y^>#W zEy1GHaqWI(!R8LlqB`ck-fXDZ7f_Z&Co}=h0(pRpeSusB{nP|8h`2H1N7`?n{|BZl zfty=no$@o+{8yMbD*6?7#ep~0GQ*O!3 zu=}QWI;^*4ZpBQ3gVy%#?=(yU9%-1ec3M9kRy~~za*V_s$sp$r1<;jACrb_s{`mbs zJkCeAieH7FH*72v)T)6WP%Nx@SB073HH{8Jdw_nY5T5jZ$3|e#an){xmTZ_3@TT!u zA%9subVF(0v%^SQw5o5b))+D4;+I*C=}{7Y`IS4vEwu9D<*mQ-J2w7l!R#G3`mj;a*-ru@MWLLSM+1mV)pIs%-}PMmCZKQkW1ljej&kJcqrw*sok7prH@MUE3TdgC-gTERqG2`hi68Jv{r#0WR0sat}mvl{) zq9X{oeCU!XJ!cwTX8t0PfbxY7B7!1HrK~_OZ%hWTO3jEw$kLnDc~}M(te|Tb4&Eun zxYHZP%{*X@d<6|__7tzsY7=}X$w9gn5l=YoNz{@b;iA`k3Tf1Bw_lh1N2eDJ)eu=c zIkDuaHed}ZB7g{740TtRW3yfyLe(GnV;h-SR3Z>;q0b*>q{OZH%RtX7;hsZonrs-4 zW+XaJl^dN!k@;tEDe|--+?E9+5d2=OKa!`tOmZ$eB+2>Vs~Ky*55!ax z!dO!loFI3;+wQxH*_^zfPlzZ-L6L=;hYr@ArS2ys#vq`mue7@V362P{-&@byqMv?V3DuhW*)$2MfL$aeB#7 zkHl1(7fM9PwH&gU>Y4+C z+d9hX-uK?W|JlN&pH05B(#maduf_9=G#-7i_}*dD`2{B5eO?_rTI-(nK$C4IKNt+q z8St6^IN2k8UoAc@Gy235I~Uit=T44M5-)p`VJ0bPc=!C>L;E*3t3aE@G0T-oRiAsV zeH&VHV@#Rakgs$Wp2U8AYgy>Nu(82wkyHB%wdf50sXib4hRg4mqpGT^k*TrTu(7@+ z*23TVvXztH2;1#5UG#q|a39K}B84xtUvQG9TXv1t-ne}W_nAEQOuwO9pgbx1p^ce$ zvTyvN34;UY%9&Z1TfW*~xYg$J_3QO}ji#?N_z{LFonLQ<4+3v&-qg{PbbcDMAiSod zqu6Bqp~*YGAEMN*=HgQON42rj|0(Xx!>QiC_hEJFRGLmjY0|VsDWVh^(x7Y#4Tj24 z(O{l4WoXnOvP&qTBAF@kd>TlKGNcePB=bxN&%KoDbI$kse15;{damp7$2qb0e((4D zHLTZK_qx};q{6u(`Gp)tKVm)Y(rBCx8{js-wW8?gN5=HbF+ew4n@8V8|Ci4GtxGF@ zkDB6k?C0y{v6cIaEw^Uc2q0=aG)aZnaX)`yXSSIz(4EG?x~_&mX6wzYodfMXGMgSk{agY#iqQ+r>vdTC5(f z^rV4da+>o$FZPdz{xm}3`0-^qz=dq4h|t8;QN;5LI@wR;$;_? zeHIom$i3X1I)8mlVqSlDW6}K`)6vM1AKj4gj$4}?(>z9snb|UUN54`o0A(#-kc8_q z>IGMm8Vr{tng^li_a3sv;b^`LO}IA*SDGyR-~%bOx3?dNK5`xuk_YF`jT=OuDz|fI zAs7X#TL>IO-tYX6HK|mEW;Szd0Z3XPGhdVV5v^fR!nu=LZ=-Ii7}pHrv-NY_{$;4@+L%5T?=AKbH?{Ub(+Zo z2M_^;T=gk6`*~)K?7;3 z_25^1hqh!qOrX}WA>z2Ulkt97H%B(PsXx||^d*P|ZEOCRtP0L|(wQ@7)QvoE-h6=e z-102K&qB*p;&ifSI(Ja>*;*+u75AZ8CgdCgySloz^nTw7TC=~5T9vRzX#2VA&PHWa z&8dR;iZ-YYo~1)zBfos{Lh1QsPG!(n)KQb|m^$d?Ie=W?!Un-P3l@X{BhmvLP5c1j z@|-KrxZlx#zWkbLpdevZ91D9-@;z7kv4?L=gz=pxbc9Odn%rbx#_b(;AJUjI9=eMF#Kjd`p|wnvLu z#tI3EvfA}CX3UUdGT-7#)G2)cAnu$QGf;G1b3O$r*k`(Zh==wipESH9YPz&So5OBs zMnq#+lV9jb1H+m*&OueL6gOAB2#PcxTdl!eC2*yGX1(91N4fWl4W;TrrA{Wwu%0HuN?#sJ-oex zX~HrFwr$@o#}ZIas5H9`W}-X`>G0|(H^X$$VLvA;Y62#j5L6rIF%ylDueAzYPW9>h zH6F9FtYadp{TXjd`pMuIT8C6O7r`!UgGQ!)AGIe7^5M)+*j1^`MJi=}QG4g#^(X3V03-fwg8H=8Owv^a3{;Qqfv{t+4wf$Ys zV)q8vah9%^;Zo(Vm>hQT@e4zaIkU^o+A^joKGS_JqWQ|^h{(8~y`;ykLT_cboV16w z+SuAAG3r4sY(C1d@Xt)ru(Z+1TM!J<+LsEG^ZYWE9%&12Up%|epa3DsOWdZig#v8p&9}4acnSq<2 zjAtgJh?dnk{Sz`0+lMqXhG>6ON{${Oa^WCp>f@PhK@2E-d|XlF#2aZ=jA`}uRDdW* z7UY9J)Et!u@S+P5Ndp)Wk+TIg!%!K4p1ljjAHX82v(_DZ#g%uG&NAuf>#AloFKac{>Vw_ zV@HFO;C`~TobEFe5WQyocR2s~&kerE5Y#BWxA)GY9-5Pk|KETA-H;!61?=_}piN}f zOF>S#7}@47RJu@jh+fJ208({>U$;U?=rsd2h8p-3H-(m0G=r537|TLA z%cwLi{HCCm=8G5f){L=}eJwRM==juCwQ>3gy8Xvl3b){gd8fDHC+Ehk9c$GSOg2+V z%FR{#@4*lspx)Bb(oPMH#2S3qgSx@TND}TgcK&`Cj(pNdkzd5^nC#A-`;Htr648NUkTiky z3Ladg*+-M{@e?N4E!pz@cIm%P$qBZvyr13A4O7um+4h3iYdp-kbLVcf=?L(=|C>XV zVdZ{tu`bGCSpt%2OEzvS$$9(hDga*PUf-YF0r9z=%spGSoY}HvOUeRlIF`_MoCws) z8Ed+E$^!gqV~oed(hP*W!krj;(ePwHs7a z;#Ij~dV`+0sO~O#yye(|xO?VLh_QBcp}5M>P>aShKDZ1x!<0R~TwZy5Mu4=9@wYIp z$WYz9g%Xkup7II`CkFqHX*bHV)$m z%^;hrf3js%%ULdc0%$G_+Tqz;_o0&i5XPo_8fJEaSnXNMwr@i5!%rl6v-p~iT-Z|M zEf^&~S@EjhE(4vTZZV^UUGWCC=lwinS6sAs8PzSCGGi|8h?0d|~^TGh51T zFTQ3vbmsFkZ?&+@NWN-iub!})`u3dMm-+dV9!vH(*acQuxBgN5V{EM4=7cU`S7W0|r4y3=5#WBjEnPB(t#XgFlKPl(_Z;mUp3z}*I&#@V zjZOW6$3mxj?Y$@2d~dkB-$lat1c;Um@4~4V_eICe-`2X?!@{tO*Y4a2hjXJ3mb+d> zS}N_X%0svwc53r@HmB)k9#fXMgWzEFulRB|w)>BVayoKJs}Te5f9~)<0-K+2 zVf*Srr_j4y9fzo*#pLS$ag?GSK0a@w;3DDN5f_GI>+eREPfVTX3X|b7hWp7sLcLLk zlFezEgA4_4{u6%%LX|78_O>>}oMJgw;}t&F-{UQceSVD>UjCO^zcv~G;YEaavwphB zPJ6Wfew>@*?9yql;Q8rL^Ccd+onsOvN5VhPnEdP8@gc=!LbG_detk_wAO`2oO~noq z3{HmCuW!(-X^_{vBxs9?sl5H$C0d^5i3y!n|sQ18i(V*p6B=X;7Dy7$xV%XcUasnOV6gB%G>TRZo|@_P=NXh zrq=RK9&$}2&i%Ye>FnJs)(-sb)Q$A`(fvDeHJ%Bh0WVur*{{eSuleryJu<$b#JKom z{^){fvza5HP61~a>LCt7F?A2Txur{&7N*#kqcIvYI5=3K_YP7mkfh+4J2&{tmlLQD z+&X`~!HdVK-HpYeXh(F})m24DCjjqQ=QwmKG#Q*TIe%IX@YwtrqrS=4l>Z|)ASbp0 zHX;O$tv7iK9MWAvbUMxffPbes6`v4$D6;o#*5KH#$-06bu zP7nW_I!~5G|A4PZaXNY56w?j(Y_ya^(T3-v$B(;lst_!|3V>zZ3h%Z6WeW&U1udK2 zzg;$nEF|~OId=W_RnCXaY37w5cEc%89-%y(pHJ7$EoP$rrU;oM1`}?j%CTb)QjI_x zX9a+wknf9^k*|7(2mm4bJy6!f7qo(5^A7(i3Q}BThuBg%LVZ88O301AUMe%c%*X)9;vqAFoZ`m zS|CTLc>cl#XhtZ@!E3?75C?USjs1!;EM*(l&oDjCs#TNe$zAc+!iB4u{fJrV__)l; z9|BfPVh`0}^DxCvHkt4tCGtPV)W>uIN}5bg^8vVIBt06PyhVn11yfE|R!LU&BBLI7 zJ*#Hjq|GcWPiV*l;(%yGE_UVK2QIn3lA)NonnW$EVB&Bs6g6?$HcnIo$7n$ZliT$V zX`xRmCppn(s;nNhLOi68LHNY@@#E_u%n|1q^x%OU>&ern8ZGl@%(#2w28Z3!GlqUC zJy$NGT!9qlrQ1dR5nSJs&RX&A+cNdy5B;7M>qE=UH7bsJF;?VndiuO&y#CXoA`9KDSf3GHv!J_4g$!-Xxk7xlV{&D4%#EcYR{Hf*3Eh_8^0)gYJ9a&@KR4cfb%R}_PlmN;x7Gfynw7ggeE0zL$HY$jhgEW# zSr6*jJLt3PxV#0khqk@3Glt12KZNa)2b&UW)cDEyKx>(A$&$^4U!X}i^>lXbfAK<= z31Ou<>fF@`LRkT5NiEu>T^fl(>Asfhejq0Pt(#~Vc#fTOCC3-h=Ld3aiY?=YYU6i} zadp2ntLk3TA;W}{;SOyMo!j3yL~_q8czRj%qfHunS@BYz$p^*9Z=4fR(q0*>o!z$z z?BWAEIr`?sSC*`p$Uo%UIsbZ<$<;c}9i{_zJ|!l7aQq&>SJ_O>xZk_ z$Mm0G*3Y^VDUUgL(aOs`ZV7Ud;+!BFjqO zap1cL(RS_ME|*a8`dGap76W1R`+A{VzfF-?As&KS<-Yrrki!I|C6G60P>V=griD;a zpmUdhY0AH)$d0}&!}aPYDEUGXE=_BA$r}%$3Su0wE)xtJcLGtE!Zb? z8X<@zYyj2LL4^-?%@4}@Qjwlcg?S*EZU?Dp4{SJ~(J+uWWG`L1^v;rf{EFD|+ zlIy;Y&w``JkMm1Per#?I3<~1s<=v6V6(2vIOrzB9(SW}8ej6BhqzW*2^^(BWqdQV?j9m& zmSZ6}YKV>-Mx6uSx^;!&hVT39rglvaTv*II@+i{TFw_!D@o@(!^B2wOYhGq&!=t|S zD@UOhQ>Hn^^+aO5n{+#nxSpGydvEH z+M)P3_wUitmv`&&2!026yOj#o^hRvCamOPtaMrQp*r}?vyHXC0*}U1v@a)i-TvL0T zLiqVAFmpzw(3s7q*{7JcZmW7erI&jV8=Gjy^eOueU2H$eWi;zAj+&lCZr{I9+k;L0(m%qNQC|U` z<-*YGH~^|#>^$~3|F7?2%U}nQG|g3?%SqAJ9N$OI>FC(hh2Q@<^553) zDZqfSg?~6r{prDuyp;2gDR}x@gj8ws)8gS9UH|vn#DTxQdN4UD>~_uHqLpI*5ncS2 z-NG00OiY*flg)bm|NBk+V;$`D|J@5$+USC??2!>`LN!~Z<}c4*;*6jlzCmunZ$nXO z%HSO%oehI9@~~`d2^pR`_-95@CqnSwe*9||{|pM~&;MK1*+4xe`dFtK$C$OF0{`2;oiiFwx6P6tw z>ok4T<;Pn488SNwR<}-MbCUo4?#eDQIN1WH{=Bf$%(H?6xU-bJir*8NY)-7R|1rS= zi3dzC7G0=h|M^M8z$dK>*)uA}oSHNGN4|Ei41^-sSU>MaQ-(!-*Gyt`j~|_( zjH_NdVUwaRgbu8131)X5*jMq?t8ccW@q|i{40Pvha!3Kf{Sat0{PbPq{JSm?t7?8j z7X!p_*cDdyh7Jub(xcLwZ-j8ppc6FF{5VR<$+uUgV;K3S*HF&Ju~CGS(!O{jjZxt8 zve;)XAke|_zlK~odSA%Q8EAtymo7p6KOD7)5q}Hi9|T*Y_*CP-<6~E?&VDzi&$-Q~ zn9R9(TP8Zw(^WNav18t<$J4YjMW(U6kPl{Vd-3X3A*9}xGLgf&X@5qV1qp+2@OE}k zzg9?N-^Zb7!@qOnk(5wST6s}0_mhD7K}=%8vJZcngcmZFhH0g10%+U~l1?FfXZ3Cn ziryeTiqXkt?Ga**e3p&fS?tfePH({g`8Scz?LYfehj4V~%#4^^QFBQ!J7GN=$A%Pv zG3BayGiRg?57{yEk#TE{DOX8DcFyhj_eVK3-F!-$<@2LQ(h{ES-eV#au6!%^xNbuz zr>GXstaQBzY>gg3h){th#{$Uy37q6ada%nX|&#t z`0R?A)cUmYbXK9JgZ5ENyYrjvJPi+CY+os7r^5HTc|T|k1-|0eDXGK5zP`YAZC({J}`7wFv|BUUc?`9J+DGa<{{nF&(|V*K2mz*-T_lxiSBSW&6Dy zB{_?l6IsJ*`%5)Hz46`kDKoUKdH?;cxZ|&_Ia4Hi+bryc(~GWrV4pUvNA@6`B=%`j z-F3Rv5wm_IJHeSt-D3)0&hv1j*rTMr>gyOKJPMhp9`G| zj7*yu#F@o6)%1X=3v2d&{)L^aYkH4IGvH820!hsf8Bk=4GX{TWc@%$)c&nrUaFJ65}RwjK&Qb>(rMddpwP z+g`jlF+T2eRiJ3DP{N+O1_=z1!Q%SQEVv)lO?OxoQ^gf1lKVvS{_{`j9G=>L1r4}< zifDYCu;8rdl?ovVQ~Lumb}%LU(i~i>X7l#?r9#b5En+Cw4G z(sjwEZ<78A)$?RbGrKQUxirptKDU_na&rEut4Y15pFfucC;@@v>5EPtFy?o=dbL~a z`v%SEU6VDk+oa3sMn+N=2pX*LE@MCEIBm+ZR6s7LFgm`#zpsaoOz9A-W#DNWp1 zhst*xF=N8do59)c#7i^)Oq9oZgZKVGQ63`KL%{WVTfUJ@8cy(sz3HE__%S6O|?|a2heT~X#<_f*X-(*c8F#455~AG=lZ7N5%x{D;Ia2h{VtHIuY>Klo%sW-Oidcc{0_z34w#N z_Vnq~%NH-sApIYOQmSsAG;!h@jM;K2sdA9~E4InV%w>WgFJ#oBn|;H}>%_Nj-&`1b zF(RI&LYn1|0dmcnHA|a8T_e5HA#6v@YL>5G|8Cs4aWy3lm@0hVlqu^bO`1eK!>hl>C{;8+d9ty=u`=m_6># zKj$%zL0otmErQ#iXTO#e207sYiZu^Z)_-b{1MV2BUf8wcvyC5$-y$M*47)u9VpOnTu3+;~PsFL-z$ALE{)hP%YQ* zO|`XLmJLiQlPZZ7dXJ`Dj#%S((DODzdv~4m31Svst`#(M@#}LOOF?>G7Z&H}u(~A* zAH3-nNmpk(s~|n{Ntq%xOIcy0R)wrEV33hX1hg&0+6e&VHMA^Ni%)kFUo&Bhac7Os zltbgKa<`5-|LKy=2LC7tk>ZI7dwl9t?8jW1$nyr$HZBY_yU;hs0ch6;toJHo)}lqN zsg-h+tn@~%y7(KG@U&VKQ{F~_M4-D4R3n;{;(-_}%YOFknHP@y4>=n!0XW=tf$l42 z-yic-3e8W}fwv*TdIbTp6M|qTS2SNoEks{n?I3?Pb*N>~#W&Bv#-qz{D2Z^V`46Jf;;X-ast<`U6U}rDTyBKKqr=ElZsH!BM1u& zOvpriA4d|n6ndHZ@P|G_)k6sK|L322-iFJy9LyhCVb}^`xGK{Igf!nJl%$8}P>Y>y zJ9gZLrtq!z@6|jnO?2_Fj5wJB^kOGyyG?bCFMIC+)%OB1P_DbX9~J~7AlY39yUbVL4>Ehtu+PvZdut!m zF7YmGhMJP&?N!flD@CE6MH+n3peY?{C<;9efXDz1W<2U9NN48Qq2GZ6CgYY!V@Mpy z8Al|M{0{RoY9h?|hM}%)$nXZ=xvr!R1}S>@9)}Hz7|vWl>yCd>OBXZv*|*wuUfSeoX{H#6jU2evvl#a3RjEXf)o?k4;*Q!67fgFQtF@@!eU6GD0qB%5DqGS zl`tjnLhzlDFFp9hn`VK91yoiUh&>g^EU_Jvu*m!`zLqeb>timz{Mf0KQPfd5x@^Oq zSTPgbFDgAb1=GP2*kvK6&0i6%?VNRFWphu&=O8L^>|l$`1RQm`hA-IFCY9?o;vQNE(77iwgEmpzk@5Ih52IbiUd58r4JIH3TjN>~sX=1uFV+@B!hDq=W#@`&i?HUY(lng|`1~-O=ruF5XLy1F^=K z=tgjT!A)TutZaCglt;k%z`1CVUu%zxNALTVO~m>mAtj<=;xX>e;#D|7k28640m|;_ z_M8XH+S4+v?;?WRZhEZNXNnSsub1;!_NRM0jih%c*hS2Unpwe93zV#KIkDB?m@e0L zfy&(xM}Z|NjmV@(>)adK4HEFPRh0D;&xbpcJBP zJ|8Yo%7PUJAmuy+XB=7My|*_QtAH7dqa+ZQA&N6{l`7cg@lVIeg@FzmiX8Pu)2ef* ztquh{%2u2q3qo4lFnbH6y}C|{zR0TWT?8eMVx*u6mm<G9AQVV{O$GT#9)WK$ay$&qmXtXsU3Ra5eXefJwS_w~5AHZswPR#zZ5{4C zv$n~>#C0Kgt_J~;5T(0YPWK!ROW{o)m|p5Rafj=Iw@A^MH)mPjUH1tSTH`7l>)fPkFY%nnoPxW>5%()5NEJQug%%&yl>t|yK^|DW&UX0X!QMC4sfpiqM zYUBq_ntbp>Yx=OjrS^{HrHPZKPglh%FfT&=>OmGYX~*$bge}`_oU3YfvfQRQCKCsF zBs?qFDA6+M5AfWfd13T%BlEm?>-mM+94@?CtzVNLtKfja5|2IAXBK1IlE#@f|jhe{EQi_E9%A zJOyqP_m$vv%w4;b;rMmK#U%a&N(kd?>gqi8li|y(03kBVZBa26s##I;yr7iK4gX0r zma&oxL-sCQBl+E@3b+R&I9<;pV0KIw&kaNjCDp*s7MH?tf-(Eo>aT&q%FwZ4O#A1O zA>_YkD;&|*R>rw%Cl-0#9#tY40T?{ z-v_S~=8kyaB_JvGw~Km8NF+Y!JgudqiK_K1N-yaF(cyAJV| z?E1iAIt&=cnBk3RJU_(9VU}vBT4rQOcOr`37~}5kt&4Uvzy>ftA9D8Y-AjV_R~gV# zRqwTYbnYvTco)X9Y*Im(W;uU@(K;B^?X3M6_SA21OA*-t=y*jpDmaXWT~`@<(;e)u zdwA5EggtQ)8bX1Z z)4Ba6a8K^Os~-HIM{H-|3|5NRI1)-k0v`0PtULtpbgpynEqE%oL5g}SD@+X0r&8xr z{kS$-JapHAl3{C`Z|{p`Zd}aeLPF7P3&%?!w6MM%^jU|EKT(^Too|ZbtK>1~Yl9QF zSL$&DB`Rx4wQKO{#<;$}7g&!y)P-AEq02p{bP3)X9aI?(X83C zk1@O=K!z`CJ3cm3|ex4XWgnSvfXb^T%CW#A_?6zp!&cx`PDNv)oKZ1Uuk;jh@IX0cDL zIW-Os#rb6O&-j}w^X1C(b0dL>&EK?X(<}}S9aKmhLYoei0UJ8uU!zRPEOxKgz$Kbq zGBvF-=&z7|3+A$fHxGVReO%{QwbL}6ymdNkp$Yo`F`XG6RTA+HV4(8drP!HJ213GS zEm&X>0>}u(4(bXf_fBBt9dcazsu2clz^oaiDe46w%KWpqxs9+JcfvC4$2#_Hwuf-% z@+C|D#*`6oWe$~8FTgj6!B&A=H%qb2dunY1_PVPL0L|t)sijMw&{3M#`IT@D#Co@p zgoEFsi%`uNMG>sy;`pX@sMOdXBqT(EGSy!Z#G-Tj_`#2XGYt{@KR}YLXXn?B+ML_a z9i=|gq92Eu64q}RJUrej zl|#H}zu;M@6OVUHOw2ut(&IEqaHz*84V%Q?n_R8D?)+eB(jJFP6EoMSsPHw7y!Zf` zCChObo;!EWi*N!4FR%Ewh5@S)DZ63Tzrj{zr7T zD@nDB_gs2mEfA^njd)wANHTV=<3t{rn3t#+{nV^P!;rxr z@l72}oJizk>S2=K!4g4IsJ8hpdCK{{%*#_G6y|`@z#J6KVCjUSbg2O}BgiiXA|$ou zJpuAI#|U}b*VD_(E3^sb{~hZ7h{Xp7@5$W=;I68cr?4l1eCt7?OXlJu69St<9YO5v zhi&H(YqdiNDmn9ZFo3$L2s&+nD-hr7BQzDbUEXyfB^QZQ zB@#hW#X4I|QF*FG;mJmXZ>YMVyuo>A=SDaH`Je@gfE5gbDn~V<>5gwZ_grVO$kF@) zah-jLRIiku-tyAe;K`FGg#S~`#(`AZSOnKieTym0xN&UoIXH)=elghkm8oP@ZGteZr~MW7~M zEN=5hy2CN}2>tGU&}2yOXQ6;^NBFvH$o5%4_M{-(`31$vDNazvYIPGiIWD*hq`M}F+EWZqC z5UFQRMGF2c;D>Ii3Im!&K_SGrtM9OlVdGmmLSS!#f~5#EpEN%!?UycFMy{0{3(7t7 ziM+Tv+JnB~agF<;OfH_~%U7;lyB8uC1eVki8JUR8#5MrmVV9qj2Iy=n_uNiH3SlSJ zqupG6kRm8saVmpgp*1drh%rlt$*&l40lKYJE*7cG_cTZj^EmytQ0|>eh&f?FV#K2)(3z(MMze`PE;TZ}f)6BkvQJ4wtUG zxl|RN3;U!=yOD}R^Qj=3U2LN94ObqAN`*c!J@@f)#(0fog79)(uby zht2G%m<9ka_6I_pAh`3nX{{jLhaxx)Myd{RpemXybV0Lfg`}jUsyNnXPaS^*Sn0Tm zlDF_tq-uD`7J$Jx7(t`?x@RKIBp9X^ok=4hsD#V zCLGcZ>S9-5!XW5=`!5pBwmXB9&kd+$1=vqLSr+yDXr( zMKCfA3mVXL0ln45{!jdTnUR(EEf-0rU<{l>BN+pT3xxpv1-(H9c4i%DrgT>7Y9TA; z7>tYy;_XL3h?O57+eNVtM%nCyV0vD1i?G4_H9$NSkaVCBbc6)ivIQCC#fks}c0=ce z3Y3l>J4R-K5HZYaAOz%ekzVUDgr{cgn0W}}jG(%C7_UJ%qGUvS+= zQgGLXGLy4AuFrtuC)VBcVK0pOZafj8ff}f-gci>kY@r8ZkKWNrvEB``C?QcvR;>}J zhEqqLjTL|vOlqqS#uY-M>OPp%520L3hl|wL%0DpkB>t=5mI!tcS?7{RN3RTZc&0zZ zy!Yc&!Piu%Pg+c|KH`jS_%IJKRFp?17#DL{55b`~XllT2A%MjZT3A@PMY3}Zf>lH} zo{aP7&WRS1S{B&^Ac#e=`l?ikF*M|9v>7Adp zjME2IFoi{Ej^K}4W-NG3BKO)SQ#meHIxB$Sa`kH5Uvfj|u~=lf_p#PNA5NWriMlQ_ zK&${7M*zs@jn<+>1c1?Xi~7Eso_^Qi$2l7!7v~RE_bkP!b*>@!{MmTATU@~HWViKYPaqPT>_FK{I6sChgX=od z#td)d+C*>kHWVSK#AXyiWou6dMDAZw`q0UVtrKQ&V-vEInVCrjRLu7oxVjY|NZ#v# zcdlc-B2=#VdWBHuBj|Z8%U17;iwB{+y0z692g_@C@J+SYKgV$VBf|yvO6!jh{OFRpQScRyD9(qmfI{jutkrHvn($cc>LP z;vxjJY7J{A*b8EgLn%*tSg(+SyQc34;`UtT17wJg0YmHQ0C-dop&>pXmf+L8u^ZtN zD0#7-en?ZG{Y;^(FxJ_69v+ZR!H!E9T_NTL*o0C?C0oItu{EQ0p*H)zf<|tBOcmrWA&C%A@HXjUsPfC5; zSrl^e#b3=qMx@>%9R?M0_TC_e+kM2s&|^^ecMB%*eU4$>vk%?JT!bJ9f;1!&Q>jgQ znGdj1@f#9-X{EF}rJ0`$9G+#aPj7>@6uSXYRPYvnps&NGjP*ROqvM71rc_p#6zmrC zFlA*iOXwC-+qQoIkh>7_4GNo*YkNQBe<*tc6ogQQV9nGd%6e}72xt-s$lFFq*_t;I zQiGlWb?KvK_S~;tz7z(&kR0OGab%#(wl*!|Wn>pfeWjZO|6r@ULge7eeAIl$TGa26 z*}lC21wyD7h)P8s_aq_f;JPim8P0zZfl)AA;b}(^Srg-u&NfK%KVyCE>_q7(AGRB1 zamn4Vvw=52Me}q{qg?FKCcR2IA|^sNFa}>EiG~~qFNbQ|_`8sr`T{4^7X6$GQPSXS z4|K)?PFo{@Km|t+4FFby;JC*`E?CSYw$n>b!AL~QE7q(0AN<_BAs0 zk(f6onP%_{zPO7r>Y-og&T!5;M=qSSE~Tw-A74TfLVzLPSO&;OKO+| zR7Wae$d!RRfXCymKcJavTf1=sZiza=FzV{ku^6a;anIhpcN}(?!4XkEiiDL1<0C}m z6Z@4S)WyJmYecb?!ayO;-Jn5k3G1kUCmPie?<2Z%hUqM21z;3QbyI_8jxP(xaX4+3 z()t$ByuRqv2OnnI(CqrJi?)xLK5?QF!aU0VHNp=82g<&tEzuAsFJa+yKaD$hSVZiC zheN5&er#YKv0PsG$pXbvad}QM9#|vbHEh8er<(8r!w4U{;C;bFz`cuERZx^@KZjhi zK66#eAs6=mb@P>|&A42)_HQ<}Z;s@#Grl`)hP8055<{v1R9EpqELPI*ZBd`Ai;IP9WrDhaL0Aq~z@|t}eze0C z7ih#fYF*dPBPRA?Nd969kQ~{S?>!sm2x`U*humA76?cX=&)fmqgTFpMgqvnXsf4k7 zxf~1m*KTajMk>NXwm%TA03X;dX?F~bv~fiCIV^FgdNv8E-WNk31ky1JmiTp`EZ<|( zXOp<_*W?gV%7}bO^yV9nD1RlmndA$=fiz+os6;()YUu+nxr}t8r0;SrUp4^VM$hyZ zh_5Pv2&xR06^tm8MQ#qxy^_aisVgFfXzfz5Ed*aRrp^_9R`1ys=P}7(QHw_6x9Gpt zdv0Pa*@RQO5XT(VkpffP2ZSq@muIK1#C{;kF^-$>%qflJKs4@IcM=G%GFUpNtNBkX zedf`zaoFaLU#OT>d1X_HFG`C^(EtG9#+_f)!;#L>$3jKma+|t6*2R&)3~H(1Xz4 z9TEJ$D5;#B-6MJlOVdtlCMsZ^F_=#Bh@7rltgy-OJ~LD`Y=-b|FK!f8I}|MlYD0Q! zSk@F<0aPFoM9`v!js`8{yhH$fQz;syTcMQt08=h%u=oD*j9aSEz3e6PBd)gmHH z4<T;@09z2YGuX6UGTJ#Zj*O3}?p1;v4g1n7-ki3eaI3jX9cLCEe;=T22k>VOTO_M;x z6s(TnDb|50QWG`~Oi{bLWjx~hmlYLSaO+Y-7QMQG!aum5Z{aO7m_QWru?(qv8d23z zNb2l3?N%#5WHufq+G2N@F`%oZ&_0yCm@OmJAiYBir%OR?P6a4DOh+l>yybj+ve;b~ zwmVBb9do%Qqkzy3BCjMfsW0Sxl_nY=MI-@grtsoAXtOyk!*DKruo83;E1R=aAs~0i4(%iu*XfSIT2H%(>31X&z6}$~Er{pw~WKAl%x&n_VfI(l_ zU@&VtaEX_(6h{Mz%!uEJ4TYixv_tYs<2wk)6Y1(fOfNuQd z*B&^o_oL&UGb77}!}#@>6fk2Bq&uXd;u-CGCc4Ez>-HJ zOA}1*6Rd<}+7}SQE#60x4MlL`;}BgMb$Bb&lf{LiSYyFWW;GYlIH=y&HK7e1d1(aZE4|RVb84h0-+H81npWT(CtNU>{!ML5Q4wqrGxMwa1Q)jR6J!1@w2l2ydaKf5ef@Hye=b2h1(vMs;7QqCJTu2(j=$r(biGOK1zMRCF7|B%UrTN;|>HNx`4bkf_S9|=Zg@)soWB3SWm^kQjO{R0P(!+6~*T%DQ*e=1^lYl zzJ*FS42T{DAB=!>s=>iM=wqLu+x=M>Qve6jP|Uk0JLrY~QUn4>U6UHa;;M^r4$G24 z1ekO%^9TK_QKGOJ(X7Izgyg4CQm{ab-+*wL^5f*BBY}tb$@W0<$#A2zzGDEsvX@0q zS-5c@{2LXlBN6MiL&UMbg4>789rTiUoH-@H4E6w60u!SFhou-8ZZIfiC*a(2UlQ4b zcCh!Q2D|oxf~H%RU`it4DFqo8CF%P5`gGCNj@H+97p}YbHqo66n1m_chhw6NXox@#0zHXJ2S}T=*I_i8 zOSf;|u7^)1S-0OK%nL{l$qdnC;dw|Yj^Fi+>uVqu9v&Wn$V|-s%s~f=^d(?@s}S70 z_FkJcbl;husD7EVt%s3N+UImF$- zgi@);yqIA!4p97|7&M-u1XFD?F(jx-KoSnPjUR*H_f!2F2|pnysE5q+UZ9GW$Zj@b zC#n!*5xaFKLAgW+-1u1;Ocn}-_AIcuXdqf0$z){ z0#ekl045O%#-AMOv=KR>U3Y8S(}}38rBX*H3FSyl4S=S!J2_DD_58}5Mi}O|T0E!&6L^8f0TA0rrG4Bm94nFq9J}y1GQ$)b2jXBRO{tPJ`(#Ei(%dB=jB;C7 zj-l0nYmFo~KsNdaWV9cPs1R0SY4?aEB?0F^B~D;NQrL%>O~khA1W`s&Kf?FMl!8?( z?dI~-Qti;C)I62MPzi5DjH8Fl43f4iU;Cc@;UJcOx}s}EQllibg3MMuj_d-Y7-$Uj z@m2*-=~I{;OTfV-UUDGn&}kAw!-Z@a@6LRIK8(X)x9ZhJn>5KE#oTtm4Inx$t?(Ei z_ato6(VJ(_uLk2wNG6CD`8WWG1x2SPziPxj#IJYaijgZOGOAu#LNuC0n7{Uw92s}G zrVHZqp&v)96FH{$!#%?QSWm!ILYw4;Wf`(ZpfJi1Ark<9ta+WnNNFbM6!tc+_&If` zFU#WkJx9ioDX&zdO^x=I55UC1&f1ml7uq@qc6SjP>)nH=T#w_i5YcitaDDVZ*1om& zxa(6>N`2si`ru-d>?aBKU`e + + +
+
+ + \ No newline at end of file diff --git a/docs/benchmarking/cli.md b/docs/benchmarking/cli.md index a7e50c907d5..c9ceb67cce6 100644 --- a/docs/benchmarking/cli.md +++ b/docs/benchmarking/cli.md @@ -108,6 +108,38 @@ P99 ITL (ms): 8.39 ================================================== ``` +#### Results Visualization + +The `--plot-timeline` and `--plot-dataset-stats` can be used to generate respectively the requests completion timeline and dataset prompt and output tokens statistics, which can be useful for debugging purpose or for deeper analysis. + +```bash +vllm bench serve \ + --backend vllm \ + --model meta-llama/Llama-3.1-8B-Instruct \ + --endpoint /v1/completions \ + --dataset-name sharegpt \ + --dataset-path /ShareGPT_V3_unfiltered_cleaned_split.json \ + --num-prompts 100 \ + --plot-timeline \ + --timeline-itl-thresholds 2,5 \ + --plot-dataset-stats \ + --save-result +``` + +##### Interactive Timeline + +The generated timeline is an interactive visualization in the form of an HTML file that can be rendered in most browsers. To customize the ITL color thresholds, one can use `--timeline-itl-thresholds` flag (default: 25ms, 50ms) + +Example output: + + + +##### Dataset statistics + +The generated figure shows the input prompt and output tokens distribution. + +Example output: ![Dataset Statistics](../assets/contributing/vllm_bench_serve_dataset_stats.png) + #### Custom Dataset If the dataset you want to benchmark is not supported yet in vLLM, even then you can benchmark on it using `CustomDataset`. Your data needs to be in `.jsonl` format and needs to have "prompt" field per entry, e.g., data.jsonl diff --git a/pyproject.toml b/pyproject.toml index 5c87de018c1..c7b77f0e91d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,7 +123,8 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer "benchmarks/sonnet.txt", "tests/lora/data/*", "build/*", "examples/pooling/token_embed/*", "tests/models/language/pooling/*", "vllm/third_party/*", "vllm/entrypoints/serve/instrumentator/static/*", "tests/entrypoints/openai/speech_to_text/test_transcription_validation.py", - "docs/governance/process.md", "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"] + "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html", + "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*"] ignore-hidden = false [tool.typos.default] diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index a4133d17ff9..dc8b293a425 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -1611,14 +1611,12 @@ def add_cli_args(parser: argparse.ArgumentParser): ) parser.add_argument( "--timeline-itl-thresholds", - type=float, - nargs=2, - default=[25.0, 50.0], - metavar=("THRESHOLD1", "THRESHOLD2"), + type=str, + default="25,50", help="ITL thresholds in milliseconds for timeline plot coloring. " - "Specify two values to categorize inter-token latencies into three groups: " - "below first threshold (green), between thresholds (orange), " - "and above second threshold (red). Default: 25 50 (milliseconds).", + "Specify two comma-separated values to categorize inter-token " + "latencies into three groups: below first threshold (green), " + "between thresholds (orange), and above second threshold (red).", ) parser.add_argument( "--plot-dataset-stats", @@ -1637,6 +1635,19 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: random.seed(args.seed) np.random.seed(args.seed) + # Validate timeline ITL thresholds + if args.plot_timeline: + try: + itl_thresholds = [ + float(t.strip()) for t in args.timeline_itl_thresholds.split(",") + ] + if len(itl_thresholds) != 2: + raise ValueError( + f"Expected 2 ITL threshold values, got {len(itl_thresholds)}" + ) + except ValueError as e: + raise ValueError(f"Invalid --timeline-itl-thresholds format: {e}") from e + # Validate ramp-up arguments if args.ramp_up_strategy is not None: if args.request_rate != float("inf"): @@ -1906,7 +1917,9 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]: timeline_path = Path(file_name).with_suffix(".timeline.html") # Convert thresholds from milliseconds to seconds - itl_thresholds_sec = [t / 1000.0 for t in args.timeline_itl_thresholds] + itl_thresholds_sec = [ + float(t) / 1000.0 for t in args.timeline_itl_thresholds.split(",") + ] generate_timeline_plot( per_request_data, timeline_path, itl_thresholds=itl_thresholds_sec ) From 447c372ac504a4696ddb54da8821a07065ef3faf Mon Sep 17 00:00:00 2001 From: Jackmin801 <56836461+Jackmin801@users.noreply.github.com> Date: Thu, 23 Apr 2026 17:00:53 -0700 Subject: [PATCH 082/153] [MoE] Move remaining PrepareAndFinalize to prepare finalize folder (#39009) Signed-off-by: Robert Shaw Signed-off-by: Jackmin801 Co-authored-by: Robert Shaw --- .../moe/modular_kernel_tools/mk_objects.py | 2 +- tests/kernels/moe/test_batched_deepgemm.py | 4 +- tests/kernels/moe/utils.py | 4 +- .../layers/fused_moe/all2all_utils.py | 4 +- .../layers/fused_moe/fused_batched_moe.py | 158 ---------------- .../fused_moe/prepare_finalize/__init__.py | 4 + .../fused_moe/prepare_finalize/batched.py | 171 ++++++++++++++++++ .../mori.py} | 0 .../nixl_ep.py} | 0 9 files changed, 184 insertions(+), 163 deletions(-) create mode 100644 vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py rename vllm/model_executor/layers/fused_moe/{mori_prepare_finalize.py => prepare_finalize/mori.py} (100%) rename vllm/model_executor/layers/fused_moe/{nixl_ep_prepare_finalize.py => prepare_finalize/nixl_ep.py} (100%) diff --git a/tests/kernels/moe/modular_kernel_tools/mk_objects.py b/tests/kernels/moe/modular_kernel_tools/mk_objects.py index a39e03abeb3..23ddc7011ac 100644 --- a/tests/kernels/moe/modular_kernel_tools/mk_objects.py +++ b/tests/kernels/moe/modular_kernel_tools/mk_objects.py @@ -223,7 +223,7 @@ if has_deep_ep() and not current_platform.has_device_capability(100): ) if has_mori(): - from vllm.model_executor.layers.fused_moe.mori_prepare_finalize import ( + from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import ( MoriPrepareAndFinalize, ) diff --git a/tests/kernels/moe/test_batched_deepgemm.py b/tests/kernels/moe/test_batched_deepgemm.py index b11098c820c..4c8b2d87d61 100644 --- a/tests/kernels/moe/test_batched_deepgemm.py +++ b/tests/kernels/moe/test_batched_deepgemm.py @@ -10,10 +10,12 @@ from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import ( BatchedDeepGemmExperts, ) from vllm.model_executor.layers.fused_moe.fused_batched_moe import ( - BatchedPrepareAndFinalize, BatchedTritonExperts, ) from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import ( + BatchedPrepareAndFinalize, +) from vllm.utils.deep_gemm import calc_diff, is_deep_gemm_supported from .test_deepgemm import make_block_quant_fp8_weights diff --git a/tests/kernels/moe/utils.py b/tests/kernels/moe/utils.py index c9c5c97b26d..d4b2350f5c2 100644 --- a/tests/kernels/moe/utils.py +++ b/tests/kernels/moe/utils.py @@ -18,7 +18,6 @@ from vllm.model_executor.layers.fused_moe.config import ( RoutingMethodType, ) from vllm.model_executor.layers.fused_moe.fused_batched_moe import ( - BatchedPrepareAndFinalize, BatchedTritonExperts, NaiveBatchedExperts, ) @@ -27,6 +26,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe import ( fused_experts, ) from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel +from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import ( + BatchedPrepareAndFinalize, +) from vllm.model_executor.layers.fused_moe.router.fused_topk_router import fused_topk from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input from vllm.utils.deep_gemm import per_block_cast_to_fp8 diff --git a/vllm/model_executor/layers/fused_moe/all2all_utils.py b/vllm/model_executor/layers/fused_moe/all2all_utils.py index e8034113983..fba1d4c692a 100644 --- a/vllm/model_executor/layers/fused_moe/all2all_utils.py +++ b/vllm/model_executor/layers/fused_moe/all2all_utils.py @@ -41,9 +41,9 @@ if current_platform.is_cuda_alike(): DeepEPLLPrepareAndFinalize, ) if has_mori(): - from .mori_prepare_finalize import MoriPrepareAndFinalize + from .prepare_finalize.mori import MoriPrepareAndFinalize if has_nixl_ep(): - from .nixl_ep_prepare_finalize import ( + from .prepare_finalize.nixl_ep import ( NIXL_EP_QUANT_BLOCK_SHAPE, NixlEPPrepareAndFinalize, ) diff --git a/vllm/model_executor/layers/fused_moe/fused_batched_moe.py b/vllm/model_executor/layers/fused_moe/fused_batched_moe.py index 5554298bd09..bd54cd636b0 100644 --- a/vllm/model_executor/layers/fused_moe/fused_batched_moe.py +++ b/vllm/model_executor/layers/fused_moe/fused_batched_moe.py @@ -14,13 +14,11 @@ from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.fused_moe import try_get_optimal_moe_config from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( TopKWeightAndReduceDelegate, - TopKWeightAndReduceNaiveBatched, ) from vllm.model_executor.layers.fused_moe.utils import ( _resize_cache, moe_kernel_quantize_input, normalize_batched_scales_shape, - normalize_scales_shape, ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, @@ -489,162 +487,6 @@ def invoke_moe_batched_triton_kernel( ) -class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): - """ - A reference prepare/finalize class that reorganizes the tokens into - expert batched format, i.e. E x max_num_tokens x K. This is the format - that the batched dispatch/combine kernels use. - """ - - def __init__( - self, - max_num_tokens: int, - num_local_experts: int, - num_dispatchers: int, - rank: int, - ): - super().__init__() - self.max_num_tokens = max_num_tokens - self.num_local_experts = num_local_experts - self.rank = rank - self.num_dispatchers_ = num_dispatchers - - @property - def activation_format(self) -> mk.FusedMoEActivationFormat: - return mk.FusedMoEActivationFormat.BatchedExperts - - def max_num_tokens_per_rank(self) -> int | None: - return self.max_num_tokens - - def topk_indices_dtype(self) -> torch.dtype | None: - return None - - def num_dispatchers(self) -> int: - return self.num_dispatchers_ - - def output_is_reduced(self) -> bool: - return False - - def prepare( - self, - a1: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - num_experts: int, - expert_map: torch.Tensor | None, - apply_router_weight_on_input: bool, - quant_config: FusedMoEQuantConfig, - defer_input_quant: bool = False, - ) -> mk.PrepareResultType: - if defer_input_quant: - raise NotImplementedError( - f"{self.__class__.__name__} does not support defer_input_quant=True. " - "Please select an MoE kernel that accepts quantized inputs." - ) - assert a1.dim() == 2 - assert topk_ids.dim() == 2 - assert topk_ids.size(0) == a1.size(0) - - if apply_router_weight_on_input: - topk = topk_ids.size(1) - # TODO: this only works for topK=1, will need to update for topK>1 - assert topk == 1, ( - "apply_router_weight_on_input is only implemented for topk=1" - ) - a1.mul_(topk_weights.to(a1.dtype)) - - num_tokens, hidden_dim = a1.size() - topk = topk_ids.size(1) - - tokens_per_expert = torch.zeros(num_experts, dtype=torch.int, device=a1.device) - - num_local_experts = self.num_local_experts - - if quant_config.quant_dtype is None: - b_type = a1.dtype - else: - b_type = quant_config.quant_dtype - - b_a1 = torch.zeros( - (num_local_experts, self.max_num_tokens, hidden_dim), - dtype=b_type, - device=a1.device, - ) - - if quant_config.is_quantized: - scale_shape = quant_config.batched_scale_shape( - num_local_experts, self.max_num_tokens, hidden_dim - ) - - b_a1_scale = torch.empty(scale_shape, dtype=torch.float32, device=a1.device) - else: - assert quant_config.a1_scale is None - b_a1_scale = None - - first_expert = num_local_experts * self.rank - last_expert = first_expert + num_local_experts - - a1_scale = normalize_scales_shape(quant_config.a1_scale) - - for expert_id in range(first_expert, last_expert): - topks = torch.any(topk_ids == expert_id, dim=1).flatten() - rows = torch.count_nonzero(topks.flatten()) - if rows == 0: - continue - idx = expert_id - first_expert - tokens_per_expert[idx] = rows - rhs = a1[: topks.numel()][topks] - if quant_config.quant_dtype is not None: - if a1_scale is not None: - if quant_config.is_per_act_token: - rhs_a1_scale = a1_scale[: topks.numel()][topks] - else: - rhs_a1_scale = a1_scale - else: - rhs_a1_scale = None - b_a1[idx, :rows, :], b_s = moe_kernel_quantize_input( - rhs, - rhs_a1_scale, - quant_config.quant_dtype, - quant_config.per_act_token_quant, - quant_config.block_shape, - ) - assert b_s is not None - if quant_config.is_per_act_token: - b_a1_scale[idx, :rows] = b_s[:rows] - else: - b_a1_scale[idx, : b_s.shape[0]] = b_s - else: - b_a1[idx, :rows, :] = rhs - - assert b_a1_scale is None or b_a1_scale.ndim == 3 - - expert_tokens_meta = mk.ExpertTokensMetadata( - expert_num_tokens=tokens_per_expert, expert_num_tokens_cpu=None - ) - - return b_a1, b_a1_scale, expert_tokens_meta, None, None - - def finalize( - self, - output: torch.Tensor, - fused_expert_output: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - apply_router_weight_on_input: bool, - weight_and_reduce_impl: mk.TopKWeightAndReduce, - ) -> None: - if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate): - weight_and_reduce_impl = TopKWeightAndReduceNaiveBatched(self.rank) - weight_and_reduce_impl.apply( - output=output, - fused_expert_output=fused_expert_output, - topk_weights=topk_weights, - topk_ids=topk_ids, - apply_router_weight_on_input=apply_router_weight_on_input, - ) - - class NaiveBatchedExperts(mk.FusedMoEExpertsModular): """ A reference MoE expert class that operates on expert batched format, diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/__init__.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/__init__.py index d388ee41140..b3529c99565 100644 --- a/vllm/model_executor/layers/fused_moe/prepare_finalize/__init__.py +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/__init__.py @@ -1,6 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.model_executor.layers.fused_moe.prepare_finalize.batched import ( + BatchedPrepareAndFinalize, +) from vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep import ( MoEPrepareAndFinalizeNaiveDPEPModular, MoEPrepareAndFinalizeNaiveDPEPMonolithic, @@ -13,6 +16,7 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import ( ) __all__ = [ + "BatchedPrepareAndFinalize", "MoEPrepareAndFinalizeNaiveDPEPMonolithic", "MoEPrepareAndFinalizeNaiveDPEPModular", "make_moe_prepare_and_finalize_naive_dp_ep", diff --git a/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py new file mode 100644 index 00000000000..943027717bb --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/prepare_finalize/batched.py @@ -0,0 +1,171 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceDelegate, + TopKWeightAndReduceNaiveBatched, +) +from vllm.model_executor.layers.fused_moe.utils import ( + moe_kernel_quantize_input, + normalize_scales_shape, +) + + +class BatchedPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular): + """ + A reference prepare/finalize class that reorganizes the tokens into + expert batched format, i.e. E x max_num_tokens x K. This is the format + that the batched dispatch/combine kernels use. + """ + + def __init__( + self, + max_num_tokens: int, + num_local_experts: int, + num_dispatchers: int, + rank: int, + ): + super().__init__() + self.max_num_tokens = max_num_tokens + self.num_local_experts = num_local_experts + self.rank = rank + self.num_dispatchers_ = num_dispatchers + + @property + def activation_format(self) -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.BatchedExperts + + def max_num_tokens_per_rank(self) -> int | None: + return self.max_num_tokens + + def topk_indices_dtype(self) -> torch.dtype | None: + return None + + def num_dispatchers(self) -> int: + return self.num_dispatchers_ + + def output_is_reduced(self) -> bool: + return False + + def prepare( + self, + a1: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + num_experts: int, + expert_map: torch.Tensor | None, + apply_router_weight_on_input: bool, + quant_config: FusedMoEQuantConfig, + defer_input_quant: bool = False, + ) -> mk.PrepareResultType: + if defer_input_quant: + raise NotImplementedError( + f"{self.__class__.__name__} does not support defer_input_quant=True. " + "Please select an MoE kernel that accepts quantized inputs." + ) + assert a1.dim() == 2 + assert topk_ids.dim() == 2 + assert topk_ids.size(0) == a1.size(0) + + if apply_router_weight_on_input: + topk = topk_ids.size(1) + # TODO: this only works for topK=1, will need to update for topK>1 + assert topk == 1, ( + "apply_router_weight_on_input is only implemented for topk=1" + ) + a1.mul_(topk_weights.to(a1.dtype)) + + num_tokens, hidden_dim = a1.size() + topk = topk_ids.size(1) + + tokens_per_expert = torch.zeros(num_experts, dtype=torch.int, device=a1.device) + + num_local_experts = self.num_local_experts + + if quant_config.quant_dtype is None: + b_type = a1.dtype + else: + b_type = quant_config.quant_dtype + + b_a1 = torch.zeros( + (num_local_experts, self.max_num_tokens, hidden_dim), + dtype=b_type, + device=a1.device, + ) + + if quant_config.is_quantized: + scale_shape = quant_config.batched_scale_shape( + num_local_experts, self.max_num_tokens, hidden_dim + ) + + b_a1_scale = torch.empty(scale_shape, dtype=torch.float32, device=a1.device) + else: + assert quant_config.a1_scale is None + b_a1_scale = None + + first_expert = num_local_experts * self.rank + last_expert = first_expert + num_local_experts + + a1_scale = normalize_scales_shape(quant_config.a1_scale) + + for expert_id in range(first_expert, last_expert): + topks = torch.any(topk_ids == expert_id, dim=1).flatten() + rows = torch.count_nonzero(topks.flatten()) + if rows == 0: + continue + idx = expert_id - first_expert + tokens_per_expert[idx] = rows + rhs = a1[: topks.numel()][topks] + if quant_config.quant_dtype is not None: + if a1_scale is not None: + if quant_config.is_per_act_token: + rhs_a1_scale = a1_scale[: topks.numel()][topks] + else: + rhs_a1_scale = a1_scale + else: + rhs_a1_scale = None + b_a1[idx, :rows, :], b_s = moe_kernel_quantize_input( + rhs, + rhs_a1_scale, + quant_config.quant_dtype, + quant_config.per_act_token_quant, + quant_config.block_shape, + ) + assert b_s is not None + if quant_config.is_per_act_token: + b_a1_scale[idx, :rows] = b_s[:rows] + else: + b_a1_scale[idx, : b_s.shape[0]] = b_s + else: + b_a1[idx, :rows, :] = rhs + + assert b_a1_scale is None or b_a1_scale.ndim == 3 + + expert_tokens_meta = mk.ExpertTokensMetadata( + expert_num_tokens=tokens_per_expert, expert_num_tokens_cpu=None + ) + + return b_a1, b_a1_scale, expert_tokens_meta, None, None + + def finalize( + self, + output: torch.Tensor, + fused_expert_output: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + apply_router_weight_on_input: bool, + weight_and_reduce_impl: mk.TopKWeightAndReduce, + ) -> None: + if isinstance(weight_and_reduce_impl, TopKWeightAndReduceDelegate): + weight_and_reduce_impl = TopKWeightAndReduceNaiveBatched(self.rank) + weight_and_reduce_impl.apply( + output=output, + fused_expert_output=fused_expert_output, + topk_weights=topk_weights, + topk_ids=topk_ids, + apply_router_weight_on_input=apply_router_weight_on_input, + ) diff --git a/vllm/model_executor/layers/fused_moe/mori_prepare_finalize.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py similarity index 100% rename from vllm/model_executor/layers/fused_moe/mori_prepare_finalize.py rename to vllm/model_executor/layers/fused_moe/prepare_finalize/mori.py diff --git a/vllm/model_executor/layers/fused_moe/nixl_ep_prepare_finalize.py b/vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py similarity index 100% rename from vllm/model_executor/layers/fused_moe/nixl_ep_prepare_finalize.py rename to vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py From fa4b70555bc1163c2ee8fa4b193be9957f05dda6 Mon Sep 17 00:00:00 2001 From: Hemanth Acharya Date: Fri, 24 Apr 2026 05:32:12 +0530 Subject: [PATCH 083/153] [ROCm] Cast score correction bias tensor during model construction for DeepSeek/Kimi-K2 (#39999) Signed-off-by: Hemanth Acharya --- vllm/_aiter_ops.py | 2 ++ .../layers/fused_moe/rocm_aiter_fused_moe.py | 2 +- .../fused_moe/router/fused_topk_bias_router.py | 2 +- vllm/model_executor/models/deepseek_v2.py | 15 +++++++++++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 8dbe49f07fc..0250fbfac70 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -1782,6 +1782,8 @@ class rocm_aiter_ops: need_renorm: bool, routed_scaling_factor: float = 1.0, ) -> None: + if correction_bias.dtype != gating_output.dtype: + correction_bias = correction_bias.to(gating_output.dtype) torch.ops.vllm.rocm_aiter_biased_grouped_topk( gating_output, correction_bias, diff --git a/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py b/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py index d24bda101ff..495b9daaff4 100644 --- a/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py +++ b/vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py @@ -152,7 +152,7 @@ def rocm_aiter_grouped_topk( if e_score_correction_bias is not None: rocm_aiter_ops.biased_grouped_topk( gating_output, - e_score_correction_bias.to(gating_output.dtype), + e_score_correction_bias, topk_weights, topk_ids, num_expert_group, diff --git a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py index bcabb1f3672..a5361b399e2 100644 --- a/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py +++ b/vllm/model_executor/layers/fused_moe/router/fused_topk_bias_router.py @@ -136,7 +136,7 @@ def fused_topk_bias( ) rocm_aiter_ops.biased_grouped_topk( gating_output, - e_score_correction_bias.to(gating_output.dtype), + e_score_correction_bias, topk_weights, topk_ids, num_expert_group=num_expert_group, diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index 3d0b1c42458..d91a41eaa7f 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -349,6 +349,21 @@ class DeepseekV2MoE(nn.Module): else torch.bfloat16 ) + # Pre-cast the bias to match the gate output dtype so the + # conversion is not repeated on every forward pass. All + # downstream references (FusedMoE, router) share the same + # nn.Parameter object, so mutating .data propagates everywhere. + # Weight loading uses copy_(), which handles the dtype conversion. + # Only needed on ROCm where the aiter biased_grouped_topk kernel + # requires the bias dtype to match the gating output dtype. + if ( + self.is_rocm_aiter_moe_enabled + and self.gate.e_score_correction_bias is not None + ): + self.gate.e_score_correction_bias.data = ( + self.gate.e_score_correction_bias.data.to(self.gate.out_dtype) + ) + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: num_tokens, hidden_dim = hidden_states.shape hidden_states = hidden_states.view(-1, hidden_dim) From 62b1bbe470ea820b728cf2b6eee4f5f8b9655a57 Mon Sep 17 00:00:00 2001 From: Sage Moore Date: Thu, 23 Apr 2026 17:21:15 -0700 Subject: [PATCH 084/153] [EPLB] Remove asyncio infrastructure from Async EPLB (#40730) Signed-off-by: Sage Moore --- tests/distributed/test_eplb_execute.py | 19 ++++++++----------- vllm/distributed/eplb/async_worker.py | 21 +++++++-------------- vllm/distributed/eplb/rebalance_execute.py | 2 +- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/tests/distributed/test_eplb_execute.py b/tests/distributed/test_eplb_execute.py index 7f8895cd2c1..d9e6a739b01 100644 --- a/tests/distributed/test_eplb_execute.py +++ b/tests/distributed/test_eplb_execute.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import asyncio import random import pytest @@ -361,16 +360,14 @@ def _test_async_transfer_layer_without_mtp_worker( communicator.set_stream(cuda_stream) for layer_idx in range(num_layers): - transfer_metadata = asyncio.run( - transfer_layer( - old_layer_indices=old_indices_cpu[layer_idx], - new_layer_indices=new_indices_cpu[layer_idx], - expert_weights=expert_weights[layer_idx], - expert_weights_buffer=expert_buffer, - ep_group=ep_group, - communicator=communicator, - cuda_stream=cuda_stream, - ) + transfer_metadata = transfer_layer( + old_layer_indices=old_indices_cpu[layer_idx], + new_layer_indices=new_indices_cpu[layer_idx], + expert_weights=expert_weights[layer_idx], + expert_weights_buffer=expert_buffer, + ep_group=ep_group, + communicator=communicator, + cuda_stream=cuda_stream, ) cuda_stream.synchronize() move_from_buffer( diff --git a/vllm/distributed/eplb/async_worker.py b/vllm/distributed/eplb/async_worker.py index a47b5ce29c2..542606fe741 100644 --- a/vllm/distributed/eplb/async_worker.py +++ b/vllm/distributed/eplb/async_worker.py @@ -4,7 +4,6 @@ The async worker that transfers experts in the background. """ -import asyncio import threading from typing import TYPE_CHECKING @@ -36,21 +35,15 @@ def start_async_worker( assert device_index is not None torch.accelerator.set_device_index(device_index) cuda_stream = torch.cuda.Stream(device=device_index) - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) try: - loop.run_until_complete( - transfer_run_periodically( - state=state, - eplb_group=eplb_group, - cuda_stream=cuda_stream, - is_profile=is_profile, - ) + transfer_run_periodically( + state=state, + eplb_group=eplb_group, + cuda_stream=cuda_stream, + is_profile=is_profile, ) except Exception as exc: # pragma: no cover - diagnostic path logger.exception("async loop error (Rank %d): %s", rank, str(exc)) - finally: - loop.close() thread = threading.Thread(target=thread_target, daemon=True) thread.start() @@ -83,7 +76,7 @@ def run_rebalance_experts( return new_physical_to_logical_map -async def transfer_run_periodically( +def transfer_run_periodically( state: "EplbState", eplb_group: ProcessGroup, cuda_stream: torch.cuda.Stream, @@ -118,7 +111,7 @@ async def transfer_run_periodically( # model_state.expert_buffer, which will be consumed by the main thread in # move_to_workspace while model_state.rebalanced and layer_idx < num_layers: - transfer_metadata = await transfer_layer( + transfer_metadata = transfer_layer( old_layer_indices=physical_to_logical_map_cpu[layer_idx], new_layer_indices=new_physical_to_logical_map[layer_idx], expert_weights=model_state.model.expert_weights[layer_idx], diff --git a/vllm/distributed/eplb/rebalance_execute.py b/vllm/distributed/eplb/rebalance_execute.py index a68fbda86cc..f348521c00e 100644 --- a/vllm/distributed/eplb/rebalance_execute.py +++ b/vllm/distributed/eplb/rebalance_execute.py @@ -418,7 +418,7 @@ def move_from_buffer( w[dst].copy_(w[src], non_blocking=True) -async def transfer_layer( +def transfer_layer( old_layer_indices: torch.Tensor, new_layer_indices: torch.Tensor, expert_weights: Sequence[torch.Tensor], From fe85a92e86211d066b07aba2a2a86640f70458d7 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 23 Apr 2026 17:35:55 -0700 Subject: [PATCH 085/153] [Core] Avoid seq_lens_cpu GPU->CPU sync (#40654) Signed-off-by: Nick Hill --- tests/v1/attention/utils.py | 1 + tests/v1/spec_decode/test_tree_attention.py | 4 ++- .../layers/attention/cross_attention.py | 16 +++++++--- .../layers/attention/mla_attention.py | 15 ++++++---- vllm/v1/attention/backend.py | 6 ++++ vllm/v1/attention/backends/flex_attention.py | 9 +++--- .../attention/backends/mla/flashmla_sparse.py | 5 +++- vllm/v1/attention/backends/mla/indexer.py | 8 +++-- vllm/v1/attention/backends/utils.py | 8 ++++- vllm/v1/spec_decode/dflash.py | 7 +++++ vllm/v1/spec_decode/llm_base_proposer.py | 10 ++++++- vllm/v1/worker/gpu/attn_utils.py | 4 +++ vllm/v1/worker/gpu/input_batch.py | 5 ++++ vllm/v1/worker/gpu/model_runner.py | 18 +++++++++++ vllm/v1/worker/gpu/model_states/default.py | 9 +++++- vllm/v1/worker/gpu/model_states/whisper.py | 8 ++++- vllm/v1/worker/gpu/states.py | 3 ++ vllm/v1/worker/gpu_model_runner.py | 2 ++ vllm/v1/worker/ubatch_utils.py | 30 +++++++++++++++---- 19 files changed, 142 insertions(+), 26 deletions(-) diff --git a/tests/v1/attention/utils.py b/tests/v1/attention/utils.py index aac4a46be3b..1d5eba74693 100644 --- a/tests/v1/attention/utils.py +++ b/tests/v1/attention/utils.py @@ -107,6 +107,7 @@ def create_common_attn_metadata( query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc_cpu, seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens_cpu, _seq_lens_cpu=seq_lens_cpu, _num_computed_tokens_cpu=num_computed_tokens_cpu, num_reqs=batch_spec.batch_size, diff --git a/tests/v1/spec_decode/test_tree_attention.py b/tests/v1/spec_decode/test_tree_attention.py index 1b6fa4f6f48..3c126c49f8c 100644 --- a/tests/v1/spec_decode/test_tree_attention.py +++ b/tests/v1/spec_decode/test_tree_attention.py @@ -241,11 +241,13 @@ def forward_attention( ) kv_cache_spec = create_standard_kv_cache_spec(vllm_config) builder = builder_cls(kv_cache_spec, [], vllm_config, q.device) + seq_lens_cpu = seq_lens.cpu() common_attn_metadata = CommonAttentionMetadata( query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc.cpu(), seq_lens=seq_lens, - _seq_lens_cpu=seq_lens.cpu(), + seq_lens_cpu_upper_bound=seq_lens_cpu, + _seq_lens_cpu=seq_lens_cpu, _num_computed_tokens_cpu=context_lens.cpu(), num_reqs=batch_size, num_actual_tokens=num_actual_tokens, diff --git a/vllm/model_executor/layers/attention/cross_attention.py b/vllm/model_executor/layers/attention/cross_attention.py index 312f906abac..091f0a1856d 100644 --- a/vllm/model_executor/layers/attention/cross_attention.py +++ b/vllm/model_executor/layers/attention/cross_attention.py @@ -90,15 +90,23 @@ def create_cross_attention_backend( assert new_metadata.encoder_seq_lens_cpu is not None max_encoder_len = int(new_metadata.encoder_seq_lens_cpu.max()) new_metadata.max_seq_len = max_encoder_len - # Any computed tokens indicated decode step>1 (no chunked prefill) - num_cache_decodes = ( - (common_attn_metadata.num_computed_tokens_cpu > 0).sum().item() + # Any computed tokens indicates decode step>1 (no chunked prefill). + # The upper bound is exact for this `> 0` test - prefill rows have + # num_computed == 0 and decode rows have num_computed > 0. + query_lens_cpu = ( + common_attn_metadata.query_start_loc_cpu[1:] + - common_attn_metadata.query_start_loc_cpu[:-1] ) + assert common_attn_metadata.seq_lens_cpu_upper_bound is not None + num_computed_tokens_cpu = ( + common_attn_metadata.seq_lens_cpu_upper_bound - query_lens_cpu + ) + num_cache_decodes = (num_computed_tokens_cpu > 0).sum().item() if num_cache_decodes > 0: # CrossAttn KV cache has already been populated on first decoder step, # skip slot_mapping calculation for requests that do not need # reshape_and_cache. - num_tokens = common_attn_metadata.num_computed_tokens_cpu.numpy() + num_tokens = num_computed_tokens_cpu.numpy() new_metadata.encoder_seq_lens_cpu = np.where( num_tokens > 0, 0, new_metadata.encoder_seq_lens_cpu ) diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 5c7dc60fe15..e649d790e82 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1822,13 +1822,18 @@ class MLACommonMetadataBuilder(AttentionMetadataBuilder[M]): prefill_metadata = None if num_prefills > 0: - num_computed_tokens_cpu = ( - common_attn_metadata.compute_num_computed_tokens().cpu() - ) - reqs_start = num_decodes # prefill_start - context_lens_cpu = num_computed_tokens_cpu[reqs_start:num_reqs] + # Upper bound is exact for prefill rows (no D2H sync). + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound + assert seq_lens_cpu is not None + prefill_query_lens_cpu = ( + query_start_loc_cpu[reqs_start + 1 : num_reqs + 1] + - query_start_loc_cpu[reqs_start:num_reqs] + ) + context_lens_cpu = ( + seq_lens_cpu[reqs_start:num_reqs] - prefill_query_lens_cpu + ) max_context_len_cpu = context_lens_cpu.max().item() num_prefills_with_context_cpu = (context_lens_cpu > 0).sum().item() prefill_query_start_loc = ( diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 7d6bba4189d..16535ee3c6c 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -397,6 +397,12 @@ class CommonAttentionMetadata: (num_computed_tokens < num_prompt_tokens). Used by some backends to distinguish actual decodes from short extends.""" + seq_lens_cpu_upper_bound: torch.Tensor | None = None + """(batch_size,) CPU upper bound on seq_lens. Precise for prefill rows + and for all rows outside async spec decode; optimistic for async-spec + decode rows (assumes every draft was accepted). Not safe for kernels + that need exact per-row context lengths on decode rows.""" + # WARNING: Deprecated fields. Will be removed in a future release (v0.15.0) _seq_lens_cpu: torch.Tensor | None = None _num_computed_tokens_cpu: torch.Tensor | None = None diff --git a/vllm/v1/attention/backends/flex_attention.py b/vllm/v1/attention/backends/flex_attention.py index a027fe52441..a917235ed8c 100644 --- a/vllm/v1/attention/backends/flex_attention.py +++ b/vllm/v1/attention/backends/flex_attention.py @@ -782,10 +782,11 @@ class FlexAttentionMetadataBuilder(AttentionMetadataBuilder[FlexAttentionMetadat def build_for_cudagraph_capture( self, common_attn_metadata: CommonAttentionMetadata ) -> FlexAttentionMetadata: - # Use actual max_seq_len instead of max_model_len to avoid - # torch.compile recompilation during CUDA graph capture. - common_attn_metadata.max_seq_len = ( - common_attn_metadata.seq_lens_cpu.max().item() + # Use actual max_seq_len (not max_model_len) to avoid torch.compile + # recompilation during CUDA graph capture. + assert common_attn_metadata.seq_lens_cpu_upper_bound is not None + common_attn_metadata.max_seq_len = int( + common_attn_metadata.seq_lens_cpu_upper_bound.max().item() ) return self.build( common_prefix_len=0, common_attn_metadata=common_attn_metadata diff --git a/vllm/v1/attention/backends/mla/flashmla_sparse.py b/vllm/v1/attention/backends/mla/flashmla_sparse.py index 1d981717cbf..e67282aab8c 100644 --- a/vllm/v1/attention/backends/mla/flashmla_sparse.py +++ b/vllm/v1/attention/backends/mla/flashmla_sparse.py @@ -364,7 +364,10 @@ class FlashMLASparseMetadataBuilder(AttentionMetadataBuilder[FlashMLASparseMetad # For pure decode batches, prefill_request_id will be None # For mixed batches, it will have -1 for decode and request_id for prefill if num_prefills > 0: - seq_lens_cpu = common_attn_metadata.seq_lens.cpu() + # Upper bound is exact for prefill rows (the `[num_decodes:]` + # slice below), so no D2H sync is needed. + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound + assert seq_lens_cpu is not None seq_lens = common_attn_metadata.seq_lens query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu diff --git a/vllm/v1/attention/backends/mla/indexer.py b/vllm/v1/attention/backends/mla/indexer.py index 3b719d10ff8..237ccfeb472 100644 --- a/vllm/v1/attention/backends/mla/indexer.py +++ b/vllm/v1/attention/backends/mla/indexer.py @@ -554,8 +554,12 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): query_start_loc_cpu[num_decodes : num_decodes + num_prefills + 1] ) max_logits_bytes = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024 + # Upper bound is exact for prefill rows (the `[num_decodes:]` + # slice below). + assert common_attn_metadata.seq_lens_cpu_upper_bound is not None + seq_lens_cpu = common_attn_metadata.seq_lens_cpu_upper_bound chunk_specs = split_indexer_prefill_chunks( - common_attn_metadata.seq_lens_cpu[num_decodes:], + seq_lens_cpu[num_decodes:], prefill_query_lens_cpu, self.max_prefill_buffer_size, max_logits_bytes, @@ -566,7 +570,7 @@ class DeepseekV32IndexerMetadataBuilder(AttentionMetadataBuilder): req_slice, query_slice, query_start_loc_cpu, - common_attn_metadata.seq_lens_cpu, + seq_lens_cpu, common_attn_metadata.block_table_tensor, skip_kv_gather=query_slice.start > 0, ) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index 0a36e6fd490..b4bdce876d8 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -356,6 +356,7 @@ def make_local_attention_virtual_batches( block_table_tensor=block_table_local, slot_mapping=common_attn_metadata.slot_mapping, causal=True, + seq_lens_cpu_upper_bound=seq_lens_cpu, _seq_lens_cpu=seq_lens_cpu, _num_computed_tokens_cpu=torch.from_numpy(num_computed_tokens_local), ), make_block_table @@ -414,6 +415,7 @@ def make_kv_sharing_fast_prefill_common_attn_metadata( block_table_tensor=common_attn_metadata.block_table_tensor, slot_mapping=common_attn_metadata.slot_mapping, causal=True, + seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound, _seq_lens_cpu=common_attn_metadata._seq_lens_cpu, _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, ) @@ -445,7 +447,11 @@ def split_decodes_prefills_and_extends( num_reqs = common_attn_metadata.num_reqs num_tokens = common_attn_metadata.num_actual_tokens query_start_loc = common_attn_metadata.query_start_loc_cpu - seq_lens = common_attn_metadata.seq_lens_cpu + # Upper bound is exact for prefill rows; decode rows still satisfy + # seq_len > query_len under the optimistic bound, so `seq_lens == + # query_lens` identifies prefills correctly either way. + assert common_attn_metadata.seq_lens_cpu_upper_bound is not None + seq_lens = common_attn_metadata.seq_lens_cpu_upper_bound if max_query_len <= decode_threshold: return num_reqs, 0, 0, num_tokens, 0, 0 diff --git a/vllm/v1/spec_decode/dflash.py b/vllm/v1/spec_decode/dflash.py index cb31a97a131..0d9d6809680 100644 --- a/vllm/v1/spec_decode/dflash.py +++ b/vllm/v1/spec_decode/dflash.py @@ -151,6 +151,12 @@ class DFlashProposer(SpecDecodeBaseProposer): if has_num_rejected: effective_seq_lens = effective_seq_lens - num_rejected_tokens_gpu + # Skip num_rejected_tokens (GPU-only); overestimating is fine here. + new_seq_lens_cpu_upper_bound = ( + cad.seq_lens_cpu_upper_bound + num_query_per_req + if cad.seq_lens_cpu_upper_bound is not None + else None + ) new_cad = CommonAttentionMetadata( query_start_loc=new_query_start_loc, seq_lens=effective_seq_lens + num_query_per_req, @@ -160,6 +166,7 @@ class DFlashProposer(SpecDecodeBaseProposer): ), _seq_lens_cpu=None, _num_computed_tokens_cpu=None, + seq_lens_cpu_upper_bound=new_seq_lens_cpu_upper_bound, num_reqs=cad.num_reqs, num_actual_tokens=num_query_total, max_query_len=num_query_per_req, diff --git a/vllm/v1/spec_decode/llm_base_proposer.py b/vllm/v1/spec_decode/llm_base_proposer.py index 1764ae8db4d..44156b60c0d 100644 --- a/vllm/v1/spec_decode/llm_base_proposer.py +++ b/vllm/v1/spec_decode/llm_base_proposer.py @@ -593,6 +593,8 @@ class SpecDecodeBaseProposer: common_attn_metadata._seq_lens_cpu += 1 if common_attn_metadata._num_computed_tokens_cpu is not None: common_attn_metadata._num_computed_tokens_cpu += 1 + if common_attn_metadata.seq_lens_cpu_upper_bound is not None: + common_attn_metadata.seq_lens_cpu_upper_bound += 1 # Rebuild attention metadata _, per_layer_attn_metadata = self.build_per_group_and_layer_attn_metadata( @@ -959,6 +961,7 @@ class SpecDecodeBaseProposer: query_start_loc_cpu=query_start_loc_cpu, _seq_lens_cpu=common_attn_metadata._seq_lens_cpu, _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, + seq_lens_cpu_upper_bound=common_attn_metadata.seq_lens_cpu_upper_bound, num_reqs=common_attn_metadata.num_reqs, num_actual_tokens=total_num_tokens, max_query_len=new_query_len_per_req.max().item(), @@ -1183,7 +1186,11 @@ class SpecDecodeBaseProposer: device = common_attn_metadata.query_start_loc.device query_start_loc_cpu = common_attn_metadata.query_start_loc_cpu - new_seq_lens_cpu = common_attn_metadata.seq_lens_cpu - num_rejected_tokens + # upper_bound - rejected = actual post-rejection seq_lens (no D2H sync). + assert common_attn_metadata.seq_lens_cpu_upper_bound is not None + new_seq_lens_cpu = ( + common_attn_metadata.seq_lens_cpu_upper_bound - num_rejected_tokens + ) # [0, q1, q1 + q2, q1 + q2 + q3] -> [q1, q2, q3] new_query_len_per_req = query_start_loc_cpu[1:] - query_start_loc_cpu[:-1] @@ -1237,6 +1244,7 @@ class SpecDecodeBaseProposer: query_start_loc_cpu=new_query_start_loc_cpu, _seq_lens_cpu=new_seq_lens_cpu, _num_computed_tokens_cpu=common_attn_metadata._num_computed_tokens_cpu, + seq_lens_cpu_upper_bound=new_seq_lens_cpu, num_reqs=common_attn_metadata.num_reqs, num_actual_tokens=total_num_tokens, max_query_len=new_query_len_per_req.max().item(), diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index ee6244c42a0..354be3cd2a4 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -227,12 +227,15 @@ def build_attn_metadata( block_tables: Sequence[torch.Tensor], slot_mappings: torch.Tensor, kv_cache_config: KVCacheConfig, + seq_lens_cpu_upper_bound: torch.Tensor | None = None, dcp_local_seq_lens: torch.Tensor | None = None, encoder_seq_lens: dict[int, tuple[torch.Tensor, np.ndarray]] | None = None, ) -> dict[str, Any]: seq_lens = seq_lens[:num_reqs] if dcp_local_seq_lens is not None: dcp_local_seq_lens = dcp_local_seq_lens[:num_reqs] + if seq_lens_cpu_upper_bound is not None: + seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound[:num_reqs] attn_metadata: dict[str, Any] = {} num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) @@ -244,6 +247,7 @@ def build_attn_metadata( query_start_loc=query_start_loc_gpu, query_start_loc_cpu=query_start_loc_cpu, seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, max_seq_len=max_seq_len, num_reqs=num_reqs, num_actual_tokens=num_tokens, diff --git a/vllm/v1/worker/gpu/input_batch.py b/vllm/v1/worker/gpu/input_batch.py index 24df137cb31..be14de272a4 100644 --- a/vllm/v1/worker/gpu/input_batch.py +++ b/vllm/v1/worker/gpu/input_batch.py @@ -60,6 +60,8 @@ class InputBatch: query_start_loc_np: np.ndarray # [num_reqs] seq_lens: torch.Tensor + # [num_reqs] CPU upper bound on seq_lens (see CommonAttentionMetadata). + seq_lens_cpu_upper_bound: torch.Tensor # [num_reqs] dcp_local_seq_lens: torch.Tensor | None @@ -121,6 +123,8 @@ class InputBatch: logits_indices = query_start_loc[1:] - 1 cu_num_logits = torch.arange(num_reqs + 1, device=device, dtype=torch.int32) cu_num_logits_np = np.arange(num_reqs + 1, dtype=np.int32) + # Dummy: seq_len == query_len (fresh-prefill shape). + seq_lens_cpu_upper_bound = torch.from_numpy(num_scheduled_tokens.copy()) return cls( req_ids=req_ids, num_reqs=num_reqs, @@ -136,6 +140,7 @@ class InputBatch: query_start_loc=query_start_loc, query_start_loc_np=query_start_loc_np, seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=None, input_ids=input_ids, positions=positions, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index a0025d8c795..820704ecff3 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -799,6 +799,15 @@ class GPUModelRunner(LoRAModelRunnerMixin): total_num_logits, ) + # CPU upper bound on seq_lens; padded entries left at zero. + seq_lens_cpu_upper_bound_np = np.zeros(num_reqs_padded, dtype=np.int32) + np.add( + self.req_states.num_computed_tokens_np[idx_mapping_np], + num_scheduled_tokens, + out=seq_lens_cpu_upper_bound_np[:num_reqs], + ) + seq_lens_cpu_upper_bound = torch.from_numpy(seq_lens_cpu_upper_bound_np) + return InputBatch( req_ids=req_ids, num_reqs=num_reqs, @@ -814,6 +823,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): query_start_loc=query_start_loc, query_start_loc_np=query_start_loc_np, seq_lens=seq_lens, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=dcp_local_seq_lens, input_ids=self.input_buffers.input_ids[:num_tokens_after_padding], positions=self.input_buffers.positions[:num_tokens_after_padding], @@ -927,6 +937,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): np.minimum( computed_prefill, self.req_states.prefill_len.np, out=computed_prefill ) + # Advance the CPU mirror optimistically (assume all scheduled accepted). + self.req_states.num_computed_tokens_np[idx_mapping_np] += ( + input_batch.num_scheduled_tokens + ) @torch.inference_mode() def execute_model( @@ -1297,6 +1311,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): np.minimum( computed_prefill, self.req_states.prefill_len.np, out=computed_prefill ) + # Advance the CPU mirror optimistically (assume all scheduled accepted). + self.req_states.num_computed_tokens_np[idx_mapping_np] += ( + input_batch.num_scheduled_tokens + ) ########### EPLB methods start ########### @property diff --git a/vllm/v1/worker/gpu/model_states/default.py b/vllm/v1/worker/gpu/model_states/default.py index 8e73867deb2..5d36b12f9c2 100644 --- a/vllm/v1/worker/gpu/model_states/default.py +++ b/vllm/v1/worker/gpu/model_states/default.py @@ -173,6 +173,12 @@ class DefaultModelState(ModelState): num_tokens = input_batch.num_tokens query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) max_query_len = input_batch.num_scheduled_tokens.max().item() + seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound + if for_capture: + # Capture with worst-case max_seq_len so the graph is valid at any replay. + max_seq_len = self.max_model_len + else: + max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item()) attn_metadata = build_attn_metadata( attn_groups=attn_groups, num_reqs=num_reqs, @@ -181,10 +187,11 @@ class DefaultModelState(ModelState): query_start_loc_cpu=query_start_loc_cpu, max_query_len=max_query_len, seq_lens=input_batch.seq_lens, - max_seq_len=self.max_model_len, + max_seq_len=max_seq_len, block_tables=block_tables, slot_mappings=slot_mappings, kv_cache_config=kv_cache_config, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=input_batch.dcp_local_seq_lens, ) return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/whisper.py b/vllm/v1/worker/gpu/model_states/whisper.py index 1268fee8821..a6faea482c2 100644 --- a/vllm/v1/worker/gpu/model_states/whisper.py +++ b/vllm/v1/worker/gpu/model_states/whisper.py @@ -117,6 +117,11 @@ class WhisperModelState(ModelState): query_start_loc_cpu = torch.from_numpy(input_batch.query_start_loc_np) max_query_len = input_batch.num_scheduled_tokens.max().item() + seq_lens_cpu_upper_bound = input_batch.seq_lens_cpu_upper_bound + if for_capture: + max_seq_len = self.max_model_len + else: + max_seq_len = int(seq_lens_cpu_upper_bound[:num_reqs].max().item()) attn_metadata = build_attn_metadata( attn_groups=attn_groups, num_reqs=num_reqs, @@ -125,10 +130,11 @@ class WhisperModelState(ModelState): query_start_loc_cpu=query_start_loc_cpu, max_query_len=max_query_len, seq_lens=input_batch.seq_lens, - max_seq_len=self.max_model_len, + max_seq_len=max_seq_len, block_tables=block_tables, slot_mappings=slot_mappings, kv_cache_config=kv_cache_config, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, dcp_local_seq_lens=input_batch.dcp_local_seq_lens, encoder_seq_lens=encoder_seq_lens, ) diff --git a/vllm/v1/worker/gpu/states.py b/vllm/v1/worker/gpu/states.py index cc371d32a91..b2683966b31 100644 --- a/vllm/v1/worker/gpu/states.py +++ b/vllm/v1/worker/gpu/states.py @@ -57,6 +57,8 @@ class RequestState: self.num_computed_tokens = StagedWriteTensor( self.max_num_reqs, dtype=torch.int32, device=device ) + # Optimistic CPU mirror of num_computed_tokens (upper bound on GPU value). + self.num_computed_tokens_np = np.zeros(self.max_num_reqs, dtype=np.int32) # Last sampled tokens. self.last_sampled_tokens = torch.zeros( @@ -100,6 +102,7 @@ class RequestState: self.total_len.stage_write_elem(req_idx, prefill_len) self.all_token_ids.stage_write(req_idx, 0, all_token_ids) self.num_computed_prefill_tokens[req_idx] = num_computed_tokens + self.num_computed_tokens_np[req_idx] = num_computed_tokens self.num_computed_tokens.stage_write_elem(req_idx, num_computed_tokens) if num_computed_tokens > 0 and num_computed_tokens <= prefill_len: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 386db4fecd4..8aca4594137 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2155,6 +2155,7 @@ class GPUModelRunner( :num_reqs_padded ] seq_lens_cpu = self.optimistic_seq_lens_cpu[:num_reqs_padded] + seq_lens_cpu_upper_bound = seq_lens_cpu # is_prefilling: True if request is still in prefill phase. # Used by mamba backends to distinguish actual decodes from @@ -2172,6 +2173,7 @@ class GPUModelRunner( seq_lens=self.seq_lens[:num_reqs_padded], _seq_lens_cpu=seq_lens_cpu, _num_computed_tokens_cpu=num_computed_tokens_cpu, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, num_reqs=num_reqs_padded, num_actual_tokens=num_tokens_padded, max_query_len=max_query_len, diff --git a/vllm/v1/worker/ubatch_utils.py b/vllm/v1/worker/ubatch_utils.py index 7c41726472d..1338b46996f 100644 --- a/vllm/v1/worker/ubatch_utils.py +++ b/vllm/v1/worker/ubatch_utils.py @@ -177,7 +177,22 @@ def _make_metadata_with_slice( query_start_loc[1:] -= tokens_skipped query_start_loc_cpu[1:] -= tokens_skipped seq_lens = attn_metadata.seq_lens[request_slice] - seq_lens_cpu = attn_metadata.seq_lens_cpu[request_slice] + # Read raw fields to avoid triggering the deprecated D2H-syncing properties. + seq_lens_cpu = ( + attn_metadata._seq_lens_cpu[request_slice] + if attn_metadata._seq_lens_cpu is not None + else None + ) + seq_lens_cpu_upper_bound = ( + attn_metadata.seq_lens_cpu_upper_bound[request_slice] + if attn_metadata.seq_lens_cpu_upper_bound is not None + else None + ) + num_computed_tokens_cpu = ( + attn_metadata._num_computed_tokens_cpu[request_slice] + if attn_metadata._num_computed_tokens_cpu is not None + else None + ) if splits_last_request: # NOTE: We use start_locs (the original query_start_loc_cpu) to calculate @@ -190,12 +205,16 @@ def _make_metadata_with_slice( # Make sure we don't modify the seq_lens tensors # (not cudagraph compatible) seq_lens = seq_lens.clone() - seq_lens_cpu = seq_lens_cpu.clone() seq_lens[-1] -= tokens_skipped - seq_lens_cpu[-1] -= tokens_skipped + if seq_lens_cpu is not None: + seq_lens_cpu = seq_lens_cpu.clone() + seq_lens_cpu[-1] -= tokens_skipped + if seq_lens_cpu_upper_bound is not None: + seq_lens_cpu_upper_bound = seq_lens_cpu_upper_bound.clone() + seq_lens_cpu_upper_bound[-1] -= tokens_skipped - max_seq_len = int(seq_lens_cpu.max()) - num_computed_tokens_cpu = attn_metadata.num_computed_tokens_cpu[request_slice] + assert seq_lens_cpu_upper_bound is not None + max_seq_len = int(seq_lens_cpu_upper_bound.max()) num_requests = request_slice.stop - request_slice.start num_actual_tokens = token_slice.stop - token_slice.start @@ -221,6 +240,7 @@ def _make_metadata_with_slice( max_seq_len=max_seq_len, block_table_tensor=block_table_tensor, slot_mapping=slot_mapping, + seq_lens_cpu_upper_bound=seq_lens_cpu_upper_bound, _seq_lens_cpu=seq_lens_cpu, _num_computed_tokens_cpu=num_computed_tokens_cpu, ) From 626daa2076b00068572f570c2b1567786eeab141 Mon Sep 17 00:00:00 2001 From: Benjamin Chislett Date: Thu, 23 Apr 2026 20:48:08 -0400 Subject: [PATCH 086/153] [Feat] Unified Synthetic Acceptance Rate for V1 and V2 (#40662) Signed-off-by: Benjamin Chislett Signed-off-by: Benjamin Chislett Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- tests/v1/e2e/spec_decode/test_spec_decode.py | 48 ++++++++ tests/v1/sample/test_rejection_sampler.py | 61 +++++++++ .../test_synthetic_rejection_sampler_utils.py | 69 +++++++---- vllm/config/speculative.py | 81 +++++++++++- vllm/v1/sample/rejection_sampler.py | 116 +++++++++++++----- vllm/v1/spec_decode/utils.py | 6 + vllm/v1/worker/gpu/model_runner.py | 1 + .../gpu/spec_decode/rejection_sampler.py | 27 ++-- .../synthetic_rejection_sampler_utils.py | 64 +--------- vllm/v1/worker/gpu_model_runner.py | 4 +- 10 files changed, 340 insertions(+), 137 deletions(-) diff --git a/tests/v1/e2e/spec_decode/test_spec_decode.py b/tests/v1/e2e/spec_decode/test_spec_decode.py index 03448e9bb3e..926cdd830bc 100644 --- a/tests/v1/e2e/spec_decode/test_spec_decode.py +++ b/tests/v1/e2e/spec_decode/test_spec_decode.py @@ -1310,6 +1310,54 @@ def test_dflash_acceptance_rates(dflash_config): cleanup_dist_env_and_memory() +@single_gpu_only +def test_synthetic_acceptance_rate(): + """Verify that synthetic rejection sampling produces an acceptance + length close to the requested mean acceptance length.""" + num_spec_tokens = 3 + expected_acceptance_len = 1.875 + tolerance = 0.15 + + spec_llm = LLM( + model="meta-llama/Llama-3.2-1B-Instruct", + trust_remote_code=True, + speculative_config={ + "method": "eagle3", + "model": "nm-testing/Llama3_2_1B_speculator.eagle3", + "num_speculative_tokens": num_spec_tokens, + "max_model_len": 2048, + "rejection_sample_method": "synthetic", + "synthetic_acceptance_length": expected_acceptance_len, + }, + max_model_len=2048, + enforce_eager=True, + disable_log_stats=False, + ) + + test_prompts = get_test_prompts(mm_enabled=False, num_prompts=50) + spec_llm.chat( + test_prompts, + SamplingParams(temperature=0, max_tokens=64, ignore_eos=True), + ) + + metrics = spec_llm.get_metrics() + acceptance_len = compute_acceptance_len(metrics) + + print( + f"Synthetic acceptance length: {acceptance_len:.3f}" + f" (expected={expected_acceptance_len:.3f}," + f" tolerance=±{tolerance})" + ) + assert abs(acceptance_len - expected_acceptance_len) <= tolerance, ( + f"Synthetic acceptance length {acceptance_len:.3f} is not within" + f" ±{tolerance} of expected {expected_acceptance_len:.3f}" + ) + + del spec_llm + torch.accelerator.empty_cache() + cleanup_dist_env_and_memory() + + def test_dflash_correctness(dflash_config): """ E2E test for DFlash (block diffusion) speculative decoding. diff --git a/tests/v1/sample/test_rejection_sampler.py b/tests/v1/sample/test_rejection_sampler.py index ecfcade2b61..ae0cbeab53b 100644 --- a/tests/v1/sample/test_rejection_sampler.py +++ b/tests/v1/sample/test_rejection_sampler.py @@ -933,3 +933,64 @@ def test_sample_recovered_tokens( device=DEVICE_TYPE, ) assert torch.equal(recovered_token_ids, ref_recovered_token_ids) + + +########################### Tests for Synthetic Rejection Sampling ######### + + +def _make_synthetic_sampler(rates: list[float]) -> RejectionSampler: + mock_sampler = Mock(spec=Sampler) + mock_sampler.logprobs_mode = "raw_logprobs" + spec_config = Mock() + spec_config.rejection_sample_method = "synthetic" + spec_config.synthetic_acceptance_rates = rates + return RejectionSampler(mock_sampler, spec_config, torch.device(DEVICE_TYPE)) + + +def _make_sampling_metadata(all_greedy: bool) -> SamplingMetadata: + temperature = None if all_greedy else torch.tensor([1.0, 1.0], device=DEVICE_TYPE) + return create_sampling_metadata(all_greedy=all_greedy, temperature=temperature) + + +@pytest.mark.parametrize("all_greedy", [True, False]) +def test_synthetic_all_accepted(all_greedy: bool): + """With all rates=1.0, every draft token is accepted.""" + sampler = _make_synthetic_sampler([1.0, 1.0]) + spec_tokens = [[1, 2], [3]] + output_tokens = [[10, 20, 50], [30, 40]] + + metadata = _make_sampling_metadata(all_greedy) + logits = create_logits_tensor(output_tokens) + bonus = torch.tensor([50, 40], device=DEVICE_TYPE) + spec_decode_metadata = create_spec_decode_metadata(spec_tokens, logits) + + mock_sampler_output(sampler, bonus) + output = sampler(spec_decode_metadata, None, logits, metadata) + expected = torch.tensor( + [[1, 2, 50], [3, 40, PLACEHOLDER_TOKEN_ID]], + dtype=torch.int, + device=DEVICE_TYPE, + ) + assert torch.equal(output.sampled_token_ids, expected) + + +@pytest.mark.parametrize("all_greedy", [True, False]) +def test_synthetic_all_rejected(all_greedy: bool): + """With all rates=0.0, the first token is always rejected.""" + sampler = _make_synthetic_sampler([0.0, 0.0]) + spec_tokens = [[1, 2], [3]] + output_tokens = [[10, 20, 50], [30, 40]] + + metadata = _make_sampling_metadata(all_greedy) + logits = create_logits_tensor(output_tokens) + bonus = torch.tensor([50, 40], device=DEVICE_TYPE) + spec_decode_metadata = create_spec_decode_metadata(spec_tokens, logits) + + mock_sampler_output(sampler, bonus) + output = sampler(spec_decode_metadata, None, logits, metadata) + result = output.sampled_token_ids + # Exactly one token emitted per sequence (the rejection fallback), + # followed by placeholders. + for row in result: + assert row[0] != PLACEHOLDER_TOKEN_ID + assert (row[1:] == PLACEHOLDER_TOKEN_ID).all() diff --git a/tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py b/tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py index d817bc1b8fe..a5a23cf1b7e 100644 --- a/tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_synthetic_rejection_sampler_utils.py @@ -2,33 +2,48 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest -from vllm.v1.worker.gpu.spec_decode.synthetic_rejection_sampler_utils import ( - compute_synthetic_rejection_sampler_params, +from vllm.config.speculative import SpeculativeConfig +from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates + + +def test_unconditional_to_conditional_rates_basic(): + # c_0 = p_0; c_i = p_i / p_{i-1} + assert unconditional_to_conditional_rates([0.9, 0.5, 0.2]) == pytest.approx( + [0.9, 0.5 / 0.9, 0.2 / 0.5] + ) + + +def test_unconditional_to_conditional_rates_handles_zero(): + # After a zero, subsequent conditional rates are clamped to 0 (the chain + # has already terminated in the kernel, so these values are unused). + assert unconditional_to_conditional_rates([1.0, 0.6, 0.0, 0.0]) == pytest.approx( + [1.0, 0.6, 0.0, 0.0] + ) + + +def test_unconditional_to_conditional_rates_all_ones(): + assert unconditional_to_conditional_rates([1.0, 1.0, 1.0]) == pytest.approx( + [1.0, 1.0, 1.0] + ) + + +@pytest.mark.parametrize( + "length,n,expected", + [ + (2.6, 3, [1.0, 0.6, 0.0]), + (1.0, 3, [0.0, 0.0, 0.0]), + (4.0, 3, [1.0, 1.0, 1.0]), + (2.0, 3, [1.0, 0.0, 0.0]), + (3.5, 4, [1.0, 1.0, 0.5, 0.0]), + ], ) - -NUM_SPECULATIVE_STEPS = [1, 2, 3, 4, 5, 7, 10] -ACCEPTANCE_RATES = [i / 100 for i in range(0, 100)] +def test_acceptance_length_to_rates(length, n, expected): + assert SpeculativeConfig._acceptance_length_to_rates(length, n) == pytest.approx( + expected + ) -@pytest.mark.parametrize("num_speculative_steps", NUM_SPECULATIVE_STEPS) -def test_compute_synthetic_rejection_sampler_params(num_speculative_steps: int): - """Test that the base acceptance rate and decay factor generated for - synthetic rejection sampling have a mean joint acceptance probability - that matches the desired acceptance rate.""" - tol = 1e-9 - for desired_acceptance_rate in ACCEPTANCE_RATES: - base_rate, decay_factor = compute_synthetic_rejection_sampler_params( - desired_acceptance_rate, num_speculative_steps, tol=tol - ) - - # Compute the mean of joint acceptance probabilities across - # all speculative positions. - joint_prob = 1.0 - mean_joint = 0.0 - for i in range(num_speculative_steps): - joint_prob *= base_rate * decay_factor**i - mean_joint += joint_prob - mean_joint /= num_speculative_steps - - assert abs(desired_acceptance_rate - mean_joint) < 10 * tol - assert base_rate <= 1.0 +def test_resolve_length_produces_minvariance_schedule(): + assert SpeculativeConfig._resolve_synthetic_acceptance_rates( + 3, None, 2.6 + ) == pytest.approx([1.0, 0.6, 0.0]) diff --git a/vllm/config/speculative.py b/vllm/config/speculative.py index 4e6a47ee46c..a0c5cd04a16 100644 --- a/vllm/config/speculative.py +++ b/vllm/config/speculative.py @@ -189,12 +189,64 @@ class SpeculativeConfig: distribution, but the latter yields a higher acceptance rate at the cost of more memory to cache draft logits.""" - synthetic_acceptance_rate: float | None = None - """Average acceptance rate for synthetic rejection sampling. Draft - tokens are accepted with a position-dependent probability that decays - geometrically, calibrated so that the mean rate across all speculative - positions equals this value. Only used when rejection_sample_method - is 'synthetic'. Must be in [0, 1].""" + synthetic_acceptance_rates: list[float] | None = None + """Per-position *unconditional* acceptance rates for synthetic rejection + sampling. Position i's entry is the marginal probability that the first + i+1 draft tokens are all accepted; the list must have length + num_speculative_tokens, each entry in [0, 1], and be monotonically + non-increasing. Only valid when rejection_sample_method is 'synthetic'. + Mutually exclusive with synthetic_acceptance_length.""" + + synthetic_acceptance_length: float | None = None + """Target mean acceptance length for synthetic rejection sampling, in + [1, num_speculative_tokens + 1]. Resolved internally to + synthetic_acceptance_rates. Only valid when rejection_sample_method is 'synthetic'. + Mutually exclusive with synthetic_acceptance_rates.""" + + @staticmethod + def _acceptance_length_to_rates(length: float, n: int) -> list[float]: + """Mean acceptance length to unconditional per-position rates, using + the minimum-variance schedule.""" + num_drafts = length - 1 # expected number of accepted draft tokens + num_full = int(num_drafts) + return ( + [1.0] * num_full + [num_drafts - num_full] + [0.0] * (n - num_full - 1) + )[:n] + + @staticmethod + def _resolve_synthetic_acceptance_rates( + n: int, + rates: list[float] | None, + length: float | None, + ) -> list[float]: + """Return per-position unconditional acceptance rates from exactly one + of `rates` or `length` (validates range, length, and monotonicity).""" + if (rates is None) == (length is None): + raise ValueError( + "rejection_sample_method='synthetic' requires exactly one of " + "synthetic_acceptance_rates or synthetic_acceptance_length." + ) + if rates is not None: + if len(rates) != n: + raise ValueError( + f"synthetic_acceptance_rates must have length {n}, got {rates}." + ) + if not all(0.0 <= r <= 1.0 for r in rates): + raise ValueError( + f"synthetic_acceptance_rates entries must be in [0, 1], " + f"got {rates}." + ) + if any(rates[i] > rates[i - 1] for i in range(1, n)): + raise ValueError( + f"synthetic_acceptance_rates must be non-increasing, got {rates}." + ) + return list(rates) + assert length is not None + if not 1.0 <= length <= float(n + 1): + raise ValueError( + f"synthetic_acceptance_length must be in [1, {n + 1}], got {length}." + ) + return SpeculativeConfig._acceptance_length_to_rates(length, n) def compute_hash(self) -> str: """ @@ -818,6 +870,23 @@ class SpeculativeConfig: f"than zero ({self.num_speculative_tokens})." ) + if self.rejection_sample_method == "synthetic": + # Consolidate to per-position rates + self.synthetic_acceptance_rates = self._resolve_synthetic_acceptance_rates( + self.num_speculative_tokens, + self.synthetic_acceptance_rates, + self.synthetic_acceptance_length, + ) + self.synthetic_acceptance_length = None + elif ( + self.synthetic_acceptance_rates is not None + or self.synthetic_acceptance_length is not None + ): + raise ValueError( + "synthetic_acceptance_rates / synthetic_acceptance_length " + "are only valid with rejection_sample_method='synthetic'." + ) + if self.draft_model_config: self.draft_model_config.verify_with_parallel_config( self.draft_parallel_config diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py index d3e8573458b..2b63893c049 100644 --- a/vllm/v1/sample/rejection_sampler.py +++ b/vllm/v1/sample/rejection_sampler.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + from collections.abc import Sequence from dataclasses import replace +from typing import TYPE_CHECKING import torch import torch.nn as nn @@ -17,6 +20,10 @@ from vllm.v1.sample.ops.penalties import apply_all_penalties from vllm.v1.sample.ops.topk_topp_sampler import apply_top_k_top_p from vllm.v1.sample.sampler import Sampler from vllm.v1.spec_decode.metadata import SpecDecodeMetadata +from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates + +if TYPE_CHECKING: + from vllm.config.speculative import SpeculativeConfig logger = init_logger(__name__) @@ -50,13 +57,33 @@ class RejectionSampler(nn.Module): output tokens = accepted tokens + recovered tokens + bonus tokens """ - def __init__(self, sampler: Sampler): + def __init__( + self, + sampler: Sampler, + spec_config: SpeculativeConfig | None = None, + device: torch.device | None = None, + ): super().__init__() self.sampler = sampler logprobs_mode = self.sampler.logprobs_mode self.is_processed_logprobs_mode = logprobs_mode.startswith("processed") self.is_logits_logprobs_mode = logprobs_mode.endswith("logits") + self.synthetic_conditional_rates: torch.Tensor | None = None + if ( + spec_config is not None + and spec_config.rejection_sample_method == "synthetic" + ): + assert spec_config.synthetic_acceptance_rates is not None + self.synthetic_conditional_rates = torch.tensor( + unconditional_to_conditional_rates( + spec_config.synthetic_acceptance_rates + ), + dtype=torch.float32, + device=device, + ) + self.synthetic_mode = self.synthetic_conditional_rates is not None + def forward( self, metadata: SpecDecodeMetadata, @@ -147,6 +174,8 @@ class RejectionSampler(nn.Module): target_logits, bonus_token_ids, sampling_metadata, + synthetic_mode=self.synthetic_mode, + synthetic_conditional_rates=self.synthetic_conditional_rates, ) logprobs_tensors = None @@ -362,6 +391,8 @@ def rejection_sample( # [batch_size, 1] bonus_token_ids: torch.Tensor, sampling_metadata: SamplingMetadata, + synthetic_mode: bool = False, + synthetic_conditional_rates: torch.Tensor | None = None, ) -> torch.Tensor: assert draft_token_ids.ndim == 1 assert draft_probs is None or draft_probs.ndim == 2 @@ -389,6 +420,20 @@ def rejection_sample( is_greedy = None else: is_greedy = sampling_metadata.temperature == GREEDY_TEMPERATURE + + # Generate uniform probabilities before either kernel because synthetic + # mode needs them in the greedy kernel too. Skip only when all requests + # are greedy *and* synthetic mode is off (the standard fast-path). + # [num_tokens] + uniform_probs: torch.Tensor | None = None + if synthetic_mode or not sampling_metadata.all_greedy: + uniform_probs = generate_uniform_probs( + num_tokens, + num_draft_tokens, + sampling_metadata.generators, + device, + ) + if not sampling_metadata.all_random: # Rejection sampling for greedy sampling requests. target_argmax = target_logits.argmax(dim=-1) @@ -400,6 +445,9 @@ def rejection_sample( bonus_token_ids, is_greedy, max_spec_len, + uniform_probs, + synthetic_conditional_rates, + SYNTHETIC_MODE=synthetic_mode, ) if sampling_metadata.all_greedy: return output_token_ids @@ -408,15 +456,6 @@ def rejection_sample( target_probs = target_logits.softmax(dim=-1, dtype=torch.float32) assert target_probs.is_contiguous() - # Generate uniform probabilities for rejection sampling. - # [num_tokens] - uniform_probs = generate_uniform_probs( - num_tokens, - num_draft_tokens, - sampling_metadata.generators, - device, - ) - # Sample recovered tokens for each position. # [num_tokens] recovered_token_ids = sample_recovered_tokens( @@ -431,6 +470,7 @@ def rejection_sample( ) # Rejection sampling for random sampling requests. + assert uniform_probs is not None rejection_random_sample_kernel[(batch_size,)]( output_token_ids, cu_num_draft_tokens, @@ -443,7 +483,9 @@ def rejection_sample( is_greedy, max_spec_len, vocab_size, + synthetic_conditional_rates, NO_DRAFT_PROBS=draft_probs is None, + SYNTHETIC_MODE=synthetic_mode, ) return output_token_ids @@ -658,6 +700,9 @@ def rejection_greedy_sample_kernel( bonus_token_ids_ptr, # [batch_size] is_greedy_ptr, # [batch_size] or None max_spec_len, + uniform_probs_ptr, # [num_tokens] or None (synthetic mode only) + synthetic_conditional_rates_ptr, # [num_speculative_tokens] or None + SYNTHETIC_MODE: tl.constexpr, ): req_idx = tl.program_id(0) # FIXME(woosuk): Because is_greedy_ptr is not None at profiling run, @@ -675,14 +720,20 @@ def rejection_greedy_sample_kernel( for pos in range(num_draft_tokens): if not rejected: draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos) - target_argmax_id = tl.load(target_argmax_ptr + start_idx + pos) + target_argmax_id = tl.load(target_argmax_ptr + start_idx + pos).to(tl.int32) + if SYNTHETIC_MODE: + uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos) + rate = tl.load(synthetic_conditional_rates_ptr + pos) + accepted = uniform_prob < rate + token_id = draft_token_id if accepted else target_argmax_id + rejected = not accepted + else: + token_id = target_argmax_id + rejected = draft_token_id != target_argmax_id tl.store( output_token_ids_ptr + req_idx * (max_spec_len + 1) + pos, - target_argmax_id, + token_id, ) - if draft_token_id != target_argmax_id: - # Reject. - rejected = True if not rejected: # If all tokens are accepted, append the bonus token. @@ -707,7 +758,9 @@ def rejection_random_sample_kernel( is_greedy_ptr, # [batch_size] max_spec_len, vocab_size, + synthetic_conditional_rates_ptr, # [num_speculative_tokens] or None NO_DRAFT_PROBS: tl.constexpr, + SYNTHETIC_MODE: tl.constexpr, ): req_idx = tl.program_id(0) is_greedy = tl.load(is_greedy_ptr + req_idx) @@ -723,23 +776,28 @@ def rejection_random_sample_kernel( for pos in range(num_draft_tokens): if not rejected: draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos) - if NO_DRAFT_PROBS: - draft_prob = 1 - else: - draft_prob = tl.load( - draft_probs_ptr + (start_idx + pos) * vocab_size + draft_token_id - ) - target_prob = tl.load( - target_probs_ptr + (start_idx + pos) * vocab_size + draft_token_id - ) uniform_prob = tl.load(uniform_probs_ptr + start_idx + pos) - # NOTE(woosuk): While the draft probability should never be 0, - # we check it to avoid NaNs. If it happens to be 0, we reject. - if draft_prob > 0 and target_prob / draft_prob >= uniform_prob: - # Accept. + if SYNTHETIC_MODE: + rate = tl.load(synthetic_conditional_rates_ptr + pos) + accepted = uniform_prob < rate + else: + if NO_DRAFT_PROBS: + draft_prob = 1 + else: + draft_prob = tl.load( + draft_probs_ptr + + (start_idx + pos) * vocab_size + + draft_token_id + ) + target_prob = tl.load( + target_probs_ptr + (start_idx + pos) * vocab_size + draft_token_id + ) + # NOTE(woosuk): While the draft probability should never be 0, + # we check it to avoid NaNs. If it happens to be 0, we reject. + accepted = draft_prob > 0 and target_prob / draft_prob >= uniform_prob + if accepted: token_id = draft_token_id else: - # Reject. Use recovered token. rejected = True token_id = tl.load(recovered_token_ids_ptr + start_idx + pos) tl.store( diff --git a/vllm/v1/spec_decode/utils.py b/vllm/v1/spec_decode/utils.py index cdcb3e05bfa..e046f013615 100644 --- a/vllm/v1/spec_decode/utils.py +++ b/vllm/v1/spec_decode/utils.py @@ -594,3 +594,9 @@ def update_num_computed_tokens_for_batch_change( num_accepted_tokens.copy_( torch.where(participating, valid_counts, num_accepted_tokens) ) + + +def unconditional_to_conditional_rates(rates: list[float]) -> list[float]: + """Convert per-position unconditional rates to per-position conditional + rates for the early-terminating rejection loop (c_i = p_i / p_{i-1}).""" + return [p / q if q > 0.0 else 0.0 for p, q in zip(rates, [1.0, *rates[:-1]])] diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 820704ecff3..b1bf56ec16b 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -220,6 +220,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.rejection_sampler = RejectionSampler( self.sampler, self.speculative_config, + self.device, ) self.prompt_logprobs_worker = PromptLogprobsWorker(self.max_num_reqs) self.structured_outputs_worker = StructuredOutputsWorker( diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py index 2f92b0c093d..6be0b26ac3f 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py @@ -5,6 +5,7 @@ import torch from vllm.config import SpeculativeConfig from vllm.triton_utils import tl, triton from vllm.v1.outputs import LogprobsTensors +from vllm.v1.spec_decode.utils import unconditional_to_conditional_rates from vllm.v1.worker.gpu.input_batch import InputBatch from vllm.v1.worker.gpu.metrics.logits import get_num_nans from vllm.v1.worker.gpu.sample.logprob import compute_topk_logprobs @@ -15,7 +16,6 @@ from vllm.v1.worker.gpu.spec_decode.probabilistic_rejection_sampler_utils import probabilistic_rejection_sample, ) from vllm.v1.worker.gpu.spec_decode.synthetic_rejection_sampler_utils import ( - compute_synthetic_rejection_sampler_params, synthetic_rejection_sample, ) @@ -102,24 +102,20 @@ class RejectionSampler: self, sampler: Sampler, spec_config: SpeculativeConfig, + device: torch.device, ): self.sampler = sampler self.num_speculative_steps = spec_config.num_speculative_tokens self.rejection_sample_method = spec_config.rejection_sample_method + self.synthetic_conditional_rates: torch.Tensor | None = None if self.rejection_sample_method == "synthetic": - synthetic_acceptance_rate = spec_config.synthetic_acceptance_rate - if ( - synthetic_acceptance_rate is None - or not 0.0 <= synthetic_acceptance_rate <= 1.0 - ): - raise ValueError( - f"synthetic_acceptance_rate must be in [0, 1], " - f"but got {synthetic_acceptance_rate}" - ) - self.base_acceptance_rate, self.decay_factor = ( - compute_synthetic_rejection_sampler_params( - synthetic_acceptance_rate, self.num_speculative_steps - ) + assert spec_config.synthetic_acceptance_rates is not None + self.synthetic_conditional_rates = torch.tensor( + unconditional_to_conditional_rates( + spec_config.synthetic_acceptance_rates + ), + dtype=torch.float32, + device=device, ) def _get_logprobs_tensors( @@ -218,8 +214,7 @@ class RejectionSampler: input_batch.positions[input_batch.logits_indices], input_batch.idx_mapping, self.sampler.sampling_states.seeds.gpu, - self.base_acceptance_rate, - self.decay_factor, + self.synthetic_conditional_rates, self.num_speculative_steps, ) else: diff --git a/vllm/v1/worker/gpu/spec_decode/synthetic_rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/synthetic_rejection_sampler_utils.py index f5388575bae..7e91075bb1a 100644 --- a/vllm/v1/worker/gpu/spec_decode/synthetic_rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/synthetic_rejection_sampler_utils.py @@ -5,8 +5,6 @@ import torch from vllm.triton_utils import tl, triton from vllm.v1.worker.gpu.sample.gumbel import tl_rand64 -MIN_ACCEPTANCE_DECAY_FACTOR = 0.85 - @triton.jit def _synthetic_rejection_sample_kernel( @@ -27,8 +25,8 @@ def _synthetic_rejection_sample_kernel( idx_mapping_ptr, # [max_num_reqs] seeds_ptr, - base_acceptance_rate, - decay_factor, + # [num_speculative_steps] + acceptance_rates_ptr, ): req_idx = tl.program_id(0) start_idx = tl.load(cu_num_logits_ptr + req_idx) @@ -38,13 +36,13 @@ def _synthetic_rejection_sample_kernel( seed = tl.load(seeds_ptr + req_state_idx) num_sampled = 0 - acceptance_rate = base_acceptance_rate rejected = False for i in range(num_tokens - 1): if not rejected: logit_idx = start_idx + i pos = tl.load(pos_ptr + logit_idx) u = tl_rand64(seed, pos, includes_zero=False) + acceptance_rate = tl.load(acceptance_rates_ptr + i) if u < acceptance_rate: sampled = tl.load(input_ids_ptr + logit_idx + 1).to(tl.int64) else: @@ -52,7 +50,6 @@ def _synthetic_rejection_sample_kernel( rejected = True tl.store(sampled_ptr + req_idx * sampled_stride + i, sampled) num_sampled += 1 - acceptance_rate *= decay_factor if not rejected: target_sampled = tl.load(target_sampled_ptr + start_idx + num_tokens - 1) tl.store( @@ -75,8 +72,8 @@ def synthetic_rejection_sample( idx_mapping: torch.Tensor, # [max_num_reqs] seed: torch.Tensor, - base_acceptance_rate: float, - decay_factor: float, + # [num_speculative_steps] + acceptance_rates: torch.Tensor, num_speculative_steps: int, ) -> tuple[torch.Tensor, torch.Tensor]: num_reqs = cu_num_logits.shape[0] - 1 @@ -92,56 +89,7 @@ def synthetic_rejection_sample( pos, idx_mapping, seed, - base_acceptance_rate, - decay_factor, + acceptance_rates, num_warps=1, ) return sampled, num_sampled - - -def compute_synthetic_rejection_sampler_params( - p_avg: float, n: int, tol: float = 1e-9 -) -> tuple[float, float]: - def mean_joint_prob(a_0: float, gamma: float, n: int): - total = 0.0 - for i in range(n): - total += a_0 ** (i + 1) * gamma ** (i * (i + 1) // 2) - return total / n - - def min_valid_decay_factor(p: float, n: int, tol: float = 1e-9) -> float: - low, high = MIN_ACCEPTANCE_DECAY_FACTOR, 1.0 - if mean_joint_prob(1, low, n) >= p: - return low - - # Sweep for a gamma decay factor that is guaranteed - # to yield a base acceptance rate <= 1. - while (high - low) > tol: - mid = (low + high) / 2 - if mean_joint_prob(1, mid, n) >= p: - high = mid - else: - low = mid - return high - - def compute_base_acceptance_rate( - p_avg: float, gamma: float, n: int, tol: float = 1e-9 - ) -> float: - if p_avg <= 0.0: - return 0.0 - if p_avg >= 1.0: - return 1.0 - - # Sweep for a base acceptance rate that yields - # the desired mean joint probability. - low, high = 0.0, 1.0 - while (high - low) > tol: - mid = (low + high) / 2 - if mean_joint_prob(mid, gamma, n) >= p_avg: - high = mid - else: - low = mid - return high - - decay_factor = min_valid_decay_factor(p_avg, n) - base_rate = compute_base_acceptance_rate(p_avg, decay_factor, n) - return base_rate, decay_factor diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 8aca4594137..0b0fed4824a 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -577,7 +577,9 @@ class GPUModelRunner( "Unknown speculative decoding method: " f"{self.speculative_config.method}" ) - self.rejection_sampler = RejectionSampler(self.sampler) + self.rejection_sampler = RejectionSampler( + self.sampler, self.speculative_config, self.device + ) self.num_spec_tokens = 0 self.valid_sampled_token_count_gpu: torch.Tensor | None = None From 92762edc535c696c3b8a5f3ffee9bc1c0fac10e6 Mon Sep 17 00:00:00 2001 From: Doug Campos Date: Thu, 23 Apr 2026 21:10:04 -0400 Subject: [PATCH 087/153] [Bugfix] Treat as implicit reasoning end in Qwen3 parser (#35687) Signed-off-by: Doug Campos --- .../reasoning/test_qwen3_reasoning_parser.py | 52 +++++++++ vllm/reasoning/qwen3_reasoning_parser.py | 108 ++++++++++++++++-- 2 files changed, 148 insertions(+), 12 deletions(-) diff --git a/tests/reasoning/test_qwen3_reasoning_parser.py b/tests/reasoning/test_qwen3_reasoning_parser.py index 411c7ba485a..f42458560f9 100644 --- a/tests/reasoning/test_qwen3_reasoning_parser.py +++ b/tests/reasoning/test_qwen3_reasoning_parser.py @@ -78,6 +78,25 @@ WITHOUT_THINK_STREAM = { "content": None, } +# --- without (implicit reasoning end) --- + +TOOL_CALL_BODY = ( + "\n\n" + "\ncat /etc/hosts\n\n\n" +) + +TOOL_CALL_NO_THINK_END = { + "output": "I need to read the file.\n\n" + TOOL_CALL_BODY, + "reasoning": "I need to read the file.\n\n", + "content": TOOL_CALL_BODY, +} + +TOOL_CALL_WITH_THINK_NO_END = { + "output": "I need to read the file.\n\n" + TOOL_CALL_BODY, + "reasoning": "I need to read the file.\n\n", + "content": TOOL_CALL_BODY, +} + # --- Edge cases --- COMPLETE_REASONING = { @@ -199,6 +218,26 @@ TEST_CASES = [ TRUNCATED_NO_START_TOKEN_STREAM, id="truncated_no_start_token_stream", ), + pytest.param( + False, + TOOL_CALL_NO_THINK_END, + id="tool_call_no_think_end", + ), + pytest.param( + True, + TOOL_CALL_NO_THINK_END, + id="tool_call_no_think_end_stream", + ), + pytest.param( + False, + TOOL_CALL_WITH_THINK_NO_END, + id="tool_call_with_think_no_end", + ), + pytest.param( + True, + TOOL_CALL_WITH_THINK_NO_END, + id="tool_call_with_think_no_end_stream", + ), ] @@ -255,6 +294,13 @@ MULTI_TOKEN_DELTA_CASES = [ "content", id="no_start_end_grouped_with_content", ), + pytest.param( + # arrives in a separate delta after reasoning text + ["I need to read the file.\n\n", "\n"], + "I need to read the file.\n\n", + "\n", + id="tool_call_implicit_reasoning_end", + ), ] @@ -296,6 +342,12 @@ THINKING_DISABLED_CASES = [ "Some output without think tokens", id="thinking_disabled_no_think_tokens", ), + pytest.param( + "I need to read the file.\n\n" + TOOL_CALL_BODY, + None, + "I need to read the file.\n\n" + TOOL_CALL_BODY, + id="thinking_disabled_with_tool_call", + ), ] diff --git a/vllm/reasoning/qwen3_reasoning_parser.py b/vllm/reasoning/qwen3_reasoning_parser.py index 9a54aa75951..e38b0de3d82 100644 --- a/vllm/reasoning/qwen3_reasoning_parser.py +++ b/vllm/reasoning/qwen3_reasoning_parser.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from typing import TYPE_CHECKING from vllm.entrypoints.openai.engine.protocol import DeltaMessage @@ -31,6 +31,10 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): use an older chat template where the model generates itself. This parser handles both styles: if appears in the generated output it is stripped before extraction (non-streaming) or skipped (streaming). + + NOTE: Qwen3.5 models may emit inside the thinking block + without closing first. is treated as an implicit + end of reasoning, matching the approach in KimiK2ReasoningParser. """ def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs): @@ -41,6 +45,11 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): # pure content when the user explicitly disables it. self.thinking_enabled = chat_kwargs.get("enable_thinking", True) + self._tool_call_tag = "" + self._tool_call_token_id = self.vocab.get(self._tool_call_tag) + self._tool_call_end_tag = "" + self._tool_call_end_token_id = self.vocab.get(self._tool_call_end_tag) + @property def start_token(self) -> str: """The token that starts reasoning content.""" @@ -51,6 +60,58 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): """The token that ends reasoning content.""" return "" + def is_reasoning_end(self, input_ids: Sequence[int]) -> bool: + start_token_id = self.start_token_id + end_token_id = self.end_token_id + tool_call_token_id = self._tool_call_token_id + tool_call_end_token_id = self._tool_call_end_token_id + + for i in range(len(input_ids) - 1, -1, -1): + token_id = input_ids[i] + if token_id == start_token_id: + # Found before or + return False + if token_id == end_token_id: + return True + if tool_call_token_id is not None and token_id == tool_call_token_id: + # Only treat as implicit reasoning end if this + # is NOT followed by . Paired occurrences are + # template examples in the prompt, not model output. + if tool_call_end_token_id is not None and any( + input_ids[j] == tool_call_end_token_id + for j in range(i + 1, len(input_ids)) + ): + continue + return True + return False + + def is_reasoning_end_streaming( + self, input_ids: Sequence[int], delta_ids: Iterable[int] + ) -> bool: + if super().is_reasoning_end_streaming(input_ids, delta_ids): + return True + if self._tool_call_token_id is not None: + return self._tool_call_token_id in delta_ids + return False + + def extract_content_ids(self, input_ids: list[int]) -> list[int]: + """ + Extract content token ids from the input_ids. + """ + result = super().extract_content_ids(input_ids) + if result: + return result + # Fall back: content starts at (implicit reasoning end). + if ( + self._tool_call_token_id is not None + and self._tool_call_token_id in input_ids + ): + tool_call_index = ( + len(input_ids) - 1 - input_ids[::-1].index(self._tool_call_token_id) + ) + return input_ids[tool_call_index:] + return [] + def extract_reasoning( self, model_output: str, request: "ChatCompletionRequest | ResponsesRequest" ) -> tuple[str | None, str | None]: @@ -78,19 +139,23 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): model_output_parts[2] if model_output_parts[1] else model_output_parts[0] ) - if self.end_token not in model_output: - if not self.thinking_enabled: - # Thinking explicitly disabled — treat everything as content. - return None, model_output - # Thinking enabled but no : output was truncated. - # Everything generated so far is reasoning. - return model_output, None + if self.end_token in model_output: + reasoning, _, content = model_output.partition(self.end_token) + return reasoning, content or None - # Extract reasoning content from the model output. - reasoning, _, content = model_output.partition(self.end_token) + if not self.thinking_enabled: + # Thinking explicitly disabled — treat everything as content. + return None, model_output - final_content = content or None - return reasoning, final_content + # No — check for implicit reasoning end via . + tool_call_index = model_output.find(self._tool_call_tag) + if tool_call_index != -1: + reasoning = model_output[:tool_call_index] + content = model_output[tool_call_index:] + return reasoning or None, content or None + # Thinking enabled but no : output was truncated. + # Everything generated so far is reasoning. + return model_output, None def extract_reasoning_streaming( self, @@ -135,6 +200,20 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): # end_token_id in IDs but not in text (already stripped) return None + # Implicit reasoning end via . + if ( + self._tool_call_token_id is not None + and self._tool_call_token_id in delta_token_ids + ): + tool_index = delta_text.find(self._tool_call_tag) + if tool_index >= 0: + reasoning = delta_text[:tool_index] + content = delta_text[tool_index:] + return DeltaMessage( + reasoning=reasoning if reasoning else None, + content=content if content else None, + ) + # No end token in this delta. if not delta_text: # Nothing left after stripping start token. @@ -142,6 +221,11 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): elif self.end_token_id in previous_token_ids: # End token already passed: everything is content now. return DeltaMessage(content=delta_text) + elif ( + self._tool_call_token_id is not None + and self._tool_call_token_id in previous_token_ids + ): + return DeltaMessage(content=delta_text) else: # No end token yet: still in reasoning phase. return DeltaMessage(reasoning=delta_text) From 30413442871490b8a0e757f590af7b1a5c4229a7 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Thu, 23 Apr 2026 21:19:30 -0400 Subject: [PATCH 088/153] [Misc] Added curl retries in install_python_libraries.sh (#36700) Signed-off-by: Dmitry Tokarev Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- tools/ep_kernels/install_python_libraries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ep_kernels/install_python_libraries.sh b/tools/ep_kernels/install_python_libraries.sh index c3deb7d6060..f61aa868581 100755 --- a/tools/ep_kernels/install_python_libraries.sh +++ b/tools/ep_kernels/install_python_libraries.sh @@ -101,7 +101,7 @@ NVSHMEM_URL="https://developer.download.nvidia.com/compute/nvshmem/redist/libnvs pushd "$WORKSPACE" echo "Downloading NVSHMEM ${NVSHMEM_VER} for ${NVSHMEM_SUBDIR} ..." -curl -fSL "${NVSHMEM_URL}" -o "${NVSHMEM_FILE}" +curl -fSL --retry 3 --retry-delay 2 "${NVSHMEM_URL}" -o "${NVSHMEM_FILE}" tar -xf "${NVSHMEM_FILE}" rm -rf nvshmem mv "${NVSHMEM_FILE%.tar.xz}" nvshmem From c9bf77df92a3526575756f576127268474068e96 Mon Sep 17 00:00:00 2001 From: yzong-rh Date: Thu, 23 Apr 2026 21:20:30 -0400 Subject: [PATCH 089/153] [BUG]: fix HF tokenizer concurrent borrow in tool parsers (#40059) Signed-off-by: Yifan Co-authored-by: timon0305 Co-authored-by: sfeng33 <4florafeng@gmail.com> --- .../test_llama3_json_tool_parser.py | 13 ++++-- .../tool_parsers/functiongemma_tool_parser.py | 42 +++++++------------ vllm/tool_parsers/llama_tool_parser.py | 20 +++++---- 3 files changed, 37 insertions(+), 38 deletions(-) diff --git a/tests/tool_parsers/test_llama3_json_tool_parser.py b/tests/tool_parsers/test_llama3_json_tool_parser.py index 53948d577c1..7040fe87d07 100644 --- a/tests/tool_parsers/test_llama3_json_tool_parser.py +++ b/tests/tool_parsers/test_llama3_json_tool_parser.py @@ -4,15 +4,22 @@ from unittest.mock import MagicMock, patch import pytest +from transformers import AutoTokenizer from vllm.entrypoints.openai.engine.protocol import ExtractedToolCallInformation -from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser +LLAMA_MODEL = "meta-llama/Llama-3.2-1B-Instruct" + + +@pytest.fixture(scope="module") +def llama_tokenizer(): + return AutoTokenizer.from_pretrained(LLAMA_MODEL) + @pytest.fixture -def parser(default_tokenizer: TokenizerLike): - return Llama3JsonToolParser(default_tokenizer) +def parser(llama_tokenizer): + return Llama3JsonToolParser(llama_tokenizer) def test_extract_tool_calls_simple(parser): diff --git a/vllm/tool_parsers/functiongemma_tool_parser.py b/vllm/tool_parsers/functiongemma_tool_parser.py index 35c4c6b84fe..776792ea1d6 100644 --- a/vllm/tool_parsers/functiongemma_tool_parser.py +++ b/vllm/tool_parsers/functiongemma_tool_parser.py @@ -34,6 +34,21 @@ class FunctionGemmaToolParser(ToolParser): call:func_name{param:value} """ + # FunctionGemma tokens + tool_call_start_token: str = "" + tool_call_end_token: str = "" + + # Regex patterns + tool_call_regex: re.Pattern = re.compile( + r"call:(\w+)\{(.*?)\}" + r"|call:(\w+)\{(.*)", + re.DOTALL, + ) + arg_regex: re.Pattern = re.compile( + r"(\w+):(.*?)", + re.DOTALL, + ) + def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): super().__init__(tokenizer, tools) @@ -42,33 +57,6 @@ class FunctionGemmaToolParser(ToolParser): self.prev_tool_call_arr: list[dict] = [] self.current_tool_id: int = -1 self.streamed_args_for_tool: list[str] = [] - - # FunctionGemma tokens - self.tool_call_start_token: str = "" - self.tool_call_end_token: str = "" - - # Regex patterns - self.tool_call_regex = re.compile( - r"call:(\w+)\{(.*?)\}" - r"|call:(\w+)\{(.*)", - re.DOTALL, - ) - self.arg_regex = re.compile( - r"(\w+):(.*?)", - re.DOTALL, - ) - - if self.model_tokenizer: - self.tool_call_start_token_ids = self.model_tokenizer.encode( - self.tool_call_start_token, add_special_tokens=False - ) - self.tool_call_end_token_ids = self.model_tokenizer.encode( - self.tool_call_end_token, add_special_tokens=False - ) - else: - self.tool_call_start_token_ids = [] - self.tool_call_end_token_ids = [] - self.buffered_delta_text = "" def _parse_arguments(self, args_str: str) -> dict: diff --git a/vllm/tool_parsers/llama_tool_parser.py b/vllm/tool_parsers/llama_tool_parser.py index be3d47acd97..4a041041f09 100644 --- a/vllm/tool_parsers/llama_tool_parser.py +++ b/vllm/tool_parsers/llama_tool_parser.py @@ -45,6 +45,12 @@ class Llama3JsonToolParser(ToolParser): llama4_json are set. """ + bot_token: str = "<|python_tag|>" + # Simple regex to find opening braces - we'll use JSON decoder for parsing + # This handles arbitrary nesting depth correctly + tool_call_start_regex: re.Pattern = re.compile(r"\{") + json_decoder: json.JSONDecoder = json.JSONDecoder() + def __init__( self, tokenizer: PreTrainedTokenizerBase, @@ -60,14 +66,12 @@ class Llama3JsonToolParser(ToolParser): self.streamed_args_for_tool: list[ str ] = [] # map what has been streamed for each tool so far to a list - self.bot_token = "<|python_tag|>" - self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[ - 0 - ] - # Simple regex to find opening braces - we'll use JSON decoder for parsing - # This handles arbitrary nesting depth correctly - self.tool_call_start_regex = re.compile(r"\{") - self.json_decoder = json.JSONDecoder() + self.bot_token_id = self.vocab.get(self.bot_token) + if self.bot_token_id is None: + raise RuntimeError( + "Llama3JsonToolParser could not locate the bot token " + f"'{self.bot_token}' in the tokenizer." + ) def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest From e9f331d72e90f34614363101528afe6c6fcdf7c5 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 23 Apr 2026 18:33:26 -0700 Subject: [PATCH 090/153] [MRV2] Ensure warmup covers prefill path (#40746) Signed-off-by: Nick Hill --- vllm/v1/worker/gpu/warmup.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index 026b6a7d7eb..83d87c74a4a 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -29,13 +29,16 @@ def warmup_kernels( triton kernels. We must call the provided worker's execute_model for pipeline parallel coordination. - The first iteration simulates a prefill with requests of 2 prompt - tokens each. The second iteration simulates a decode step with all - requests generating 1 token each. + The first iteration simulates a prefill with requests of + 2 + num_spec_steps prompt tokens each. The second iteration simulates + a decode step with all requests generating 1 + num_spec_steps tokens. """ - prompt_token_ids = [0, 1] - prompt_len = len(prompt_token_ids) num_spec_steps = model_runner.num_speculative_steps + # Use 1 + num_spec_steps + 1 tokens so the prefill batch's per-request + # query length exceeds decode_query_len (= 1 + num_spec_steps), preventing + # it from being misclassified as a uniform decode batch. + prompt_len = 2 + num_spec_steps + prompt_token_ids = list(range(prompt_len)) # After prefill, decode generates 1 verified + num_spec_steps draft tokens. decode_len = prompt_len + 1 + num_spec_steps @@ -76,7 +79,7 @@ def warmup_kernels( nonlocal next_block_id return list(range(next_block_id, next_block_id := next_block_id + num_blocks)) - # Step 1: Prefill all requests with 2 prompt tokens each. + # Step 1: Prefill all requests with 2 + num_spec_steps prompt tokens each. new_reqs = [ NewRequestData.from_request( Request(req_ids[i], prompt_token_ids, sampling_params, pooling_params), From eba73068ea861c2a76753ab82218b08176fce765 Mon Sep 17 00:00:00 2001 From: Vinayak Kumar <46377914+VinayakMishra95@users.noreply.github.com> Date: Fri, 24 Apr 2026 07:53:54 +0530 Subject: [PATCH 091/153] [Doc] fix capitalization consistency in README (vLLM, Hugging Face) (#40729) Signed-off-by: Vinayak Mishra --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d94e33ba8b0..42777436c63 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Easy, fast, and cheap LLM serving for everyone | Documentation | Blog | Paper | Twitter/X | User Forum | Developer Slack |

-🔥 We have built a vllm website to help you get started with vllm. Please visit [vllm.ai](https://vllm.ai) to learn more. +🔥 We have built a vLLM website to help you get started with vLLM. Please visit [vllm.ai](https://vllm.ai) to learn more. For events, please visit [vllm.ai/events](https://vllm.ai/events) to join us. --- @@ -50,7 +50,7 @@ vLLM is flexible and easy to use with: - Efficient multi-LoRA support for dense and MoE layers - Support for NVIDIA GPUs, AMD GPUs, and x86/ARM/PowerPC CPUs. Additionally, diverse hardware plugins such as Google TPUs, Intel Gaudi, IBM Spyre, Huawei Ascend, Rebellions NPU, Apple Silicon, MetaX GPU, and more. -vLLM seamlessly supports 200+ model architectures on HuggingFace, including: +vLLM seamlessly supports 200+ model architectures on Hugging Face, including: - Decoder-only LLMs (e.g., Llama, Qwen, Gemma) - Mixture-of-Expert LLMs (e.g., Mixtral, DeepSeek-V3, Qwen-MoE, GPT-OSS) From 56bdf85e10b807be13225f659f2593051306c77d Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 23 Apr 2026 19:49:16 -0700 Subject: [PATCH 092/153] [Feature] Avoid eager import of the "mistral_common" package. (#40043) Signed-off-by: Neil Schemenauer --- .../openai/chat_completion/serving.py | 43 +++++++++++-------- vllm/entrypoints/openai/engine/serving.py | 5 +-- vllm/entrypoints/serve/render/serving.py | 5 +-- vllm/tool_parsers/mistral_tool_parser.py | 2 + vllm/utils/mistral.py | 15 +++++++ 5 files changed, 47 insertions(+), 23 deletions(-) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index fd8a5a66029..12dc2cd98e2 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -73,13 +73,9 @@ from vllm.reasoning import ReasoningParser from vllm.renderers import ChatParams from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike -from vllm.tool_parsers.mistral_tool_parser import ( - MistralToolCall, - MistralToolParser, -) from vllm.tool_parsers.utils import partial_json_loads from vllm.utils.collection_utils import as_list -from vllm.utils.mistral import is_mistral_tokenizer +from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser if TYPE_CHECKING: from vllm.entrypoints.serve.render.serving import OpenAIServingRender @@ -143,10 +139,12 @@ class OpenAIServingChat(OpenAIServing): enable_auto_tools=enable_auto_tools, model_name=self.model_config.model, ) - _is_mistral_tool_parser = self.tool_parser is not None and issubclass( - self.tool_parser, MistralToolParser - ) - if _is_mistral_tool_parser and self.reasoning_parser_cls is not None: + if ( + is_mistral_tool_parser(self.tool_parser) + and self.reasoning_parser_cls is not None + ): + from vllm.tool_parsers.mistral_tool_parser import MistralToolParser + MistralToolParser.model_can_reason = True self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none @@ -823,6 +821,10 @@ class OpenAIServingChat(OpenAIServing): harmony_tools_streamed[i] |= tools_streamed_flag # Mistral grammar path: combined reasoning + tool streaming elif is_mistral_grammar_path: + from vllm.tool_parsers.mistral_tool_parser import ( + MistralToolParser, + ) + assert tool_parser is not None assert isinstance(tool_parser, MistralToolParser) assert reasoning_end_arr is not None @@ -904,6 +906,10 @@ class OpenAIServingChat(OpenAIServing): else: # Generate ID based on tokenizer type if is_mistral_tokenizer(tokenizer): + from vllm.tool_parsers.mistral_tool_parser import ( + MistralToolCall, + ) + tool_call_id = MistralToolCall.generate_random_id() else: tool_call_id = make_tool_call_id( @@ -1275,8 +1281,6 @@ class OpenAIServingChat(OpenAIServing): request_metadata: RequestResponseMetadata, reasoning_parser: ReasoningParser | None = None, ) -> ErrorResponse | ChatCompletionResponse: - from vllm.tokenizers.mistral import MistralTokenizer - created_time = int(time.time()) final_res: RequestOutput | None = None @@ -1393,12 +1397,17 @@ class OpenAIServingChat(OpenAIServing): enable_auto_tools=self.enable_auto_tools, tool_parser_cls=self.tool_parser, ) - tool_call_class = ( - MistralToolCall if is_mistral_tokenizer(tokenizer) else ToolCall - ) + if is_mistral_tokenizer(tokenizer): + from vllm.tool_parsers.mistral_tool_parser import MistralToolCall + + tool_call_class: type[ToolCall] = MistralToolCall + else: + tool_call_class = ToolCall use_mistral_tool_parser = request._grammar_from_tool_parser if use_mistral_tool_parser: + from vllm.tool_parsers.mistral_tool_parser import MistralToolParser + tool_call_items = MistralToolParser.build_non_streaming_tool_calls( tool_calls ) @@ -1436,7 +1445,7 @@ class OpenAIServingChat(OpenAIServing): # Generate ID using the correct format (kimi_k2 or random), # but leave it to the class if it's Mistral to preserve # 9-char IDs - if isinstance(tokenizer, MistralTokenizer): + if is_mistral_tokenizer(tokenizer): tool_call_class_items.append(tool_call_class(function=tc)) else: generated_id = make_tool_call_id( @@ -1469,7 +1478,7 @@ class OpenAIServingChat(OpenAIServing): # Generate ID using the correct format (kimi_k2 or random), # but leave it to the class if it's Mistral to preserve # 9-char IDs - if isinstance(tokenizer, MistralTokenizer): + if is_mistral_tokenizer(tokenizer): tool_call_class_items.append( tool_call_class(function=tool_call) ) @@ -1519,7 +1528,7 @@ class OpenAIServingChat(OpenAIServing): # Generate ID using the correct format (kimi_k2 or random), # but leave it to the class if it's Mistral to preserve # 9-char IDs - if isinstance(tokenizer, MistralTokenizer): + if is_mistral_tokenizer(tokenizer): tool_call_items.append(tool_call_class(function=tc)) else: generated_id = make_tool_call_id( diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index ab33008aeeb..d5fb5e137ef 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -65,7 +65,6 @@ from vllm.renderers.inputs.preprocess import ( from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.tool_parsers import ToolParser -from vllm.tool_parsers.mistral_tool_parser import MistralToolParser from vllm.tracing import ( contains_trace_headers, extract_trace_headers, @@ -73,6 +72,7 @@ from vllm.tracing import ( ) from vllm.utils import random_uuid from vllm.utils.async_utils import collect_from_async_generator +from vllm.utils.mistral import is_mistral_tool_parser logger = init_logger(__name__) @@ -615,8 +615,7 @@ class OpenAIServing: # let the parser handle the output. use_mistral_tool_parser = ( isinstance(request, ChatCompletionRequest) - and tool_parser_cls is not None - and issubclass(tool_parser_cls, MistralToolParser) + and is_mistral_tool_parser(tool_parser_cls) and request._grammar_from_tool_parser ) diff --git a/vllm/entrypoints/serve/render/serving.py b/vllm/entrypoints/serve/render/serving.py index 25c5a6d199e..99966e590bd 100644 --- a/vllm/entrypoints/serve/render/serving.py +++ b/vllm/entrypoints/serve/render/serving.py @@ -55,9 +55,8 @@ from vllm.renderers.inputs.preprocess import ( prompt_to_seq, ) from vllm.tool_parsers import ToolParser -from vllm.tool_parsers.mistral_tool_parser import MistralToolParser from vllm.utils import random_uuid -from vllm.utils.mistral import is_mistral_tokenizer +from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser from vllm.utils.mistral import mt as _mt logger = init_logger(__name__) @@ -582,7 +581,7 @@ class OpenAIServingRender: tool_choice = getattr(request, "tool_choice", "none") tokenizer = renderer.get_tokenizer() is_mistral_grammar_eligible = ( - issubclass(tool_parser, MistralToolParser) + is_mistral_tool_parser(tool_parser) and is_mistral_tokenizer(tokenizer) and tokenizer.supports_grammar ) diff --git a/vllm/tool_parsers/mistral_tool_parser.py b/vllm/tool_parsers/mistral_tool_parser.py index 5170f6eb097..54c61d89bdf 100644 --- a/vllm/tool_parsers/mistral_tool_parser.py +++ b/vllm/tool_parsers/mistral_tool_parser.py @@ -118,6 +118,8 @@ class MistralToolParser(ToolParser): set. """ + IS_MISTRAL_TOOL_PARSER = True # used by vllm.utils.mistral + # Used to generate correct grammar in `adjust_request` model_can_reason: bool = False diff --git a/vllm/utils/mistral.py b/vllm/utils/mistral.py index c9c24a2e306..276ca8170f1 100644 --- a/vllm/utils/mistral.py +++ b/vllm/utils/mistral.py @@ -12,8 +12,10 @@ from vllm.utils.import_utils import LazyLoader if TYPE_CHECKING: # if type checking, eagerly import the module import vllm.tokenizers.mistral as mt + import vllm.tool_parsers.mistral_tool_parser as mtp else: mt = LazyLoader("mt", globals(), "vllm.tokenizers.mistral") + mtp = LazyLoader("mtp", globals(), "vllm.tool_parsers.mistral_tool_parser") def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralTokenizer]: @@ -26,3 +28,16 @@ def is_mistral_tokenizer(obj: TokenizerLike | None) -> TypeGuard[mt.MistralToken getattr(cls, "IS_MISTRAL_TOKENIZER", False) and isinstance(obj, mt.MistralTokenizer) ) + + +def is_mistral_tool_parser(cls: type | None) -> bool: + """Return true if *cls* is (a subclass of) MistralToolParser. + + Uses a class attribute check so that importing + ``vllm.tool_parsers.mistral_tool_parser`` — and transitively + ``mistral_common`` — is not required. + """ + return bool( + getattr(cls, "IS_MISTRAL_TOOL_PARSER", False) + and issubclass(cls, mtp.MistralToolParser) # type: ignore[arg-type] + ) From 100c7b65e7579c8caf4ee0b04a6410b2796b905c Mon Sep 17 00:00:00 2001 From: lyd1992 <105697319+lyd1992@users.noreply.github.com> Date: Fri, 24 Apr 2026 12:33:05 +0800 Subject: [PATCH 093/153] [Platform] Fix RISC-V platform detection (lscpu parsing + non-NUMA meminfo) (#40427) Signed-off-by: liuyudong --- setup.py | 4 +++- vllm/platforms/__init__.py | 1 - vllm/utils/cpu_resource_utils.py | 40 +++++++++++++++++++++++++++----- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index c05280e40e7..7c226a72425 100644 --- a/setup.py +++ b/setup.py @@ -927,7 +927,9 @@ def get_vllm_version() -> str: elif _is_tpu(): version += f"{sep}tpu" elif _is_cpu(): - if envs.VLLM_TARGET_DEVICE == "cpu": + # Check the local VLLM_TARGET_DEVICE (may be set by auto-detect above), + # not envs.VLLM_TARGET_DEVICE, so CPU-only hosts still get `+cpu`. + if VLLM_TARGET_DEVICE == "cpu": version += f"{sep}cpu" elif _is_xpu(): version += f"{sep}xpu" diff --git a/vllm/platforms/__init__.py b/vllm/platforms/__init__.py index af344acfcbc..645da0a1fe9 100644 --- a/vllm/platforms/__init__.py +++ b/vllm/platforms/__init__.py @@ -177,7 +177,6 @@ def cpu_platform_plugin() -> str | None: logger.debug( "Confirmed CPU platform is available because the machine is MacOS." ) - except Exception as e: logger.debug("CPU platform is not available because: %s", str(e)) diff --git a/vllm/utils/cpu_resource_utils.py b/vllm/utils/cpu_resource_utils.py index 4a56e7f6433..bbf554d0ccd 100644 --- a/vllm/utils/cpu_resource_utils.py +++ b/vllm/utils/cpu_resource_utils.py @@ -87,7 +87,13 @@ def get_memory_node_info(node_id: int = 0) -> MemoryNodeInfo: meminfo_path = f"/sys/devices/system/node/node{node_id}/meminfo" if not os.path.exists(meminfo_path): - raise RuntimeError(f"{meminfo_path} doesn't exit.") + # Non-NUMA systems (e.g. many RISC-V boards) don't expose per-node + # meminfo. Fall back to system-wide numbers from psutil. + vm = psutil.virtual_memory() + return MemoryNodeInfo( + total_memory=vm.total, + available_memory=vm.available, + ) meminfo = {} with open(meminfo_path) as f: @@ -147,19 +153,36 @@ def get_visible_memory_node() -> list[int]: @cache +def _synthesize_cpu_list() -> list[LogicalCPUInfo]: + """Synthesize a flat CPU list: each logical CPU is its own core on + NUMA node 0. Used when lscpu output is unavailable or unparsable + (e.g. macOS, RISC-V).""" + cpu_count = os.cpu_count() + assert cpu_count + return [LogicalCPUInfo(i, i, 0) for i in range(cpu_count)] + + def _get_cpu_list() -> list[LogicalCPUInfo]: if platform.system() == "Darwin": # For MacOS, no user-level CPU affinity and SMT, return all CPUs - cpu_count = os.cpu_count() - assert cpu_count - return [LogicalCPUInfo(i, i, 0) for i in range(cpu_count)] + return _synthesize_cpu_list() lscpu_output = subprocess.check_output( "lscpu --json --extended=CPU,CORE,NODE --online", shell=True, text=True ) - # For platform without NUMA, replace '-' to '0' - lscpu_output = re.sub(r'"node":\s*-\s*(,|\n)', r'"node": 0\1', lscpu_output) + # For platforms without NUMA, map bare `-` node to 0 so non-NUMA + # systems keep the existing behavior from #39781. + lscpu_output = re.sub(r'"node":\s*-\s*(,|\n|\})', r'"node": 0\1', lscpu_output) + + # On some architectures (notably RISC-V), lscpu also emits bare `-` + # for cpu/core. Quote them so the JSON parses; they will decode to + # -1 and be filtered out below, triggering the synthesized fallback. + lscpu_output = re.sub( + r'("(?:cpu|core)":\s*)-\s*(,|\n|\})', + r'\1"-"\2', + lscpu_output, + ) logical_cpu_list: list[LogicalCPUInfo] = json.loads( lscpu_output, object_hook=LogicalCPUInfo.json_decoder @@ -170,4 +193,9 @@ def _get_cpu_list() -> list[LogicalCPUInfo]: x for x in logical_cpu_list if -1 not in (x.id, x.physical_core, x.numa_node) ] + # If lscpu returned no valid entries (e.g. RISC-V where all fields + # are bare `-`), fall back to synthesized topology. + if not logical_cpu_list: + logical_cpu_list = _synthesize_cpu_list() + return logical_cpu_list From c662b4359e307f422e507230d2e20e2303610b6a Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 24 Apr 2026 13:08:58 +0800 Subject: [PATCH 094/153] [Bugfix] Avoid mutating `chat_template_kwargs` in `HYV3ReasoningParser` initialization (#40713) Signed-off-by: Bugen Zhao Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../reasoning/test_hy_v3_reasoning_parser.py | 31 +++++++++++++++++++ vllm/reasoning/hy_v3_reasoning_parser.py | 8 +++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/tests/reasoning/test_hy_v3_reasoning_parser.py b/tests/reasoning/test_hy_v3_reasoning_parser.py index 4c1858cc99b..e527c979f6e 100644 --- a/tests/reasoning/test_hy_v3_reasoning_parser.py +++ b/tests/reasoning/test_hy_v3_reasoning_parser.py @@ -4,6 +4,7 @@ import pytest from tests.reasoning.utils import run_reasoning_extraction from vllm.reasoning import ReasoningParser, ReasoningParserManager +from vllm.reasoning.hy_v3_reasoning_parser import HYV3ReasoningParser from vllm.tokenizers import get_tokenizer parser_name = "hy_v3" @@ -241,3 +242,33 @@ def test_is_reasoning_end_full_prompt( token_ids = hy_v3_tokenizer.convert_tokens_to_ids(tokens) check_is_reasoning_end = parser.is_reasoning_end(token_ids) assert check_is_reasoning_end == is_reasoning_end + + +def test_constructor_does_not_mutate_shared_chat_template_kwargs(hy_v3_tokenizer): + parser_cls = ReasoningParserManager.get_reasoning_parser(parser_name) + chat_template_kwargs = {"reasoning_effort": "low"} + + first_parser: ReasoningParser = parser_cls( + hy_v3_tokenizer, + chat_template_kwargs=chat_template_kwargs, + ) + second_parser: ReasoningParser = parser_cls( + hy_v3_tokenizer, + chat_template_kwargs=chat_template_kwargs, + ) + + assert chat_template_kwargs == {"reasoning_effort": "low"} + assert isinstance(first_parser, HYV3ReasoningParser) + assert isinstance(second_parser, HYV3ReasoningParser) + assert first_parser._identity_parser is None + assert second_parser._identity_parser is None + + +def test_constructor_falls_back_to_outer_reasoning_effort(hy_v3_tokenizer): + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( + hy_v3_tokenizer, + reasoning_effort="low", + ) + + assert isinstance(parser, HYV3ReasoningParser) + assert parser._identity_parser is None diff --git a/vllm/reasoning/hy_v3_reasoning_parser.py b/vllm/reasoning/hy_v3_reasoning_parser.py index 6acaa13bb76..5beac22996d 100644 --- a/vllm/reasoning/hy_v3_reasoning_parser.py +++ b/vllm/reasoning/hy_v3_reasoning_parser.py @@ -34,8 +34,12 @@ class HYV3ReasoningParser(BaseThinkingReasoningParser): # at the outer level of the chat message. # Otherwise, If both are empty, assign "no_think". - chat_kwargs = kwargs.pop("chat_template_kwargs", {}) or {} - reasoning_effort = chat_kwargs.pop("reasoning_effort", "no_think") + chat_kwargs = kwargs.get("chat_template_kwargs", {}) or {} + reasoning_effort = ( + chat_kwargs.get("reasoning_effort") + or kwargs.get("reasoning_effort") + or "no_think" + ) logger.debug("reasoning_effort for choosing parser: %s", reasoning_effort) From 9744b699bafed423909ed10da96b80eb0542424b Mon Sep 17 00:00:00 2001 From: "wang.yuqi" Date: Fri, 24 Apr 2026 13:37:50 +0800 Subject: [PATCH 095/153] [Deprecate] Deprecate LLM.reward offline api, use LLM.encode instead. (#40688) Signed-off-by: wang.yuqi Signed-off-by: wang.yuqi Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Cyrus Leung --- docs/models/pooling_models/README.md | 48 +++++++------ docs/models/pooling_models/reward.md | 10 +++ .../pooling/reward/sequence_reward_offline.py | 62 ++++++++++++++++ .../pooling/reward/sequence_reward_online.py | 71 +++++++++++++++++++ .../reward/token_reward_offline.py} | 13 +++- .../token_reward_online.py} | 7 +- tests/conftest.py | 2 +- tests/entrypoints/pooling/pooling/__init__.py | 0 ...ffline.py => test_token_reward_offline.py} | 0 .../test_token_reward_online.py} | 0 vllm/entrypoints/llm.py | 28 ++++---- 11 files changed, 203 insertions(+), 38 deletions(-) create mode 100644 examples/pooling/reward/sequence_reward_offline.py create mode 100644 examples/pooling/reward/sequence_reward_online.py rename examples/{basic/offline_inference/reward.py => pooling/reward/token_reward_offline.py} (74%) rename examples/pooling/{pooling/pooling_online.py => reward/token_reward_online.py} (83%) delete mode 100644 tests/entrypoints/pooling/pooling/__init__.py rename tests/entrypoints/pooling/reward/{test_offline.py => test_token_reward_offline.py} (100%) rename tests/entrypoints/pooling/{pooling/test_online.py => reward/test_token_reward_online.py} (100%) diff --git a/docs/models/pooling_models/README.md b/docs/models/pooling_models/README.md index 6fd6064f6f7..7c8d6187148 100644 --- a/docs/models/pooling_models/README.md +++ b/docs/models/pooling_models/README.md @@ -78,7 +78,7 @@ The scoring models is designed to compute similarity scores between two input pr |-----------------------|---------------|----------------------------------------------|--------------------|--------------------------| | `classify` (see note) | Sequence-wise | reranker score for each sequence | `cross-encoder` | linear classifier | | `embed` | Sequence-wise | vector representations for each sequence | `bi-encoder` | cosine similarity | -| `token_classify` | Token-wise | probability vector of classes for each token | nan | nan | +| `token_classify` | Token-wise | probability vector of classes for each token | N/A | N/A | | `token_embed` | Token-wise | vector representations for each token | `late-interaction` | late interaction(MaxSim) | !!! note @@ -86,14 +86,15 @@ The scoring models is designed to compute similarity scores between two input pr ### Pooling Usages -| Pooling Usages | Description | -|-----------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| Classification Usages | Predicting which predefined category, class, or label best corresponds to a given input. | -| Embedding Usages | Converts unstructured data (text, images, audio, etc.) into structured numerical vectors (embeddings). | -| Token Classification Usages | Token-wise classification | -| Token Embedding Usages | Token-wise embedding | -| Scoring Usages | Computes similarity scores between two inputs. It supports three model types (aka `score_type`): `cross-encoder`, `late-interaction`, and `bi-encoder`. | -| Reward Usages | Evaluates the quality of outputs generated by a language model, acting as a proxy for human preferences. | +| Pooling Usages | Description | +|-----------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------| +| Classification Usages | Predicting which predefined category, class, or label best corresponds to a given input. | +| Embedding Usages | Converts unstructured data (text, images, audio, etc.) into structured numerical vectors (embeddings). | +| Token Classification Usages | Token-wise classification | +| Token Embedding Usages | Token-wise embedding | +| Reward Usages | Evaluates the quality of outputs generated by a language model, acting as a proxy for human preferences. | +| Scoring Usages | Computes similarity scores between two inputs. It supports three model types (aka `score_type`): `cross-encoder`, `late-interaction`, and `bi-encoder`. | +| Plugins Usages | Allow users to customize input and output processors. For more information, please refer to [IO Processor Plugins](../../design/io_processor_plugins.md). | We also have some special models that support multiple pooling tasks, or have specific usage scenarios, or support special inputs and outputs. @@ -101,9 +102,9 @@ For more detailed information, please refer to the link below. - [Classification Usages](classify.md) - [Embedding Usages](embed.md) -- [Reward Usages](reward.md) - [Token Classification Usages](token_classify.md) - [Token Embedding Usages](token_embed.md) +- [Reward Usages](reward.md) - [Scoring Usages](scoring.md) - [Specific Model Examples](specific_models.md) @@ -113,15 +114,17 @@ Each pooling model in vLLM supports one or more of these tasks according to [Pooler.get_supported_tasks][vllm.model_executor.layers.pooler.Pooler.get_supported_tasks], enabling the corresponding APIs. -### Offline APIs corresponding to pooling tasks +### Offline APIs corresponding to pooling usages -| Task | APIs | -|------------------|---------------------------------------------------------------------------------------| -| `embed` | `LLM.embed(...)`, `LLM.encode(..., pooling_task="embed")`, `LLM.score(...)`(see note) | -| `classify` | `LLM.classify(...)`, `LLM.encode(..., pooling_task="classify")`, `LLM.score(...)` | -| `token_classify` | `LLM.reward(...)`, `LLM.encode(..., pooling_task="token_classify")` | -| `token_embed` | `LLM.encode(..., pooling_task="token_embed")`, `LLM.score(...)` | -| `plugin` | `LLM.encode(..., pooling_task="plugin")` | +| Pooling Usages | Dedicated API | Pooling task for `LLM.encode` API | Score Types | scoring function | +|-----------------------------|---------------------|-----------------------------------|----------------------------|--------------------------| +| Classification Usages | `LLM.classify(...)` | `classify` | `cross-encoder` (see note) | linear classifier | +| Embedding Usages | `LLM.embed(...)` | `embed` | `bi-encoder` | cosine similarity | +| Token Classification Usages | N/A | `token_classify` | N/A | N/A | +| Token Embedding Usages | N/A | `token_embed` | `late-interaction` | late interaction(MaxSim) | +| Reward Usages | N/A | `classify` & `token_classify` | N/A | N/A | +| Scoring Usages | `LLM.score(...)` | N/A | N/A | N/A | +| Plugins Usages | N/A | `plugin` | N/A | N/A | !!! note Only when a classification model outputs num_labels equal to 1 can it be used as a scoring model and have its scoring API enabled. @@ -147,7 +150,7 @@ It is primarily designed for [score models](scoring.md). The [encode][vllm.LLM.encode] method is available to all pooling models in vLLM. -Please use one of the more specific methods or set the task directly when using `LLM.encode`, refer to the [table above](#offline-apis-corresponding-to-pooling-tasks). +Please use one of the more specific methods or set the task directly when using `LLM.encode`, refer to the [table above](#offline-apis-corresponding-to-pooling-usages). ### Examples @@ -183,9 +186,12 @@ Our Pooling API (`/pooling`) is similar to `LLM.encode`, being applicable to all The input format is the same as [Embeddings API](embed.md#openai-compatible-embeddings-api), but the output data can contain an arbitrary nested list, not just a 1-D list of floats. -Please use one of the more specific APIs or set the task directly when using the Pooling API, refer to the [table above](#offline-apis-corresponding-to-pooling-tasks). +Please use one of the more specific APIs or set the task directly when using the Pooling API, refer to the [table above](#offline-apis-corresponding-to-pooling-usages). -Code example: [examples/pooling/pooling/pooling_online.py](../../../examples/pooling/pooling/pooling_online.py) +Code examples: + +- [Online example](../../../examples/pooling/reward/token_reward_online.py) +- [Offline example](../../../examples/pooling/reward/token_reward_offline.py) ### Examples diff --git a/docs/models/pooling_models/reward.md b/docs/models/pooling_models/reward.md index 8555060e66b..7cb6e8b5bb6 100644 --- a/docs/models/pooling_models/reward.md +++ b/docs/models/pooling_models/reward.md @@ -134,3 +134,13 @@ print(f"Data: {data!r}") ## Online Serving Please refer to the [pooling API](README.md#pooling-api). Pooling task corresponding to reward model types refer to the [table above](#summary). + +## More examples + +More examples can be found here: [examples/pooling/reward](../../../examples/pooling/reward) + +## Deprecated Features + +### `LLM.reward` + +`llm.reward` api is deprecated and will be removed in v0.23. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead. diff --git a/examples/pooling/reward/sequence_reward_offline.py b/examples/pooling/reward/sequence_reward_offline.py new file mode 100644 index 00000000000..0727bceee11 --- /dev/null +++ b/examples/pooling/reward/sequence_reward_offline.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +""" +Example offline usage of sequence reward models. + +The key distinction between sequence classification and token classification +lies in their output granularity: sequence classification produces a single +result for an entire input sequence, whereas token classification yields a +result for each individual token within the sequence. +""" + +from argparse import Namespace + +from vllm import LLM, EngineArgs +from vllm.utils.argparse_utils import FlexibleArgumentParser +from vllm.utils.print_utils import print_embeddings + + +def parse_args(): + parser = FlexibleArgumentParser() + parser = EngineArgs.add_cli_args(parser) + # Set example specific arguments + parser.set_defaults( + model="Skywork/Skywork-Reward-V2-Qwen3-0.6B", + runner="pooling", + enforce_eager=True, + max_model_len=1024, + trust_remote_code=True, + ) + return parser.parse_args() + + +def main(args: Namespace): + # Sample prompts. + prompts = [ + "Hello, my name is", + "The president of the United States is", + "The capital of France is", + "The future of AI is", + ] + + # Create an LLM. + # You should pass runner="pooling" for reward models + llm = LLM(**vars(args)) + + # Generate rewards. The output is a list of PoolingRequestOutput. + # Use pooling_task="classify" for sequence reward models. + outputs = llm.encode(prompts, pooling_task="classify") + + # Print the outputs. + print("\nGenerated Outputs:\n" + "-" * 60) + for prompt, output in zip(prompts, outputs): + rewards = output.outputs.data + print(f"Prompt: {prompt!r}") + print_embeddings(rewards.tolist(), prefix="Reward") + print("-" * 60) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/examples/pooling/reward/sequence_reward_online.py b/examples/pooling/reward/sequence_reward_online.py new file mode 100644 index 00000000000..40d8d28e390 --- /dev/null +++ b/examples/pooling/reward/sequence_reward_online.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Example online usage of sequence reward models. + +Run `vllm serve --runner pooling` +to start up the server in vLLM. e.g. + +vllm serve Skywork/Skywork-Reward-V2-Qwen3-0.6B + +The key distinction between sequence classification and token classification +lies in their output granularity: sequence classification produces a single +result for an entire input sequence, whereas token classification yields a +result for each individual token within the sequence. +""" + +import argparse +import pprint + +import requests + + +def post_http_request(prompt: dict, api_url: str) -> requests.Response: + headers = {"User-Agent": "Test Client"} + response = requests.post(api_url, headers=headers, json=prompt) + return response + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--host", type=str, default="localhost") + parser.add_argument("--port", type=int, default=8000) + + return parser.parse_args() + + +def main(args): + base_url = f"http://{args.host}:{args.port}" + models_url = base_url + "/v1/models" + pooing_url = base_url + "/pooling" + + response = requests.get(models_url) + model = response.json()["data"][0]["id"] + + # Input like Completions API + prompt = {"model": model, "input": "vLLM is great!"} + pooling_response = post_http_request(prompt=prompt, api_url=pooing_url) + print("-" * 50) + print("Pooling Response:") + pprint.pprint(pooling_response.json()) + print("-" * 50) + + # Input like Chat API + prompt = { + "model": model, + "messages": [ + { + "role": "user", + "content": [{"type": "text", "text": "vLLM is great!"}], + } + ], + } + pooling_response = post_http_request(prompt=prompt, api_url=pooing_url) + print("Pooling Response:") + pprint.pprint(pooling_response.json()) + print("-" * 50) + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/examples/basic/offline_inference/reward.py b/examples/pooling/reward/token_reward_offline.py similarity index 74% rename from examples/basic/offline_inference/reward.py rename to examples/pooling/reward/token_reward_offline.py index b6aece26ace..4705c049124 100644 --- a/examples/basic/offline_inference/reward.py +++ b/examples/pooling/reward/token_reward_offline.py @@ -1,6 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Example offline usage of token reward models. + +The key distinction between sequence classification and token classification +lies in their output granularity: sequence classification produces a single +result for an entire input sequence, whereas token classification yields a +result for each individual token within the sequence. +""" + from argparse import Namespace from vllm import LLM, EngineArgs @@ -36,14 +45,14 @@ def main(args: Namespace): llm = LLM(**vars(args)) # Generate rewards. The output is a list of PoolingRequestOutput. - outputs = llm.reward(prompts) + outputs = llm.encode(prompts, pooling_task="token_classify") # Print the outputs. print("\nGenerated Outputs:\n" + "-" * 60) for prompt, output in zip(prompts, outputs): rewards = output.outputs.data print(f"Prompt: {prompt!r}") - print_embeddings(rewards, prefix="Reward") + print_embeddings(rewards.tolist(), prefix="Reward") print("-" * 60) diff --git a/examples/pooling/pooling/pooling_online.py b/examples/pooling/reward/token_reward_online.py similarity index 83% rename from examples/pooling/pooling/pooling_online.py rename to examples/pooling/reward/token_reward_online.py index e8ff38889a1..64ee0c9dfdc 100644 --- a/examples/pooling/pooling/pooling_online.py +++ b/examples/pooling/reward/token_reward_online.py @@ -1,12 +1,17 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """ -Example online usage of Pooling API. +Example online usage of token reward models. Run `vllm serve --runner pooling` to start up the server in vLLM. e.g. vllm serve internlm/internlm2-1_8b-reward --trust-remote-code + +The key distinction between sequence classification and token classification +lies in their output granularity: sequence classification produces a single +result for an entire input sequence, whereas token classification yields a +result for each individual token within the sequence. """ import argparse diff --git a/tests/conftest.py b/tests/conftest.py index 4dbf3c8da15..9ec31d83c75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1183,7 +1183,7 @@ class VllmRunner: return [req_output.outputs.data for req_output in req_outputs] def reward(self, prompts: list[str]) -> list[list[float]]: - req_outputs = self.llm.reward(prompts) + req_outputs = self.llm.encode(prompts, pooling_task="token_classify") return [req_output.outputs.data for req_output in req_outputs] def score( diff --git a/tests/entrypoints/pooling/pooling/__init__.py b/tests/entrypoints/pooling/pooling/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/entrypoints/pooling/reward/test_offline.py b/tests/entrypoints/pooling/reward/test_token_reward_offline.py similarity index 100% rename from tests/entrypoints/pooling/reward/test_offline.py rename to tests/entrypoints/pooling/reward/test_token_reward_offline.py diff --git a/tests/entrypoints/pooling/pooling/test_online.py b/tests/entrypoints/pooling/reward/test_token_reward_online.py similarity index 100% rename from tests/entrypoints/pooling/pooling/test_online.py rename to tests/entrypoints/pooling/reward/test_token_reward_online.py diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 0146cb83aa6..29cc2b47e7b 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -1166,19 +1166,16 @@ class LLM: if pooling_task is None: raise ValueError( - "pooling_task required for `LLM.encode`\n" - "Please use one of the more specific methods or set the " - "pooling_task when using `LLM.encode`:\n" - " - For embeddings, use `LLM.embed(...)` " - 'or `pooling_task="embed"`.\n' - " - For classification logits, use `LLM.classify(...)` " - 'or `pooling_task="classify"`.\n' - " - For similarity scores, use `LLM.score(...)`.\n" - " - For rewards, use `LLM.reward(...)` " - 'or `pooling_task="token_classify"`\n' - " - For token classification, " - 'use `pooling_task="token_classify"`\n' - ' - For multi-vector retrieval, use `pooling_task="token_embed"`' + """ + pooling_task required for `LLM.encode`. + Please use one of the more specific methods or set the pooling_task when using `LLM.encode`: + - For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`. + - For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`. + - For similarity scores, use `LLM.score(...)`. + - For rewards, `pooling_task="classify"` or `pooling_task="token_classify"`. + - For token classification, use `pooling_task="token_classify"`. + - For multi-vector retrieval, use `pooling_task="token_embed"`. + """ # noqa: E501 ) if ( @@ -1340,6 +1337,11 @@ class LLM: A list of `PoolingRequestOutput` objects containing the pooled hidden states in the same order as the input prompts. """ + logger.warning_once( + "`llm.reward` api is deprecated and will be removed in v0.23. " + 'Please use `LLM.encode` with `pooling_task="classify"` or ' + '`pooling_task="token_classify"` instead.' + ) return self.encode( prompts, use_tqdm=use_tqdm, From 079a4cf399ad548d442fd92bfffbfbe460b66133 Mon Sep 17 00:00:00 2001 From: Jackmin801 <56836461+Jackmin801@users.noreply.github.com> Date: Thu, 23 Apr 2026 23:05:49 -0700 Subject: [PATCH 096/153] [MoE] Move cutlass moe to fused_moe/experts/ (#40574) Signed-off-by: Jackmin801 Co-authored-by: Claude --- benchmarks/kernels/benchmark_cutlass_moe_fp8.py | 2 +- benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py | 2 +- benchmarks/kernels/benchmark_grouped_gemm_cutlass.py | 2 +- docs/design/moe_kernel_features.md | 4 ++-- tests/kernels/moe/modular_kernel_tools/mk_objects.py | 4 +++- tests/kernels/moe/test_cutlass_moe.py | 2 +- tests/kernels/moe/test_nvfp4_moe.py | 2 +- vllm/model_executor/layers/fused_moe/__init__.py | 8 ++++---- .../layers/fused_moe/{ => experts}/cutlass_moe.py | 0 vllm/model_executor/layers/fused_moe/oracle/fp8.py | 2 +- vllm/model_executor/layers/fused_moe/oracle/nvfp4.py | 2 +- .../model_executor/layers/fused_moe/triton_cutlass_moe.py | 2 +- .../compressed_tensors_moe_w4a4_mxfp4.py | 4 ++-- .../compressed_tensors_moe_w4a8_fp8.py | 2 +- 14 files changed, 20 insertions(+), 18 deletions(-) rename vllm/model_executor/layers/fused_moe/{ => experts}/cutlass_moe.py (100%) diff --git a/benchmarks/kernels/benchmark_cutlass_moe_fp8.py b/benchmarks/kernels/benchmark_cutlass_moe_fp8.py index 3f80b024e10..03d7fb386f7 100644 --- a/benchmarks/kernels/benchmark_cutlass_moe_fp8.py +++ b/benchmarks/kernels/benchmark_cutlass_moe_fp8.py @@ -16,7 +16,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( maybe_make_prepare_finalize, ) from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config -from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8 from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk from vllm.platforms import current_platform from vllm.utils.argparse_utils import FlexibleArgumentParser diff --git a/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py b/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py index 2d4afd38c09..7379bf85888 100644 --- a/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py +++ b/benchmarks/kernels/benchmark_cutlass_moe_nvfp4.py @@ -22,7 +22,7 @@ from vllm.model_executor.layers.fused_moe.config import ( fp8_w8a8_moe_quant_config, nvfp4_moe_quant_config, ) -from vllm.model_executor.layers.fused_moe.cutlass_moe import ( +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsFp4, ) from vllm.model_executor.layers.fused_moe.fused_moe import fused_experts, fused_topk diff --git a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py index dd4060bbdb9..04fc2960d1e 100644 --- a/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py +++ b/benchmarks/kernels/benchmark_grouped_gemm_cutlass.py @@ -13,7 +13,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( maybe_make_prepare_finalize, ) from vllm.model_executor.layers.fused_moe.config import fp8_w8a8_moe_quant_config -from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8 from vllm.model_executor.layers.fused_moe.fused_moe import ( fused_experts, fused_topk, diff --git a/docs/design/moe_kernel_features.md b/docs/design/moe_kernel_features.md index 231bca3646f..4e3706645ef 100644 --- a/docs/design/moe_kernel_features.md +++ b/docs/design/moe_kernel_features.md @@ -83,8 +83,8 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k | triton | standard | all1 | G,A,T | silu, gelu,
swigluoai,
silu_no_mul,
gelu_no_mul | Y | Y | [`fused_experts`][vllm.model_executor.layers.fused_moe.fused_moe.fused_experts],
[`TritonExperts`][vllm.model_executor.layers.fused_moe.fused_moe.TritonExperts] | | triton (batched) | batched | all1 | G,A,T | silu, gelu | 6 | Y | [`BatchedTritonExperts`][vllm.model_executor.layers.fused_moe.fused_batched_moe.BatchedTritonExperts] | | deep gemm | standard,
batched | fp8 | G(128),A,T | silu, gelu | 6 | Y |
[`DeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe.DeepGemmExperts],
[`BatchedDeepGemmExperts`][vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe.BatchedDeepGemmExperts] | -| cutlass_fp4 | standard,
batched | nvfp4 | A,T | silu | Y | Y | [`CutlassExpertsFp4`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp4] | -| cutlass_fp8 | standard,
batched | fp8 | A,T | silu, gelu | Y | Y | [`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassExpertsFp8],
[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.cutlass_moe.CutlassBatchedExpertsFp8] | +| cutlass_fp4 | standard,
batched | nvfp4 | A,T | silu | Y | Y | [`CutlassExpertsFp4`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassExpertsFp4] | +| cutlass_fp8 | standard,
batched | fp8 | A,T | silu, gelu | Y | Y | [`CutlassExpertsFp8`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassExpertsFp8],
[`CutlasBatchedExpertsFp8`][vllm.model_executor.layers.fused_moe.experts.cutlass_moe.CutlassBatchedExpertsFp8] | | flashinfer | standard | nvfp4,
fp8 | T | 5 | N | Y | [`FlashInferExperts`][vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe.FlashInferExperts] | | gpt oss triton | standard | N/A | N/A | 5 | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],
[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] | | marlin | standard,
batched | 3 / N/A | 3 / N/A | silu,
swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.fused_marlin_moe],
[`MarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.MarlinExperts],
[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.fused_marlin_moe.BatchedMarlinExperts] | diff --git a/tests/kernels/moe/modular_kernel_tools/mk_objects.py b/tests/kernels/moe/modular_kernel_tools/mk_objects.py index 23ddc7011ac..812164ea287 100644 --- a/tests/kernels/moe/modular_kernel_tools/mk_objects.py +++ b/tests/kernels/moe/modular_kernel_tools/mk_objects.py @@ -367,7 +367,9 @@ else: CutlassExpertsFp8 = None if cutlass_fp4_supported(): - from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp4 + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( + CutlassExpertsFp4, + ) register_experts( CutlassExpertsFp4, diff --git a/tests/kernels/moe/test_cutlass_moe.py b/tests/kernels/moe/test_cutlass_moe.py index e06672f41d0..a613e7d2e29 100644 --- a/tests/kernels/moe/test_cutlass_moe.py +++ b/tests/kernels/moe/test_cutlass_moe.py @@ -21,7 +21,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, fp8_w8a8_moe_quant_config, ) -from vllm.model_executor.layers.fused_moe.cutlass_moe import ( +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsFp8, run_cutlass_moe_fp8, ) diff --git a/tests/kernels/moe/test_nvfp4_moe.py b/tests/kernels/moe/test_nvfp4_moe.py index e12659729c9..e2a6cd1a7dc 100644 --- a/tests/kernels/moe/test_nvfp4_moe.py +++ b/tests/kernels/moe/test_nvfp4_moe.py @@ -19,7 +19,7 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import ( maybe_make_prepare_finalize, ) from vllm.model_executor.layers.fused_moe.config import nvfp4_moe_quant_config -from vllm.model_executor.layers.fused_moe.cutlass_moe import ( +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsFp4, ) from vllm.model_executor.layers.fused_moe.prepare_finalize import ( diff --git a/vllm/model_executor/layers/fused_moe/__init__.py b/vllm/model_executor/layers/fused_moe/__init__.py index 1d273bd31e4..75a9faddc1f 100644 --- a/vllm/model_executor/layers/fused_moe/__init__.py +++ b/vllm/model_executor/layers/fused_moe/__init__.py @@ -73,15 +73,15 @@ __all__ = [ if HAS_TRITON: # import to register the custom ops - from vllm.model_executor.layers.fused_moe.cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import ( + BatchedDeepGemmExperts, + ) + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassBatchedExpertsFp8, CutlassExpertsFp8, CutlassExpertsW4A8Fp8, cutlass_moe_w4a8_fp8, ) - from vllm.model_executor.layers.fused_moe.experts.batched_deep_gemm_moe import ( - BatchedDeepGemmExperts, - ) from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import ( DeepGemmExperts, ) diff --git a/vllm/model_executor/layers/fused_moe/cutlass_moe.py b/vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py similarity index 100% rename from vllm/model_executor/layers/fused_moe/cutlass_moe.py rename to vllm/model_executor/layers/fused_moe/experts/cutlass_moe.py diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index ca13d0d901d..2e75e6f4ae7 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -173,7 +173,7 @@ def backend_to_kernel_cls( return [TritonOrCutlassExperts] elif backend == Fp8MoeBackend.BATCHED_VLLM_CUTLASS: - from vllm.model_executor.layers.fused_moe.cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassBatchedExpertsFp8, ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py index 724f6d5399b..48e48a97ef9 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/nvfp4.py @@ -107,7 +107,7 @@ def backend_to_kernel_cls( return [FlashInferCuteDSLBatchedExperts] elif backend == NvFp4MoeBackend.VLLM_CUTLASS: - from vllm.model_executor.layers.fused_moe.cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsFp4, ) diff --git a/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py b/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py index 4aa396d24b0..70431878932 100644 --- a/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py +++ b/vllm/model_executor/layers/fused_moe/triton_cutlass_moe.py @@ -10,7 +10,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEConfig, FusedMoEQuantConfig, ) -from vllm.model_executor.layers.fused_moe.cutlass_moe import CutlassExpertsFp8 +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import CutlassExpertsFp8 from vllm.model_executor.layers.fused_moe.fallback import FallbackExperts from vllm.model_executor.layers.fused_moe.fused_moe import TritonExperts from vllm.platforms import current_platform diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py index 9d3e0e7a787..629e1c5ef1b 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py @@ -14,7 +14,7 @@ from vllm.model_executor.layers.fused_moe.config import ( FusedMoEQuantConfig, mxfp4_moe_quant_config, ) -from vllm.model_executor.layers.fused_moe.cutlass_moe import ( +from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( CutlassExpertsMxfp4, ) from vllm.model_executor.layers.fused_moe.fused_marlin_moe import ( @@ -149,7 +149,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod): if self.use_cutlass_mxfp4: # Swizzle weight scales from flat checkpoint layout [E, N, K//32] # to CUTLASS tiled layout [E, numMTiles*numKTiles*512]. - from vllm.model_executor.layers.fused_moe.cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( swizzle_mxfp4_scales, ) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py index ab805591dee..b14571fe501 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a8_fp8.py @@ -315,7 +315,7 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod): ) assert self.moe_quant_config is not None - from vllm.model_executor.layers.fused_moe.cutlass_moe import ( + from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import ( cutlass_moe_w4a8_fp8, ) From 01acf96c6f57914479e6bfe79d7bd5777a2fc49f Mon Sep 17 00:00:00 2001 From: xiangdong <40376367+zxd1997066@users.noreply.github.com> Date: Fri, 24 Apr 2026 14:08:45 +0800 Subject: [PATCH 097/153] [XPU][CI] Fix Docker cleanup races on Intel CI runners (#40761) Signed-off-by: zengxian --- .../scripts/hardware_ci/run-intel-test.sh | 92 +++++++++++++++++-- .../scripts/hardware_ci/run-xpu-test.sh | 4 +- 2 files changed, 85 insertions(+), 11 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-intel-test.sh b/.buildkite/scripts/hardware_ci/run-intel-test.sh index 0399f30b61b..eae3b231b4b 100755 --- a/.buildkite/scripts/hardware_ci/run-intel-test.sh +++ b/.buildkite/scripts/hardware_ci/run-intel-test.sh @@ -25,22 +25,100 @@ export PYTHONPATH=".." ############################################################################### cleanup_docker() { + # Share the same lock with image pull to avoid cleanup/pull races on one node. + local docker_lock="/tmp/docker-pull.lock" + exec 9>"$docker_lock" + flock 9 + docker_root=$(docker info -f '{{.DockerRootDir}}') if [ -z "$docker_root" ]; then echo "Failed to determine Docker root directory." >&2 - exit 1 + flock -u 9 + return 1 fi echo "Docker root directory: $docker_root" disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//') threshold=70 if [ "$disk_usage" -gt "$threshold" ]; then - echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..." - docker image prune -f - docker volume prune -f && docker system prune --force --filter "until=72h" --all - echo "Docker images and volumes cleanup completed." + echo "Disk usage is above $threshold%. Running aggressive CI image cleanup..." + cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 1 else - echo "Disk usage is below $threshold%. No cleanup needed." + echo "Disk usage is below $threshold%. Checking old CI images anyway." + cleanup_old_ci_images "${REGISTRY}/${REPO}" "${image_name}" "${DOCKER_IMAGE_CLEANUP_HOURS:-72}" 0 + fi + echo "Old CI image cleanup completed." + + flock -u 9 +} + +cleanup_old_ci_images() { + local repo_prefix="$1" + local current_image_ref="$2" + local ttl_hours="$3" + local aggressive_cleanup="$4" + + if [[ -z "$repo_prefix" || "$repo_prefix" == "/" ]]; then + echo "Skip old-image cleanup: invalid repo prefix '${repo_prefix}'" + return 0 + fi + + if ! [[ "$ttl_hours" =~ ^[0-9]+$ ]]; then + echo "Invalid DOCKER_IMAGE_CLEANUP_HOURS='${ttl_hours}', fallback to 72" + ttl_hours=72 + fi + + local now_epoch cutoff_epoch + now_epoch=$(date +%s) + cutoff_epoch=$((now_epoch - ttl_hours * 3600)) + + local -a used_image_ids + mapfile -t used_image_ids < <(docker ps -aq | xargs -r docker inspect --format '{{.Image}}' | sort -u) + + local removed_count=0 + local examined_count=0 + declare -A seen_ids=() + + while read -r image_ref image_id; do + [[ -z "$image_ref" || -z "$image_id" ]] && continue + ((examined_count++)) + + # Keep the image this job is going to use. + if [[ "$image_ref" == "$current_image_ref" ]]; then + continue + fi + + # Avoid duplicate deletes when multiple tags point to same image id. + if [[ -n "${seen_ids[$image_id]:-}" ]]; then + continue + fi + seen_ids[$image_id]=1 + + # Never delete images that are used by any container on this node. + if printf '%s\n' "${used_image_ids[@]}" | grep -qx "$image_id"; then + continue + fi + + local created created_epoch + created=$(docker image inspect -f '{{.Created}}' "$image_id" 2>/dev/null || true) + [[ -z "$created" ]] && continue + created_epoch=$(date -d "$created" +%s 2>/dev/null || true) + [[ -z "$created_epoch" ]] && continue + + if (( created_epoch < cutoff_epoch )) || [[ "$aggressive_cleanup" == "1" ]]; then + if docker image rm -f "$image_id" >/dev/null 2>&1; then + ((removed_count++)) + fi + fi + done < <(docker image ls --no-trunc "$repo_prefix" --format '{{.Repository}}:{{.Tag}} {{.ID}}') + + # Also trim old dangling layers; this is safe and does not remove referenced images. + docker image prune -f --filter "until=${ttl_hours}h" >/dev/null 2>&1 || true + + if [[ "$aggressive_cleanup" == "1" ]]; then + echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} unused images under disk pressure." + else + echo "Examined ${examined_count} images under ${repo_prefix}, removed ${removed_count} old images (>${ttl_hours}h)." fi } @@ -265,8 +343,6 @@ fi remove_docker_container() { docker rm -f "${container_name}" || true - docker image rm -f "${image_name}" || true - docker system prune -f || true } trap remove_docker_container EXIT diff --git a/.buildkite/scripts/hardware_ci/run-xpu-test.sh b/.buildkite/scripts/hardware_ci/run-xpu-test.sh index 6579810e982..14bd08cfc1c 100644 --- a/.buildkite/scripts/hardware_ci/run-xpu-test.sh +++ b/.buildkite/scripts/hardware_ci/run-xpu-test.sh @@ -12,9 +12,7 @@ docker build -t "${image_name}" -f docker/Dockerfile.xpu . # Setup cleanup remove_docker_container() { - docker rm -f "${container_name}" || true; - docker image rm -f "${image_name}" || true; - docker system prune -f || true; + docker rm -f "${container_name}" || true } trap remove_docker_container EXIT From cf8a613a87264183058801309868722f9013e101 Mon Sep 17 00:00:00 2001 From: Xin Yang <105740670+xyang16@users.noreply.github.com> Date: Thu, 23 Apr 2026 23:51:05 -0700 Subject: [PATCH 098/153] Support only half types for concat_mla_q kernel (#37892) Signed-off-by: Xin Yang --- csrc/cache_kernels.cu | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/csrc/cache_kernels.cu b/csrc/cache_kernels.cu index 6bea5abc3df..1dd9be8b46a 100644 --- a/csrc/cache_kernels.cu +++ b/csrc/cache_kernels.cu @@ -1490,6 +1490,9 @@ void concat_mla_q(torch::Tensor& ql_nope, // [num_tokens, num_heads, nope_dim] TORCH_CHECK(ql_nope.stride(2) == 1, "ql_nope must have stride 1 in dim 2"); TORCH_CHECK(q_pe.stride(2) == 1, "q_pe must have stride 1 in dim 2"); TORCH_CHECK(q_out.stride(2) == 1, "q_out must have stride 1 in dim 2"); + TORCH_CHECK(ql_nope.scalar_type() == at::ScalarType::Half || + ql_nope.scalar_type() == at::ScalarType::BFloat16, + "ql_nope must be float16 or bfloat16 dtype"); if (num_tokens == 0) return; @@ -1501,7 +1504,7 @@ void concat_mla_q(torch::Tensor& ql_nope, // [num_tokens, num_heads, nope_dim] const at::cuda::OptionalCUDAGuard device_guard(device_of(ql_nope)); const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); - VLLM_DISPATCH_FLOATING_TYPES(ql_nope.scalar_type(), "concat_mla_q", [&] { + VLLM_DISPATCH_HALF_TYPES(ql_nope.scalar_type(), "concat_mla_q", [&] { vllm::ConcatMLAQKernel<<>>( q_out.data_ptr(), ql_nope.data_ptr(), q_pe.data_ptr(), num_tokens, num_heads, q_out.stride(0), From 4c34b2f6fc63435c791c9054c579ca3f8c902bb6 Mon Sep 17 00:00:00 2001 From: Yuwen Zhou Date: Fri, 24 Apr 2026 16:26:16 +0800 Subject: [PATCH 099/153] [XPU] Enable torch.compile for XPU GDN attention (#39466) Signed-off-by: yuwenzho Signed-off-by: Yuwen Zhou Co-authored-by: Kunshang Ji --- vllm/_xpu_ops.py | 73 +++++++++++++++++++ vllm/config/compilation.py | 1 + .../layers/mamba/gdn_linear_attn.py | 48 ++---------- 3 files changed, 81 insertions(+), 41 deletions(-) diff --git a/vllm/_xpu_ops.py b/vllm/_xpu_ops.py index 7db074bf920..0b39a400012 100644 --- a/vllm/_xpu_ops.py +++ b/vllm/_xpu_ops.py @@ -92,6 +92,72 @@ if hasattr(torch.ops._xpu_C, "int4_gemm_w4a16"): return torch.empty((M, N), dtype=input.dtype, device=input.device) +def _gdn_attention_core_xpu_impl( + core_attn_out: torch.Tensor, + z: torch.Tensor, + projected_states_qkvz: torch.Tensor, + projected_states_ba: torch.Tensor, + layer_name: str, +) -> None: + """Custom op wrapping the XPU SYCL GDN kernel for torch.compile.""" + from vllm.forward_context import get_forward_context + from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata + + forward_context = get_forward_context() + self = forward_context.no_compile_layers[layer_name] + attn_metadata_raw = forward_context.attn_metadata + + if attn_metadata_raw is None: + return + + assert isinstance(attn_metadata_raw, dict) + attn_metadata = attn_metadata_raw[self.prefix] + assert isinstance(attn_metadata, GDNAttentionMetadata) + + # TODO: xpu does not support speculative decoding yet + assert attn_metadata.spec_sequence_masks is None # type: ignore[attr-defined] + + conv_weights = self.conv1d.weight.view( + self.conv1d.weight.size(0), self.conv1d.weight.size(2) + ) + + torch.ops._xpu_C.gdn_attention( + core_attn_out, + z, + projected_states_qkvz, + projected_states_ba, + self.num_k_heads, + self.num_v_heads, + self.head_k_dim, + self.head_v_dim, + conv_state=self.kv_cache[0], + ssm_state=self.kv_cache[1], + conv_weights=conv_weights, + conv_bias=self.conv1d.bias, + activation=self.activation, + A_log=self.A_log, + dt_bias=self.dt_bias, + num_prefills=attn_metadata.num_prefills, # type: ignore[attr-defined] + num_decodes=attn_metadata.num_decodes, # type: ignore[attr-defined] + has_initial_state=attn_metadata.has_initial_state, # type: ignore[attr-defined] + non_spec_query_start_loc=attn_metadata.non_spec_query_start_loc, # type: ignore[attr-defined] + non_spec_state_indices_tensor=attn_metadata.non_spec_state_indices_tensor, # type: ignore[attr-defined] + num_actual_tokens=attn_metadata.num_actual_tokens, # type: ignore[attr-defined] + tp_size=self.tp_size, + reorder_input=not self.gqa_interleaved_layout, + ) + + +def _gdn_attention_core_xpu_fake( + core_attn_out: torch.Tensor, + z: torch.Tensor, + projected_states_qkvz: torch.Tensor, + projected_states_ba: torch.Tensor, + layer_name: str, +) -> None: + return + + def _xpu_ops_deepseek_scaling_rope_impl( positions: torch.Tensor, query: torch.Tensor, @@ -618,6 +684,13 @@ class xpu_ops: fake_impl=_xpu_mxfp4_quantize_fake, ) + direct_register_custom_op( + op_name="gdn_attention_core_xpu", + op_func=_gdn_attention_core_xpu_impl, + mutates_args=["core_attn_out", "z"], + fake_impl=_gdn_attention_core_xpu_fake, + ) + _OPS_REGISTERED = True diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index f7483db52a4..5b726899c2f 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -744,6 +744,7 @@ class CompilationConfig: "vllm::linear_attention", "vllm::plamo2_mamba_mixer", "vllm::gdn_attention_core", + "vllm::gdn_attention_core_xpu", "vllm::olmo_hybrid_gdn_full_forward", "vllm::kda_attention", "vllm::sparse_attn_indexer", diff --git a/vllm/model_executor/layers/mamba/gdn_linear_attn.py b/vllm/model_executor/layers/mamba/gdn_linear_attn.py index 7a0b54335ba..a621ab962f0 100644 --- a/vllm/model_executor/layers/mamba/gdn_linear_attn.py +++ b/vllm/model_executor/layers/mamba/gdn_linear_attn.py @@ -618,54 +618,20 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase): # ============================================================ # Part 2: Core Attention # ============================================================ - forward_context = get_forward_context() - attn_metadata_raw = forward_context.attn_metadata core_attn_out = torch.zeros( (num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim), dtype=hidden_states.dtype, device=hidden_states.device, ) z = torch.empty_like(core_attn_out) - if attn_metadata_raw is not None: - assert isinstance(attn_metadata_raw, dict) - attn_metadata = attn_metadata_raw[self.prefix] - # TODO: xpu does not support this param yet - spec_sequence_masks = attn_metadata.spec_sequence_masks # type: ignore[attr-defined] - assert spec_sequence_masks is None - - conv_weights = self.conv1d.weight.view( - self.conv1d.weight.size(0), self.conv1d.weight.size(2) - ) - - conv_state = self.kv_cache[0] - ssm_state = self.kv_cache[1] - - torch.ops._xpu_C.gdn_attention( - core_attn_out, - z, - projected_states_qkvz, - projected_states_ba, - self.num_k_heads, - self.num_v_heads, - self.head_k_dim, - self.head_v_dim, - conv_state=conv_state, - ssm_state=ssm_state, - conv_weights=conv_weights, - conv_bias=self.conv1d.bias, - activation=self.activation, - A_log=self.A_log, - dt_bias=self.dt_bias, - num_prefills=attn_metadata.num_prefills, # type: ignore[attr-defined] - num_decodes=attn_metadata.num_decodes, # type: ignore[attr-defined] - has_initial_state=attn_metadata.has_initial_state, # type: ignore[attr-defined] - non_spec_query_start_loc=attn_metadata.non_spec_query_start_loc, # type: ignore[attr-defined] - non_spec_state_indices_tensor=attn_metadata.non_spec_state_indices_tensor, # type: ignore[attr-defined] - num_actual_tokens=attn_metadata.num_actual_tokens, # type: ignore[attr-defined] - tp_size=self.tp_size, - reorder_input=not self.gqa_interleaved_layout, - ) + torch.ops.vllm.gdn_attention_core_xpu( + core_attn_out, + z, + projected_states_qkvz, + projected_states_ba, + self.prefix, + ) # ============================================================ # Part 3: Output Projection From 512f52219240b0aa1be687955ab52fcdd0c5a40e Mon Sep 17 00:00:00 2001 From: Luciano Martins Date: Fri, 24 Apr 2026 01:27:46 -0700 Subject: [PATCH 100/153] [Model] Gemma4: add bidirectional vision attention for sliding layers with window guard (#40534) Signed-off-by: Luciano Martins Signed-off-by: Luciano Martins Signed-off-by: Isotr0py Co-authored-by: Luciano Martins Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py --- vllm/model_executor/models/gemma4_mm.py | 59 +++++++++++++++++++++++++ vllm/v1/worker/gpu_model_runner.py | 15 ++++++- 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/gemma4_mm.py b/vllm/model_executor/models/gemma4_mm.py index 46d0308f4c8..cdc54609a65 100644 --- a/vllm/model_executor/models/gemma4_mm.py +++ b/vllm/model_executor/models/gemma4_mm.py @@ -969,6 +969,16 @@ class Gemma4ForConditionalGeneration( self.language_model.make_empty_intermediate_tensors ) + # --- Precompute full-attention layer indices for bidi clearing --- + self._full_attn_layer_idxs: frozenset[int] = frozenset() + text_config = config.text_config + if getattr(text_config, "use_bidirectional_attention", None) == "vision": + layer_types = getattr(text_config, "layer_types", None) + if layer_types: + self._full_attn_layer_idxs = frozenset( + i for i, lt in enumerate(layer_types) if lt != "sliding_attention" + ) + # --- MixtureOfExperts delegation to language_model --- self.expert_weights = self.language_model.expert_weights self.moe_layers = self.language_model.moe_layers @@ -1310,6 +1320,12 @@ class Gemma4ForConditionalGeneration( else None ) + # Gemma4 bidi: clear mm_prefix_range for full_attention layers. + # Must run here (outside @support_torch_compile boundary) because + # _run_decoder_layers is inside a compiled graph where Python + # side effects are eliminated. + self._clear_mm_prefix_for_full_attn_layers() + hidden_states = self.language_model.model( input_ids, positions, @@ -1327,6 +1343,49 @@ class Gemma4ForConditionalGeneration( ) -> torch.Tensor | None: return self.language_model.compute_logits(hidden_states) + # ------------------------------------------------------------------ # + # Bidirectional attention helpers + # ------------------------------------------------------------------ # + + def _clear_mm_prefix_for_full_attn_layers(self) -> None: + """Clear mm_prefix_range for non-sliding layers. + + Gemma4 with use_bidirectional_attention='vision' applies + bidirectional attention only to sliding_attention layers. + Full attention layers use plain causal masking. + + Uses _full_attn_layer_idxs (precomputed in __init__) for O(1) + lookup instead of per-call regex parsing. + """ + if not self._full_attn_layer_idxs: + return + + from vllm.forward_context import get_forward_context + + attn_metadata = get_forward_context().attn_metadata + if attn_metadata is None: + return + + def _process(metadata_dict: dict) -> None: + for layer_name, metadata in metadata_dict.items(): + if ".layers." not in layer_name: + continue + try: + layer_idx = int(layer_name.split(".layers.")[1].split(".")[0]) + except (ValueError, IndexError): + continue + if layer_idx in self._full_attn_layer_idxs: + if hasattr(metadata, "mm_prefix_range"): + metadata.mm_prefix_range = None + if hasattr(metadata, "mm_prefix_range_tensor"): + metadata.mm_prefix_range_tensor = None + + if isinstance(attn_metadata, list): + for ub_metadata in attn_metadata: + _process(ub_metadata) + elif isinstance(attn_metadata, dict): + _process(attn_metadata) + # ------------------------------------------------------------------ # # Weight loading # ------------------------------------------------------------------ # diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 0b0fed4824a..0362011a6e6 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -2314,13 +2314,26 @@ class GPUModelRunner( if self.is_mm_prefix_lm: req_doc_ranges = {} + + # Gemma4 bidi: skip ranges that exceed the sliding + # window. When image tokens > sliding_window, bidi causes + # early image tokens to attend to the entire image + # (e.g. 6 → 1092 targets), degrading spatial precision. + # Per-range filtering keeps bidi for small images/video + # frames while skipping oversized images. + hf_text_config = self.model_config.hf_text_config + _bidi_sw = getattr(hf_text_config, "sliding_window", None) + for req_id in self.input_batch.req_ids: image_doc_ranges = [] req_state = self.requests[req_id] for mm_feature in req_state.mm_features: pos_info = mm_feature.mm_position img_doc_range = pos_info.extract_embeds_range() - image_doc_ranges.extend(img_doc_range) + for r in img_doc_range: + if _bidi_sw is not None and (r[1] - r[0] + 1) > _bidi_sw: + continue + image_doc_ranges.append(r) req_idx = self.input_batch.req_id_to_index[req_id] req_doc_ranges[req_idx] = image_doc_ranges From 7d3195ea9fc88e31131099d2d2122fe38558a87a Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Fri, 24 Apr 2026 01:40:20 -0700 Subject: [PATCH 101/153] [Bugfix] Fix IMA in DSA + MTP (#40772) Signed-off-by: Woosuk Kwon --- csrc/cache_kernels.cu | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/csrc/cache_kernels.cu b/csrc/cache_kernels.cu index 1dd9be8b46a..7e456d32598 100644 --- a/csrc/cache_kernels.cu +++ b/csrc/cache_kernels.cu @@ -599,6 +599,11 @@ __global__ void cp_gather_indexer_k_quant_cache_kernel( const int head_idx = (blockIdx.y * blockDim.x + threadIdx.x) * VEC_SIZE; // Find batch index within a block __shared__ int batch_idx[BLOCK_Y_SIZE]; + if (threadIdx.x == 0) { + batch_idx[threadIdx.y] = -1; + } + __syncthreads(); + for (int iter = 0; iter < cuda_utils::ceil_div(batch_size, int(blockDim.x)); iter++) { int tid = iter * blockDim.x + threadIdx.x; @@ -611,16 +616,18 @@ __global__ void cp_gather_indexer_k_quant_cache_kernel( } } -#ifndef USE_ROCM - __syncwarp(); -#endif + __syncthreads(); - if (head_idx >= head_dim || token_idx >= num_tokens) { + // num_tokens may be an allocation upper bound when Python avoids a D2H sync. + // Only tokens covered by the exact device-side cu_seq_lens are valid to + // gather. + const int batch = batch_idx[threadIdx.y]; + if (head_idx >= head_dim || token_idx >= num_tokens || batch < 0) { return; } - const int inbatch_seq_idx = token_idx - cu_seq_lens[batch_idx[threadIdx.y]]; - const int block_idx = block_table[batch_idx[threadIdx.y] * num_blocks + - inbatch_seq_idx / cache_block_size]; + const int inbatch_seq_idx = token_idx - cu_seq_lens[batch]; + const int block_idx = + block_table[batch * num_blocks + inbatch_seq_idx / cache_block_size]; const int64_t src_block_offset = block_idx * block_stride; const int64_t cache_inblock_offset = (inbatch_seq_idx % cache_block_size) * head_dim + head_idx; From 9ad5abe7722ba4eb9cb484689dd90529e76c41c5 Mon Sep 17 00:00:00 2001 From: milesial Date: Fri, 24 Apr 2026 02:18:55 -0700 Subject: [PATCH 102/153] Fix Nano Nemotron VL static image inputs (#40724) Signed-off-by: Alexandre Milesi --- vllm/model_executor/models/nano_nemotron_vl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index b0424675943..684ced0a6ab 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -1124,7 +1124,9 @@ class NemotronH_Nano_VL_V2( ) else: return NanoNemotronVLImagePixelInputs( - num_patches=kwargs.pop("image_num_patches"), **kwargs + pixel_values_flat=pixel_values_flat, + num_patches=kwargs.pop("image_num_patches"), + **kwargs, ) def _process_image_input_dynamic( From b5587e1013d0e352bb33c30b456d5221aebecd8c Mon Sep 17 00:00:00 2001 From: Shanshan Shen <467638484@qq.com> Date: Fri, 24 Apr 2026 18:12:14 +0800 Subject: [PATCH 103/153] [CI/Build] Add e2e test for ViT CUDA graph (#40780) Signed-off-by: shen-shanshan <467638484@qq.com> --- .buildkite/test_areas/models_multimodal.yaml | 1 + .../generation/test_vit_cudagraph.py | 166 ++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 tests/models/multimodal/generation/test_vit_cudagraph.py diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index ff0fd2e7a62..245ef24026d 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -28,6 +28,7 @@ steps: - pip install git+https://github.com/TIGER-AI-Lab/Mantis.git - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model + - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model mirror: amd: device: mi325_1 diff --git a/tests/models/multimodal/generation/test_vit_cudagraph.py b/tests/models/multimodal/generation/test_vit_cudagraph.py new file mode 100644 index 00000000000..7adea0771b6 --- /dev/null +++ b/tests/models/multimodal/generation/test_vit_cudagraph.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from dataclasses import dataclass, field + +import pytest + +from vllm.multimodal.video import sample_frames_from_video +from vllm.platforms import current_platform + +from ....conftest import IMAGE_ASSETS, VIDEO_ASSETS +from ....utils import create_new_process_for_each_test +from .vlm_utils.builders import sample_frames_with_video_metadata + + +@dataclass +class VitCudagraphTestConfig: + model: str + modalities: list[str] = field(default_factory=lambda: ["image", "video"]) + image_prompt: str | None = None + video_prompt: str | None = None + dtype: str = "bfloat16" + max_model_len: int = 4096 + max_tokens: int = 64 + max_num_seqs: int = 2 + num_video_frames: int = 16 + needs_video_metadata: bool = False + vllm_runner_kwargs: dict = field(default_factory=dict) + marks: list = field(default_factory=list) + + +def params_with_marks( + configs: dict[str, VitCudagraphTestConfig], +) -> list[pytest.param]: + return [ + pytest.param(model_id, marks=cfg.marks) for model_id, cfg in configs.items() + ] + + +def qwen_vl_chat_template(content: str) -> str: + return f"<|im_start|>user\n{content}<|im_end|>\n<|im_start|>assistant\n" + + +MODEL_CONFIGS: dict[str, VitCudagraphTestConfig] = { + "qwen3_vl": VitCudagraphTestConfig( + model="Qwen/Qwen3-VL-2B-Instruct", + image_prompt=qwen_vl_chat_template( + "<|vision_start|><|image_pad|><|vision_end|>What is in this image?" + ), + video_prompt=qwen_vl_chat_template( + "<|vision_start|><|video_pad|><|vision_end|>" + "Describe this video in one sentence." + ), + needs_video_metadata=True, + marks=[pytest.mark.core_model], + ), + # TODO: Add more models below. +} + + +def get_compilation_config(): + return { + "cudagraph_mm_encoder": True, + "encoder_cudagraph_max_vision_items_per_batch": 1, + "encoder_cudagraph_max_frames_per_batch": 16, + } + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS)) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +@create_new_process_for_each_test() +def test_vit_cudagraph_image(model_id, vllm_runner, image_assets): + config = MODEL_CONFIGS[model_id] + + if "image" not in config.modalities: + pytest.skip(f"{model_id} does not support the image modality.") + + image_prompts = IMAGE_ASSETS.prompts( + { + "stop_sign": config.image_prompt, # type: ignore[typeddict-item] + "cherry_blossom": config.image_prompt, # type: ignore[typeddict-item] + } + ) + images = [[asset.pil_image] for asset in image_assets] + + with vllm_runner( + config.model, + dtype=config.dtype, + max_model_len=config.max_model_len, + max_num_seqs=config.max_num_seqs, + limit_mm_per_prompt={"image": 1}, + compilation_config=get_compilation_config(), + **config.vllm_runner_kwargs, + ) as vllm_model: + outputs = vllm_model.generate_greedy( + image_prompts, config.max_tokens, images=images + ) + + # Basic validation that we got a response + assert len(outputs) == 2 + output_ids, output_text = outputs[0] + + # Ensure we got some output + assert len(output_ids) > 0 + assert len(output_text) > 0 + + # Ensure the output is a string + assert isinstance(output_text, str) + + +@pytest.mark.parametrize("model_id", params_with_marks(MODEL_CONFIGS)) +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +@create_new_process_for_each_test() +def test_vit_cudagraph_video(model_id, vllm_runner, video_assets): + config = MODEL_CONFIGS[model_id] + + if "video" not in config.modalities: + pytest.skip(f"{model_id} does not support the video modality") + + video_prompts = VIDEO_ASSETS.prompts( + { + "baby_reading": config.video_prompt, # type: ignore[typeddict-item] + } + ) + if config.needs_video_metadata: + sampled_vids = [ + sample_frames_with_video_metadata( + (asset.np_ndarrays, asset.metadata), config.num_video_frames + ) + for asset in video_assets + ] + else: + sampled_vids = [ + sample_frames_from_video(asset.np_ndarrays, config.num_video_frames) + for asset in video_assets + ] + videos = [sampled_vids[0]] + + with vllm_runner( + config.model, + dtype=config.dtype, + max_model_len=config.max_model_len, + max_num_seqs=config.max_num_seqs, + limit_mm_per_prompt={"video": 1}, + compilation_config=get_compilation_config(), + **config.vllm_runner_kwargs, + ) as vllm_model: + outputs = vllm_model.generate_greedy( + video_prompts, config.max_tokens, videos=videos + ) + + # Basic validation that we got a response + assert len(outputs) == 1 + output_ids, output_text = outputs[0] + + # Ensure we got some output + assert len(output_ids) > 0 + assert len(output_text) > 0 + + # Ensure the output is a string + assert isinstance(output_text, str) From 6dec49f27ece339c59d5eb92f33120c11c0c3b74 Mon Sep 17 00:00:00 2001 From: Dmitry Tokarev Date: Fri, 24 Apr 2026 06:27:11 -0400 Subject: [PATCH 104/153] [Build] Bump CUDA to 13.0.2 to match PyTorch 2.11.0 (#40669) Signed-off-by: Dmitry Tokarev --- .../image_build/image_build_torch_nightly.sh | 2 +- .buildkite/release-pipeline.yaml | 12 ++++++------ docker/Dockerfile | 2 +- docker/versions.json | 6 +++--- .../dockerfile-stages-dependency.png | Bin 322215 -> 322797 bytes .../installation/gpu.cuda.inc.md | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.buildkite/image_build/image_build_torch_nightly.sh b/.buildkite/image_build/image_build_torch_nightly.sh index a23c658d46b..cbd08aa7bd0 100755 --- a/.buildkite/image_build/image_build_torch_nightly.sh +++ b/.buildkite/image_build/image_build_torch_nightly.sh @@ -46,7 +46,7 @@ echo "Image not found, proceeding with build..." # --- CUDA 13.0 for nightly builds --- # Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9 -NIGHTLY_CUDA_VERSION="13.0.0" +NIGHTLY_CUDA_VERSION="13.0.2" NIGHTLY_BUILD_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-devel-ubuntu22.04" NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04" diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index ee41ae2868e..8fce1568017 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -37,7 +37,7 @@ steps: agents: queue: arm64_cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" @@ -76,7 +76,7 @@ steps: agents: queue: cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh manylinux_2_35" @@ -121,7 +121,7 @@ steps: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" # re-tag to default image tag and push, just in case arm64 build fails - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" @@ -134,7 +134,7 @@ steps: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" - label: "Build release image - x86_64 - CUDA 12.9" @@ -167,7 +167,7 @@ steps: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" @@ -179,7 +179,7 @@ steps: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.1 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.1-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg UBUNTU_VERSION=24.04 --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg INSTALL_KV_CONNECTORS=true --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 --target vllm-openai --progress plain -f docker/Dockerfile ." - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04" diff --git a/docker/Dockerfile b/docker/Dockerfile index 258754b777d..7b59eba20f1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -22,7 +22,7 @@ # docker buildx bake -f docker/docker-bake.hcl -f docker/versions.json # ============================================================================= -ARG CUDA_VERSION=13.0.0 +ARG CUDA_VERSION=13.0.2 ARG PYTHON_VERSION=3.12 ARG UBUNTU_VERSION=22.04 diff --git a/docker/versions.json b/docker/versions.json index f3d848cba10..b6b555790d2 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -2,7 +2,7 @@ "_comment": "Auto-generated from Dockerfile ARGs. Do not edit manually. Run: python tools/generate_versions_json.py", "variable": { "CUDA_VERSION": { - "default": "13.0.0" + "default": "13.0.2" }, "PYTHON_VERSION": { "default": "3.12" @@ -11,10 +11,10 @@ "default": "22.04" }, "BUILD_BASE_IMAGE": { - "default": "nvidia/cuda:13.0.0-devel-ubuntu22.04" + "default": "nvidia/cuda:13.0.2-devel-ubuntu22.04" }, "FINAL_BASE_IMAGE": { - "default": "nvidia/cuda:13.0.0-base-ubuntu22.04" + "default": "nvidia/cuda:13.0.2-base-ubuntu22.04" }, "GET_PIP_URL": { "default": "https://bootstrap.pypa.io/get-pip.py" diff --git a/docs/assets/contributing/dockerfile-stages-dependency.png b/docs/assets/contributing/dockerfile-stages-dependency.png index a6b0bf7b8fd472528a45679862947dcf4bf3104f..d9d501319595e3dd2fcb47c339605e28d76c7061 100644 GIT binary patch delta 233247 zcmZs@1yogA`#p?$)oT}m2#5hns0adzl!+Wtkdjoo5s((Pc`Xnn3_wsqx}@8pMWm5b z5s~h$|6KUi?;GD4<9Zd&*=O&ypO|w#bM1)QEq!0NTuWKftVJaa%;eQ<*6f|p*nFQm zE<8F~+HW{~d@d#}I_}Jay1L*5x2Ts6p5dumW*9yynM6lKUt&xPcNo&2F&{g2ETup{(gr|$DGZu?1@_(&FtN~cL^_EX=ghmh?!oV z8txb#8*}UEOExGOrW$=#WJ=5z%nkiDQIwzY|MR!a@8_1}+;1=PGH=PUZ+v6;U_G07 zY_{WoKkKT*n0m3`?|-+pwhoVvKOAwO=BFa&XGauPvY)iry?ghSnejfggG>E{EH(TN zYHQf{)cHMq`n2>ZFYms6`$QuCqNAg`M&0VH3YRIgou8fZG<;AHvgho>ZJps)-Q4U4 zTE()x7JgUmv203{Yf7_Bp2=%7*e4(O&Ia$7DG`xHNepMT_5a-9EW>yg-wtB;cr|jW0pS>sK^{B%&7skG;ISBt5phviSNY3M)AM;{mU7#^p=N?Mv{{ zYag_7oGR5n2TSB1)XI2yA=#)az0JLPtfyX1b_JdFoom!acBrVk zD*VCluX1w-(Ld)$zWR2&bLf5ckX6LU-o7CsFfdR#K}QTLqOfkez}~ zfnvG8ph~Wby$Th1?nH^tKCcQB@-dzF6%`6`n(0s2{djeCTCyFVW@l%YL_In;H9c)s z{Aia(wnJR)v_Xk?luoY8@bIvAL`1}Ddd`bVC+VbS@i1?0TvH`VsUYkY`MQf#7gZfmKQ}Yco+3B=vqUA$;(|)7dgUaS zRz~8@%}bB}aO|q$h||s2L5ekI*~h&9@FC7>Vsx|yK^DVrSo+L&BRS6E>#on_fzr#P zzZE@qh-onL>Il>ci-^S7c9fcZeG_h06S>RSxO4LUGQ4-^diVJ$=S8fov{Ox#opm25f>RXw%9ZF9RYS(1i85>pvwS2rqH|UV%IuZY?JRl;N zn!nKb`9*S8oZHN}zo_#YoQr|dF>)SGZcCOd>8=dD!*D=@-n^~%YjTXsSa-B)oIgqY z4I4IuOFiDbLuu*TrcNAbn(BC}D_~x?Upw#WSM`CPKf?}cXPL}S4oL?H#ob#)Qf%>c zO}#)t@t;SJHqK49duild&E(@MDk-6*n%AqQTeW2HbK_luD-zZBH|97eyH9uRo0^$n z?x~JYNDz0Mj#Lbl-23{}$;3+^R@nD7)^;!KaqMs2E@biSm;==t8AkK*f3}`(rpKR;~jF@`_tkQTMC<(RirlQVf6b0N**xy$J9R{=+?3o{Ow*TqJwCxz5* zIAgSA=lmH-Nwe?oZ*Ld2CD&jOTk-sGgwf}~sMm4<{*3GYTn7Gh5-wsi92^-rpYJ{w zD=tg%TzC_P1c*RY6R}+k!u1=On4(awv?XQ-Z{E0Z!(&X&U!d~s^<{pd&bC{s->u!W zY18&Y7q9J=^AC%#=4)wg7SQ|nx7%2)+N=WI@ul>eHobBl`4vSi%&-3Y??ZKty$yUg?$-!i(7wJ6^tYfyw`y!nkjMWQIg=oQ>2zeLA)wx0D^ zSfM8~>bjE8Xxu!R9$LzXl3x>|760>T{aeq zD`<_VFu35D`MDV$=i#3ZSyq!{(+=w7L|W%fNg{E**)y}U%--Kx6|I%|CiC65Ln}Az zP)4@f&x~8rl;X8AQ?7I=sZc0JKMm-vsoDSf#N;;kZsjre*^ID#KK^8p9oyXM@k_i0 zdD+<5awl3{SFc^`%f9uXmU`J!-o@frLPwX-O#sKYn;U+1cJ8A|2aD&{#cRjlmOKw! zyhG#wbdTuS&M=X``wv*PHU)e}UhVO+rV|yiP!vske_bp6r3Vn3pei z(v$#IQr+iV3`@KhPqLl_Vya2yNzHUCKlaRw43myhKa^p9;l(3MAua1%2Xc+IpCIrqIBpA(8ATvKp=nmKi#!k7h{vZF2V-(_3`oX z{>SX2yX)e3XfCspNh{W~M1HulAs}$=_>~_YS2;U7Gc9K#Q3_DR$5wG0#?mO+t=86?GUR%Z>Hb6)zFr|;R9lGv_E@* zd*{yM-ZhDrPLbnF`|R~3-#-5H$Bz(1H9;=kVPC$S!!pa}<>mc}#lf^=l=E;Hy609F zmi;tx3scN$xx;0jSCQN9D(%PZR~LIB6qQEG8hKVe5%QR}HsP(#L%T)FH^F37A#N?csr zI}Dd;wlF^@{o+{apLJh6zriQlWfaMw=+_}I(w8&dm+&66QGQL-OhEuY8ojfudtL--e_tG ztRfEQIO*wHmre|{sd;V}yfQm!LsJS6vP?0l-0gGzKdJlefZ!8r1yTeq-=*hLI`?ST z<4My0JZ`MDva-_1wB=aNMEC8$_wMfKjzIX1dK;4VMl8k02GgD$wFyyt zv8VSXO>Hxn}aZmhR_MpqKQFl^lWY(IM@8cWn-7 z-!DKN95fl~$$mwwy&T?u*ILA(_uS1D>(D`VF)}i;iaMP`u@?|wAV)dfi&oB!>I+=+ z6QB)fJ_>!+sAwtO$7I~^Xkgzr>icc&?Q4_tiB4qEQz9{Giu6pO($>+WCxUj>YAMzA$j&Si5!7__p4X0xRt{00@`weC0VxR zX(My^aSz@a5GLTWSB{-_d-c{TfL35+{{ItReE(f!<99cfW06r=&Y?o>q!9q7k!r?4 z%d%LIgkSyp#qO^EqCKmiInxd&voqDQNfik6;opBws1n%9^Tt|a zWysEDbZ?~}?>;k%`Y7Yw-;y1H!bm+2{F|JCUJ|-tOP-siH((1Jsp=QQPu^Z_Ru}u6 zy$g&(BiYEa=rICRr&<;&dn9_Z)hN zuIr1x_;mriCnO|1SuYDPuGh-DdSQNk-kS=0V_3!wfV+OppS3FjNAu`F`+iQ= z?hDT1G~&xU){BjNdXPKFO}8?|^gLOz%~{4?TLPc{C-;wEP0Zx{5G3a6(SfiNW+z?h zTDlBM$U`=XcmRsz(Bn$#>d(@w+YZtI8lN_|v`}*Lf6+-5zcH?e$C}D`6LAB#r0C~q z@^!7sjsvZ=z*rnKLVgJ}UVQs~M&9_P%U4TzU%h&jo`uEYe-@G5L5R`tsLhXCNm@}# z9CvP!Iy5-0o@pBnJP`v{++F;5k5SR}pqzm<>(=>E?C3NV8Pmdabi7qBPb{N(7wA=j z7|Y6&KJiV?{`Z$%U>4He-Sr99w%h6627K#sjv5@iOz!nh0dkUyFR5OOgQb_pdoR$G z-WZhl^6HA8?O6QkCztK)4##U<>2DVHsXp#q`;-Ax+gxumjue8(pbFn0+3wNEL2^ot z@}dp!e3n@(p5#Y6#lI^1?gFAA_bLpf3UtL_J(?tbHN*p!nsoL1`{i4FzC}SGVlKlX z@$;?1X@BqUx7yDp;_&GsJ2_0N0l7^dw|0h;e=WW?23v5KZrn(1e0lfo-!!ElQD>Iz zBop5@?NSI5Ma+A%w`ACS_$lxxj7KegTiKcW^n8nlR*XEitKDh7*JiPu{J1fqM0YL2 zW!&kVGuOWkO-FasU+xUk4pFCz`y*&Rg_kaoU*w2+v(t8~sj1QG281sLp=~ggbyZUl zM;4Fz7!luo^*frP+`m?SY&6IB!Gi}jpEuH7drEtH@P*=jUwqfUUzbykG8$rS?bNrC zA6Y*I29ycmJaux@wr!kUC4Xt=&90|QQ~|5u80v5&Z%OFKox9Ri^*rz|y1~~{{sK`- z;nEbxz5hLEPFUI%5X{F(fxFMFzH|FsnJ3d>NnHQceoMQ~&rlA+e+X~t4Tr3kb#=dfe%kwIcek0AHnsY)FUlAfH9uRC8waA)n52JpcE!@c(jfOKv7<*De*U?E z$_f=SVq&^~H&mlckAks?Po#bSaL4@P-AzNxoI8M51~P*)r4hYJW!nnwZzU93FBW$) z6L_JQf5qT#K%CX<*Z)H4`}d3G^Xm|5?fCRghP!e=OC&^TRlNDLAw`Er`e@HqpDBPJu)vJ zbv{D%A&W5S__a~z@09e>$pQ?n>ub(DU-Xc*I@7Kb3a2G#URgoG8S-W<_|N|Scob$X z>(-oA4j%QJSXg3lZKlY%ESIt9{^rb3#0i0R(MsX>b8>Ph-+c-@vc0kmN;bPqv^Yv; zTeW2QAG&m-rO26S@BA{l!C&o8L_Bz_<3#-@EAj8my94g?@3bO?4JfP&eRF#7!-m>a zbG7VV$1g8WY!@(j#BK`8-u*Q>qP$7+%o$Tq!N$Z(HvyhQ7v?AOfn0;NZKch9uf8Z?ZEJZNa6jWbZ`s##U%9CpFaK3Zl)#IRgKhOU`hhGAR9`Z`e!ZS;eA0|@tbGb zcZ+*$<9na+rsCbrm16~ry3d$@Z{>K>wyX|XS*mIEUg!<7P>r3ef>CFH7k@)_B7M=Q zA_y$aj52+DZAodMu-%PqA9kRc3z*e#w&YW7W^>h@zkT*8nBkU9|MXE9#i2P(m1c5o zLAS|ztq^SW)NRSzm|`rV%BEsyo;?*33j%!q0!o`hQRt)fiylgf@X;@1Ky}1BBJvmJ zHKBoid4GEiqAz;e9CRYeV87QkzLSu>Q(T`wuVOA+wqorUXb^qki>&2Ot~~P$S-bua zPHk#xist?6)ALY#J`%O$wNYc9TlN!P-Gj7a_T9cjU|~9J^vs_ttFI}YxeUmf5SM$V z7dJr61{n0!-lzIP+)V>_GEqY<-AbDxA4Tt{bD8!^W zI;KPLq$_fi$RnyfrB4pTpvaW8v}lunJAInE1*98|n`AQdqhR%_Rh3gCok+HQ;61WH z1xe<6>743-;RIbqO+x#i4?;O9tFAr|=%#{G9v%FCiX<$SHe1M|VJASf#KP?PvkzG+ zfx%RObgGM-w1I3Pa`|{%WxG0fT_epRN-NX03o(pZN8KT-2_RekLPQNDv(A_6ye~V= zyMBl3=j)tfxioY#9!arvM(Bmq}o#*M%x2@ z*V)M_61RB!(aSFpLG;_UMWO(kV-6t=QR)WMo_LaFhNbkOVpk_G$51zpLhw>U`B%pH z!M>~N%tU`nwJtRm4t>z%+uQ32g5%LDF`Gvh7TY~pX{V&$ZBuve-aW*%>c0!0!y0JW z<%_0CBr^~=WdsMyi!RS22E`jY$^+C;+LZ3BXH|~Z5>DPj*bU)hm^Dx;nB|Q1HP!b@ zsV-f*^k+{G4~@_xM2InjXy`G!X@nH=*)(nv6&0mwkfWVO&e6xO zFWoMFHKXz&$}7^}FiD-|$r`$#ZnO&zy;YS&k3gIpX(Y>o(B4wB&4vgXeENmk3UZLy z1}cxfUqJGx;XZqg-`~n}1#=;_9W%iBM9gB3x;izSki*u{M*8=M4HCbRNJeMDzyG}0 zFGBY|kAmLO-ObI*%RAOxTh=S}-}QYI2_U${2F;R*L>`&+xs3OkIq6oc2Ns6xEQPi~ID z+gkCflZh?KEMhLo__*XuwSt7;{ez+kD8aQEHu|Ar)cjRaoBK1_qtH}^#$q&5qgONR zH%Z}z?1>t6jC~88s2@9Rb3kxKQ@S;;aVMplXVsih`I<+fplF-T7_~4#W!ZH;tn9A} zFO!*Zb#bA%F$$1$1x!m29Vb(jZ{IE&eh$2V2pr=XSUK+4UBflj`*ks-uqJ}`-O4cw zTmHQ+>|Czny`%!~5WmW4ZEcOQSl$0!HAUYgtKOp$dUl{bgC~tY61Fpv3lhB$D(R_& z7^PVFC4ox`#t4}HRs8rvv>O?Gc4F1;CG|qQ!|rspvr|fuZ6 z;0{Q0rT}(Tnp$t~Chgk1{$S`F09=DWNh&f;8tAdI75Zokf5+z+n&g9EMn6VpjGzZc zm)B-4IVwH!tK9zE+a(8da*oP36?8arix)a=g}_NB3YGDg&d4>L8Ayjvd(@K|AO-5O z&Z0;PMTSSd=>rReMg}P@92G1)_Chl0*+g2$%<$90|7pPgzS5kn76Fxk*R-;((AF^Q z|B~l6bEWe$D};7#Tqg-=@#)`t@K1l%p;WDG2RQ(}O3VeX05+oKN}-x7U0=Gq2GNWr zcNQpydQeZ`$rnP$QF|bZ03LCnMw0;;viSAtt=z?BWj%fi;{+2qIc2a%L6;Ef>%>3^ z{R=hiSQuiRhm%u^)HqCGNFmS6&W?fnf_B@)V`zH|@S?b>si`;+xj~|vOjfI3%c;a` zQJM9P3qXLW&Lc*w$F4}BEmVU9b)d!zm{xJ>=3NCq_Gon-5WBM=$8S*l?aN&LeNd4L zsf0$>&#^%y+_h_$WcKNMn}sZ&;`mz+Xei?bc!yyryFi_Vbjw>dK{8cos(a6vP+IA= zri)hT0TsinK3@A6g-HtPE;P;I{kqi5pBE z^Qy=+%O-BMcrBYo-ai37pGw*HsFLfz$AF?UaO=+huf){9L- z$ZE~%)m30~_E5J@*=DC7cDmf9yeq|s?tVV|V#F#RI(;lp)vo+0eK!=0o{WCND z%$YM?UJ~BFETK}Dtkvs8V&w*^Tg43*{tDvA$Ym{LaK_J}*SVcivx@OrTW z)e&B>#Z+wTvFj)eM<*{DZlXcw-7ew~OVuUlvbCq7>uBdVDI!)>pzsP@e0xoOg0ThP zINFntZ-$o6xA(EM?)!V#T#XsxeVy~ z5zIE)6KuJ^Pl1lK&HmLUCB)4L{L5q6On#stTl=_Y>Cyjp183O$&b2fP(N2h6>Y0rP{WE? zS35vbR~zsB>e(3v`MMh+Dzw$J0e%7m$6ng>uK;UbmB0EI*2acUq?EY> z7ds72d5lcFfF{f;LvE9H0@XNxbzLGeR-9nj!Ef&*Pxwb1)Xj^#62tE9eO zE!n7C2}q_kT!s-?MFu%~*S-ygqH_o!d(eSx>99qVwYBMDWoj|A9SdHdX3qkNxO{hY z`;)*QA!o+o+RV|L_^imB`*WahC}dw6LcS7UG+6Vp-H(s=c=zAU!P#^MWm^JBgY9h5 zcSCmC!^gF)$GSP=6>gw}qX$rc*hF%C{TO%}QdK1%{uhs?rOTE<9OfzyIHEuWUmd<@ z^y%59iJ||>8#`Y)=^GgiKxaYfe__s8Nt|6Vw9#Bi2D#4t>!HdDEq@oQcK@(SfNiqF@~b zu+MSi$Ppq+plFjiOU4$+N5VkKKo2=#xuX-&(1^iJ*uE{g=;>4M^N@MO!r`$h5ml?! zIM{73xDWB7p&Jr>ubQ9}hdM}eQA+#`jIcQP;2b+-v^!vkcke4_i^qDr!R3M##V5e*(x% z#iQ9YxOq%EX-QQj05=WMg+mz|NLVHJCMuZx9b3=1`fTQ(!B}xy<{2D`}<2D z?fSrcr116{#@I%vj%m(K`!BrN2H)47JskrB3(ZS;E3&ACJVo}`m@2$xuyqbH)-C&1 z3k~2*^=nG5I>oeFImgKcf|fM+kDa;=$Id(tI1-kvT(692D6uMl(-zm(s&rNa*TZ-s z;5=-A#@dbGQpP7z0d$_`4fNDR`3xN^D=(Kr6{=pzcJ(EZ%elFw0eXGY)6@6L2S%bx znc%lX`y=x+qQB=}&WFCGf_x>djHaN|G>+6ovYy8{dMn0a7+RVjtEQpoZUvb}Ih9AH zP+k5r$oSJUzU`t;iIwgPa}#JQykrnW^JcHaDlFoYXMCq{AGRMh@<0^4hG;D2X#1~5 z9Sg@{Dk>%gQ=u~2``B0G;3)MZz59qBOhyH9iE%o)8W_4kPe~>k4QzjTTW|k!>f}k- z0JvZW(bq-?9IaK;je>{8JBWT30%{s*2*{-<;4`Cw#fi?JFT5jt=nNBqAjstHeE_!H zSeYsx9x7Vl*`X7t@Wf0~xesbX_m@{UF(FKNyyvX6i*4#BXrocITEV}fW^RZF|7aqT63 zO+1)n1X`V^7E2MS69~m$*zUDllNk9l>3|(Se;UNfz{!Fd{TfBg6aard}K|HgnuERl$d>bslv z_yWlWOf1Br$eN?-LRoo{bP&P>${txqEK~%W^R@GDl$(8SB(UY z{EY^nkxH5Q@g=35dvy}`<{8FXRueYC@(WIFH1c{sPAGh3wu2%7Ij zsm%7;!shh=WKCnM!EYSfgO*POL^w5xgaaw+?BmCe34R7nj|*aEahr9j!!R;#H9zA0 z48lmr$`PG2s1qt`JOve9+ME0hiIta<=oyMP6_`yqtCifdA{guqs4vjySZO=j-2}1Y zFzi=<2@tovM_WsaSW6+RoDy>#uK_n?rxB+cITK{F(}EbwoW@yiV%`N-_&6zsYDSQU zIz`-e%v1~}2~@y*1j?2eis}*K2~$Q4$YrYk z4;BA-=V^tq)J}k-2y`BGIi;&l0Ca2rN`;7=b6FP(X(Ln7n*TN-E$_22zdjZ~#TplNF`N8G_S} zmQM76wj&cxtL0sFb{j6)M@hP3VTHD?CnI*ugGfNnVe}#-w}bx_j9#R65UBv%{0s02 z*6IL}&rly=L3(^@HNO=9>Ib4*pO&O@DTO7PLG)?jsE$#G+A}2a_?W#J)cddf`X7th ze?afsF5#Y2iJ?P^_`>W6<2Vp`7Nv6C8(Ml7kR2aE^+}dZLg@C)eb5!uw;J4y2UJvD zs0Qq)!+#c~zR#L-oX@vi9q~l`7{h}1yU98~)g>0}uF$N??c~LZ`eXOLLKpThbaZrt zVQa#Zx!wLDlhUVl$XZKG5G~QDXb}iL>(F$c{w!^C=f2Yun}-_Ugp6D(Xr9hV@4McFDgrJ8n4-%bN7H-&^{naxR)ut*twwG;QrR!ndW<) z6=X;^Khw6*4XGAZ&l*4&%uc7(N==CQy6RLFzY3Dcp+ zlMYbA=CEXvGZG6^UeJmzPT2ftw;4i!a6vXSff0Eqd`mlf6xOS7tKR{J2MoMA6icXA zs25v>{Hz_u5Ww2HS-Qr?#xdgX@IZ*uk9FQ0Ky*3GfqDCU`1B=-`?3$9#RTtt_>f`W z_|-TcbuyAgo%LqtPzch3fzop<>BYvPyYr&KqRzudE!lDx%I@=vIt?C&g7-NG80{73XniWA${Y1xqYE?OJ(P(9rS zp@?%LicEIL-FOOPfn%4yT`Ql1HetB`4hE#?2AB=E$)pQo8wVseb{f1Fib&7kCTmnR z)jj~K2Hz-{*Q9~@xztW=ShubgrP9LzeFg)TO!agg<;X`c++7lk+9za_862|UsSqq4 z2X2;=xPa^QEGjt<(KFG%IgmOhI#(qF>8`&m9~&g(O(8v~O2Cm>1DlK=wDRnSa~4bJ zXwpyiKgSrGw{)YM?obCpN^q|m@u`6rJFx8$-@ypXK@7j14lwIka)mhqD(au&ocR zq)A{Ge_q2(s=VG72oae6Gxb58;DMB4-MmfdQ|D9|t$o~9beiv%VUZ_x6m_9v!((>) zpY^QUBWyk;qFx`(ji-6ATsYkw1y*0u)TG(k+Di6E5DTf0CDj;-{7RB{U-={8Kmsip zJ|uwS?Owq|^gyo|9vr+|Iq|hVB#l!&@g>x)T{OwcpiYQSaYL;Rm<~a+i9G6O0pRIiyTb<8B zRr5&Ih0;ZQg<|fr>Qz2^qc{x(-%b@aU`Eti=01qRWV)AR1$xZ|rAYxS%=Y5q&W}Hp z?@$r5qVFqAvy#S3e7Ss7W&kpH10$d9IlEpZP6Hba`n=hRs}cFlo5MXzVL8Fl*xI?f z5_o`DF@hCp{QUVdA)j(hi0B3@;oe=-A3Gp%6lnNf0_`re1uP40+Q-kYggGzY;=3_{ zmV55KPaz%^ujzH(xB~{)F?t(elzih{W%%mkZ8+O@9rWHS$Bp8me{u?7L#Htx+o&kj zx!`!2VwkIaVG4oK{0xv|)g3nXNN7Eruv8PC4-6vc*sn!w=vMlY!9Pbw9~@1)+j<0f z;ZQ=)h>)$uednNoj1oR}s_>pyQ2|OsY&@qH8SE5sLVs2C{oXZ_40Nu-c9vCQJp|uc zGXGDJdyc_rC~6$3xu8*UAn>xjy7MlZG`#`VzUE zIA76}YHmF7W6uy@;rU80CYu90w&pH8c*sn8l;YbMELka7RA3?BP4G`SwTwG+e=T2b z3NlH=T-d}y>f$KP0M(tV?;Zp?70Nf_#pGH=(FbT+G@bL5me>r7E3Th$+`KU*>Q_ZD zZx>DRF7RX&W|~m%UsOT$?zGvkLo^a*oNa{FHEK=9!ImJY_z$#b7>Ex^?)sP1Jj-)1 zxQy60pgef9i=h^y0^A*;3*gt!@M6;K?gxU^(L{`zD!E;mKR6@UpbtLi|X-S@-~JI~KL$2g## zS+v$YS(xY~avCA=72-3pplQmeh5F43XveC{8T7J)$(|PI{5;U9iThrBeo`L=DpyH7 z*%wx$Hn)ivd_&W?#bhw9U3A&rKGjTOCWCrd4qT;^#e231TPwkX4tACJUE8M+WHB;w zPOAX8?d!=S-ozvnP*nVkFH~g)D6pch^KSc{T-!UhZ!;gxWQ zq1i_EUb-CSAjHZ9mGo$@J>5n%4C&J|t&vEvjS3Go=M{Q^d6R(UJ*4+Q(>39OL zlkKlhQBav_-o*HgkC%S^nk8R=Q~KH7zNf@xcf8nND^_Tvw%exmOg_uVwWB)a$?g;A z+;>OPlIG?rf+Z3#^OSZOel`0b{)+p;yj#q8fbE&V$sB7kWrjA=^Sd(CrcwU>oV#}z z>Sz>4QDCzZ0U5TAD$=U=uM%J#M9YT{qXW%Hp5cRkBan`-W@KURt=A&6Nyv=@wM zbusuzp)vAxaWC{@W8-aKmUOSoyZ~)NJ$2dAr9n-+hYp>C z>9`6=vi&NH+l2+9SwN5OZpltZo>F@B@8Pt04D-%QB$bpmwo|?Y%ZdT-Ak)h=6O6f+ zYfv4rNiga#QmEf>$JhKGvcOz1y^<1eCw_ zjiC(WOt~tbd#lNq2)3`-X|iJikimuOeBnrj@7YFo-NVZ>wEXHSeR^_oa}n5_`usFp z-5Betl;iE9bXk#4bDb$H4Tn}u^CmJJ;<#yUfS!%gKm8Q#o-{sS2k+6JsRY-T zFP8)>)vtZ%-h9+)@Fngq2d%p~GkF+uk~=j+p-}gD_t%W2k?F{XB2M`I$iRbeAvFH= zY#W?ZY@Ed(&a+Iz9ly;D<5J}ac~(~Os}1U%*n>Luw-&$AcI|U-2hLr`?5haDAm4*EtZy> za;-<*gx^ELs!6xjC4)4?nZLMO)`vCxQbQvdcjp5up`H*j(cj)MGr37@f*dnca-GQD zf9OZ3gVPJ_n;+utGO7KQfUO=tvW63Ss{jlh&~qyncTW-Jq7?g_<(kB^u{Dt9d&F8N zrpd9NYBtp>cZ1aI4{jCs2GmRr9gVM!?dav5@-rvid*gj%CnqUO=tLhYY)8l0LkDFM!P%+8LKmV2pDw$?Lb8 zRxie)dnx%qlZn_rH|Z|3=I2u!D=Mi3fCnG%KR^7o@HKlRgS&Koq1}3Y#a1)^VgYvnaaPcjJo&4)zqX|n_W2Qtob#%CEU-{>Vm(7lFZNrYwf}Q@)C#Xw5*9IQ0?Ud*x>`xL&@WEpPX2fxEhpy( zu~d9^@DvmrMi&VbI_^=rma56jjB&v|TwEpn`t*`Ex)Rbn>0$`v2+$cns&S~-O}kv? z(tTGg*L9iW_OCeaPm5Y@Ef{jcG~jkCwctim0>Bjn<>@gxn9ua^aWJ$G6UZ=(?8dUAnrUwQV7+hf1Fq985Ib z$H8HW{ZEFHO=V>=pr+hP;;uCby1JNu6ZLzvh>bEqw<#R(@+N-sZj@nSTC_~dPTTk^ z?pXWc{w2EO7tRF^oZ5QE#lgiyh9q~`ZVZ>$8zk1;YRAmR248`l9na-}*x3Eq<8FtK z!WHpqTGzEd{)Jl_XX#Ul*ZT0=aR_y3<`;+I_prF{aRrE=fnU1^E>O~IyuH2uwTV|- zjeS=HTo_!Sr3W-pIMddK57|{>$1eXgT=?YxuU^#&=Y)X+C&bl6O z7RytcY^*>=iDYvpo}E&IIi;|$@N?F6oZbMvRHIl-D;y>kji^>{OwXP@+mzDx;@u(R zU4c0t3kyr%y*mLCV8SE(%j)jFqdNlVB zHoT^JJ@>Xx`;Q+M_xUq2GrM#brq$xz>=;I$#xGb%96^0e;W=yi_uqdfQe0*(n3QMV zca#~YB>snB7}+&38a^^OY$7Bst^-e5C|{W~eB{a)({C1W02sGy%%2^8pxp$dpUwV5 zD$D(HzSv==Teok2O&xglY#Z4a+%7oC5S`luX(iKH zE%p5jV#CFIBt`b_m7Jj_2GY%&CMTVUw(DJk9cU;nj2Ut^+7d)xKL70!D!sI{^c!>R zTbpn!lt+x&P?z^p-V@(rySf1AFsBl`{qouwtJ-yb&*!Zx7Hx(2&- zqL;tDUB^nE?gG(1?aiAvFj&W6eDp1AeskL(5TJ_}FUWwVO;jRauHFD;2R%>QE!#fBhUK=JOPHtKFp% z79@xR74dz4Dv6aEQGT%M*XM}iqEAx1V!#q4Q1kiyS?AwEo8nBN*tD zG8Ojh+Vw7eMgU;<&yM{+hxzv844&4oL1H$s{N)D#cnv(G;E93Amn`-xJ$2gZ?&U<>T*< z36m-NliBHMTPvv5(1q=)+&o9kB;LYQ_8~{ZVHVgK)>gFw5usvG<)mBGAd@2l_z}K4UDqSZImxPLz zRv-&2tDy1cKs{b5#cdpOcGNB!JQi=ByTQ}Tf!cvi&n+zcwc(B!>K7j&VZt071@Qx( zwPv+v+1%K>O26eUS+;_{`&mIBoWd!lY{IteZSxpN$vyUh5<$b(up@2a*l|(l{5TBw zvs3f!dyh`V&9>`@Q#9i0%P$hEDfTL%5}E+5Qkngspu&a2AVec`7iQt%S}o`#h{bLa z&@WtjAx;(Jw(gr7#Idi=@zsR>I3GxB$ekT7QV=xKdlo)IqlKnnngW+cp!EXA3Qbq3 zB2Q+=_DeSBu^-;7EiacyLGZD3qb}Byw`1d-gTTv=KkeO0O#!Q>gz6(0b)3FuiDScy zA|y+XaRdWBBjeKn_YEvZFFIaIw18#E{<<618)A)K%0Omkkk*V`;L*xoovy%04u;Q= zT|+_dF{A2h{4*>4=J4e4<5!rTbu{hc;gPY<9eYygaOFykU4__F;{_y3)33S6Hg{^~ zK-(NIeBtMU?>zYjR;$Wib}cZIGiH=SWY_)`^gpDUp7B2a@6z5r)L?-|Vp@j&+tJ=0 z3d5A&?97k00QA=fmfdZLuK;%@3cvLAmtnzJHDGfp*ksF1nv)2|A#39a}7Hat;k z^zE_@4F+ZALZiS;GP>?lhObs!7P4w?FkOpJ&Uq4Qi@mwEmjBxl$fQOK{i|dnHvu!b0)xA=;q@)>zMcI z>b8ld6L%GM(ZqrPaPHc50%=zZTD1BT1P^xoy0LHHzV)oQYbYZvU4{ML9t$7HelL*V zHO&h|Q${Lt^YJM{0wvB-&)rKisZ8va0{sg8{o4fS!~}mu$hciE;-VotcF|f}TL}%U zY|W?g0nnZetUHe3R}FABhc?r;3Cxa~>|=lAxWiic*{JY2yeMX0cJV$A7Cz$BGmI1J@GlRed_#NZ}siT+9A8(gLPsqi4VR|SY8%jwLAbLqu zR8(OWF&>tba8ByOy(`7m5JvN|Q4bklhHz^S@87 z1VXHW`@=9*qvO}F*NgSO`Haf;BHZ;mB!5tG&zwGOf|(MvoMGg$gF_wG)FLNuaq_+V zmMs}W0)<+J&Nb+JMS<29O#F53wV-vY7IEDpJjd{Sh8|d!Qn`LFX^uusA=W6|aufuP&x&LXt2Ja{iU&jsdC*EH|DGCr^^K)~8>aqiSIh3x2Lc{r*G_$wtjSwhkr_ zen6-34ukCZbIS1=o;4t6lZ5ah=DOku$3|GFy5JIj%uei=3Swi*mxBN7K7VUjgtmet zQ_Tx5BZ~(?T%eIRchs9eIO7Gt&97HmC&?TcU@CJQ#UlYemWw0QSL(u-kTYw@{M-cI zl$O_r3R1}ck!<}Jn(+TGtbH4ID`f40E(9RU=vf)1U~xC;l*tomkQB{-e|aV83QRyY z5b?PhYiRi2xOsC7js-Iy@%vt0^=Pm%`xSt&M&aaT#x^~B|jp|qS&}cL=P~*_fT=fb)>UL=gTC~38H1ew(t*{DGHfhNo9%~B<3(e3> zT&BAsJl`)NV&5JwQSVN7I2o-1+YwE~`i(sbY6y_3>guh~o3W6laIbY&QlWPU|H(cq zqh@5f4$rGPZ9gjiE^kt;{aI&Z?jy%P=TF$ zOV?-2tp=9!W4@j8)#9HN;pWGn~obnF)^huVuDH3vqiQCY8c7g0IBhKtAzt53} z%2oMYAE7v$Klc&~JJ~pBf^oSLctkxarLV7VSWS&GtAx82j{SOZ{h>A%AI|Z@-9*M$ zomUVeb5c%hChzL%y6@>phQI=lx3RIY?KK6UT-rRsxeB(N6fWt}cPrV*6SlaxxEfK) z_)Mw`3OFim9H6{JhwsAo;xUe}drc091!7`iy8rND9h5i$>~th%I&4l$7HPwj;x)L{ z_S=y^+S|vlE4Oj(n4jIt=;uX`;wiV!rKN9rT)w_KwNeZbBk2l#l=yx@w;>KG!(Y(_ zFMZwN3eixMADWugjX>;+qqH-v7=X*Nz-3}!(|*Pp{wbsVp$OTbIEV0WOt%iU5&O$( zA4+!}ELqW{wIK!SaIN&r%rU;c^mr13G7&8h^m;jk^)Ty@ZXJ|=6I)8_e-#o88;u(} z4I;Lcl{F3t1J65g!PdgkLB(K_*P;25>&U%(_sC3*Y_lX=SXQlCRr=|Zz-nf(p^saL zJF$|2)6bbf(mM?J=QoMKp+Hh2p+iVRsZ@moLy~LDmQb(@GNB{YAXYVoh^sgS+hu>A zIC+f~6>yKPt}b#~Zp&Uu_6w#7ez@7UZv8bk-IGAFiwuuP!R4|+>{PzFlm6O1_Z*ih zHu>!)c?Kc|mtiE&gzY3@00ICi3Vz z`9RLD(rcY4UH~fCXTF-{8W#-+vliAAG{(Wa#hsY10AgdHxT#*uho{*L>S?BU8lIa# z1Wlo2XWg=~;ip!U?P!f9FW+{$0lO1G%p>1w)sqf}WsE^}>1e9JCLTCLwu`x3 zxxOqEs$Bwvyk9e-=sJ{*9Q?1O>>}E2`;xJ*7%XHUYYd5k1}-0ztZPiBuVlMA?pPQ4 zUPY>cRVNq5DrDn~g@r}l%mmDq5CzA8V16CjV*w#QKH&B^(GTFwE4PRxmk14KSga0P znxcp!piMk;3fQ4GggW!!>I{qcRdr%NM=wd>349a`*xUoV@sSldo%`@aFWjxPuL}T< z#uFFYi7&xVO{Wqiaip&v1K?a#Ft`*F>UkPj{ZfGcjPI3BWGltJ-bA!##j|spk1p zN$UFxd-QC$yHMZo_}DrG$?$$1>M=3y2!JtUSDckUlj*L-1GI__67|^NWA-T|kP$P* zbvs1WQ8Yi>e`$eaKZdkqbovxnUl;NQC)D0L|Kt+iM?>t!H)))h=BttbxLs?@P#*P)M@^PrXl zfh_bY02t(;=-J__R0^8vTNArF&TB-v#cDn*?T zpZdAQlT}0&ZNPclxj(aWu_LpIYw5a;sl>ZMkPV^cdUVNCOfiwlnhoa|^c29$MhQ(ZyPuc+VfACTqiEqp{g zq=$i|xBfXx;B+ge<9(hajc~PLWJ-?bDBvk7DrRQUc%&BD_#TC-N4AG!V_htmk*`w_ z%*K%*2E-4GZC?tg_B;Un&}HzbLM3#H8VGT7Fxl0i@=|M>+mLpi9n3E}Ug1$>8aYl$ zK28X#CE{#Yyc$5%y86F6Kcv41^tI+qN9j>wy&30ZG%@G8{luXS`PDa9kM7+1~ zJUX0m=7_>WC-K)Ou;RvGwrT=i0jr=d9xMfYq(7_YAj&y)6LUSg>&w|~k+jSN6%fG> zdy2>gor0pG80g_}mDdBg#vv%WF>6+ZPi}SUKz4tY44wq>hS&wU1qD^S#HUU2mO09t z!hve>geH!C`+DGqt%qhBMT!RYA_I!5NlV}7R*mJvb9it;J^fVnfGSE)4IprKem-G4 z$Y(q{HmspRo%ouGh9RPlUNMTa<3`aV@E#a|JmnObP=eV3;I<0%xLSC^Pl`cj zp(LfFRL0ObIw(;IrBXQ*9d!C%_u6~o`}zI*d|uN_t-aQ>p67n9`?|0DdR$y6bNk$U z>C{`~ZjVtWaEC{x&YCrayOU`Zl3p9~I4a#Msn(ZlztxOLXAERS8siq?wA@=*Bx*4T zr?H6L^k$0np6DPgmAhe==W)8Kxw zl2qv!2I^EKbbNZ<;_bivK53jqYlwZsMrw0GY-$1hPd62K=FDscbuB1!3CZJAbzd!* zEQK38G}zm8psU(G*}cV(b3)nw@gf2}watPwyN=2OKK6yPfz~VBKk`wMR>2DBj4KdVgF&Nn?ztzGL})z8Nq9t3~`Bo)Y18zS6NM5;)@_8 zY>U)jH)i)w-#23}nT4oH=BcfaJ@}ZjnvyW?T3=bMYI)xRggYCJI*gO=Xx`I>Dk_`H zp0twX7*Aqz*NaIXKYlDIC{R)xkL#TVxeB-|$b=u-SBqHI-elv^uo2OT&WS{^`CeQr z5NyOg!1q%MoUG=&<%FLfXMSdP23aUrd_fHyz~JHb&~`}cOh`z+7VJvyH1y9n(*4Z^ z&7iDJwxAxGC`1=3@=aH=Su3z7PQgVArJ@tJ+TOosoVWjE6KkEFb;}$3JsICQO(BKf zt*}mYj(zy^9vHeMi5IX%o;MS8fN^)wSWEp-rTV`yoRJzbeXgP%`eTD&^5IY+sbWp zAE4Hr-?lbYEB1*jn#|C(&0Ir|CN?h2hsGlRkvy6Ew0Na7BF;{@ik^CLVeA6}``^;@rp*cYXIYtCb`dV0CbM>6OiFN z+86X-c8TY&-&~&R{Lci(*<$knYcdUE$*XMu{k>!l8zsLVLON=X@|sXooApz zJhEa|&qY3wi;pxrPvBOB{=SssO4tB${ZF{M1L+tGW;b|0HaabB8S$7!s>67x^HF;w zb1cC6va{V+$u?K?zsoYKu-L4;8Yh!u1zJ>%QIS7Bhf~Exjr;GoaS-N(2S!9F zpm7XMA}gu1gD?;k*pCoIwJVgIn1=fLC~$zQ!2_rWW5n|uB*V0OV1c2SzY748`Vh#& z_%hiPLoON$>IuC3XL61JprZFH`pRjy_rJWXO9?|j?Lz8sAcq0K?ARw;X58VnjKE>g zgp>l#^95#RX44rUV3ZMUUD5|rsG-u(@R(D=H11$ii%C(rX2AZkGo z02(faOt59fFuC+j31Rx+Rfo^|q3^ z5sQ?SJnPtKdJTY*5q=lyxIHE6gela#$TSxU2!o>r0K*J!qy%ZmJ`iy1By4t8Ts;Fv ze)7(yp+csz6RCpG+8B}d0AM9FrQ2MUcq;^HfUL+{w&S<3MqwqryOT{Mm|z- zBMOFvyg?X*gfMno!yN?BfL#Stab9_Ox$wvVCllZ`loTQv_bSVM0+!N#O5-tM$?rdE z8IdDsgsp-#!(0IxSYl8Bq`Z>+{zYH)o+f*Artz;~;*$y5#^9Hs&ROJQah7Eb!;aC< z8H^%7M9}NEX?jo}u3hfRA84;a**_X}Qgf#4$?tdIK(v_q)P6MFsf^r*f+B$xjANnb zcK5+elujRcm-c#sz-08hndS4+1uDtPQnsW5^}6}t8j}9PiBr!|Hm~*;q}nYkN8&xs_Se-gp;Ry;A!MgJCXe! zsQ(?^&xn_FkOTspCXIsMcIZdQs}3n4UjSqP4aA)jAv@{0bN0~xhY_eY*r`pdJ49rZ z(U8G2`P}P#ggGD)x_*B#f1T!oZW$sLYEzQk`tQbOU3gNS_UpU+$zfFB-%Ke|dKbrz}B1uWd%@clzQ zVX2xlFKQ9Xuq+*Xp!K=Jh!P<_$)DgO5{l&1-j(Y-N+2O@`_o(ZWu~5I*^4Ly?M9^s zyTKCf3Mdv($pZ#66+K}@*U%vra6Qg%5x)hVZ|u5gnIn9y4>=>q0t5;yxHqC4L`4=H zkzfNDdbVO4jawi~>^w;u;PBxXcd+5in%gjMMQ_uSe{EzI;uUgr3(^kAn+#@JKtxG; zYvkwW!zS>yo&uw23|6!XNvd<46uN}~+U9~^o5ao96?ITBCS#`qOI~N13es|^6H{Xf z(nDqOG?>8Ro6TU5$c=PrD-nF4kT(J`FUG%$zg)BkUdOSTE0B^CZw&l3y%nl+O4uKs9gop{Z@zM9e^Pg)S`^d>%?aoY`L467eUp91W&MC36DBpt4oj)O>ZSJ z3Mx!@0iS?3BmTyO;8p^DX9GNdHY2;cl?95%lPq9K93?sk=%gCFO=$RGuG#P<4c*;P zt#9jWeED)?!GdA+U7#@3NVGt6Nl8iR#Aha~+Y<)Afsc39)+Mi>Upnk!LEf4gm_yXGcl>?-a|mq8)P>L*7%`1;24(0sce#mCO>Zj-fZ*Ir1Kv(Cx{n>;i0*YS{LMEOedBn;u~=J~$v24=%s zJXq#C<>BZ*yZfn3_W-yYIJ(UZFl4?x*Pq+7D!0K6&s5P%7&?it;pnp{}wVGbbmgE|ToswHN%Vh8J_*dQ*_j=R( zI`>X)dhM6!vK-?AKTwk$mA}ADlFYLJ2?Cg7$JU%leA)FQ%8f|PTEUu^LQm+{j`q&U z6udF}g~lHmBmNi`^*`{%RG9>UN}?l-IQDbH_(KEz{S*L*SO?8|IgBGm)>&b9M*s7> z?bRkw9ud4j;!M_o*NtK({JDGB&&PV0p8c;6T|@gf`2#{Iliqa$+J6Wurk%t|W2h-C zY>fUot5=*U+N-2-?<6?#Cs2!;L?6-%UUVVv$S7=0FjM_YsxAx2L1*TYsSs?SI)OpN z+K_|wJvT0~7#A0ATpskM9p=VGmMO`dZ2Qt1eLKb|u5qq&fK+tOgTKGOPQf3~Pob%j z%BDe92Lyz_vidu8Nl9pD7YEQzMNLE*UvqP8hNOW-ZJ#=l*nsl&edm6rJD0c$=KaCu zz}cX&sciuQljDw&JQoe9TCZm?Fpi??CMbOfx}anK`mj#_-M5v(XkIx*bI`}h2a$Ezl4MPs;0cP_-pzJ|~T zNhi(Wn#WP%(fwk@7WCQm>Ht_W3_>2-fac+K(s~f5A;0zkP~I5?Gcx&vL1X z9}(em|2Y=M6fcz)_|~BcJ{i-t#%;qHr;;Xb3yd;m-^l-WT5jB2GTLAZO16_{GBe{b zmnfUros2^tWmpuv^f7JdCsYUE+y#m({mz|2d?wZeEP9t}aQqiH!UjLRsksfdhVp-K2-m5a1bN^& z?xZg&3FZG?6ma(JWy}_;JEl*ufa&4yP#s)CEI)s`M3bmx)TO9u0NwIt5R8A;9fQ0L zr&}8171L=(B&xdm}=5zq6g=}0JlP;^q6PqZZTABdz@v(U6gfo zb?HbOI#x?SG(2sA0Y)JXaCIZ~@!$I+2^5NqrNuM(tLHf$Ob_y>aDE^uOOG-gff<#o{WQ>_F$l0u`6>P-;Il<2_E!x(!N9Wsh#Or9xFm~C3Udi9yE_vmbe zo&%miXa?Q@>($k>kD=^$9r&c<(|c!nu$Lux?@1b|Y||H_9+GCxq68N+oJ6CXW9)C+63 zM%IW#eNaMibC~FYw$y8cjR=T|q2)Cb0(xIAm;0xQDJ`JPo{bIMU+vE-n{J%a4wZmVx~no&hOuId(+2r0wpP z0{u$HoN&Hbn@c7aAPcEe^6As3+t)E8A55i}D*8%uge^1oeN;5EII8Kg(Wo>wM;v^& zSZ!l<2P!?+$;)fZ7}mx-7^mm1#3U^PWMUQoaVqbq`-I3yCyrv9(P6>JT}AbCEm29O z^A!|IuB=_vwdZzqJKEST`L5mwer7%3t^w}o>kw*6p(=7PVa4H-fl{E|^IgGA*{ zURqa|%9y1+Jw3F-aCLfOkXmT0ZES8fmCq$!(S_XXaXa*ua4ZA_p;JdraUGPBk}{Xi zg&WM`F$&PyRZCdsS#^E-bo(H4^KKV&9_wynw4THc(W~VCGbB1%5D&c$bQf}SQm-NO z%7R+~@UqL5bps41-@dz`oF1x!?Xt~wgnd>F^0MowaN6l)&IO~s3)u;IBisT^j3`I) z4|?2CJ{N_a{G_d`O14j6Ac}q9s)=|iVV|c^0USZD?)PONsN1rX{7A6-KIqcK<>Oa(=&CFhckE8TmKS;)24`t5>fmN532s1FLR5c2la8 zSq*J1V|xH(RK_X|5k;P&BPuY&!S>iGk)Sk6Zo+#mXXI> z2CR7CsfWikikf{Vc$F37=tn7iRNQft^6^t}&pB~;ELaJu&*1;xnt_)IKi2J9DG-+6 zMF3HLZXNvm`Sa3eh{p>mUq6#W5g+Nu!U~oD`x3pP#G8_U3~3gT9y2(M6qHcG^XKN2f|J`I^8Qu8MT<4+7QV)lR$*s9S?!EW^9IUMpZ3Gx@s1@7 zn4P585@uk?H(q^7NuT_^YUR;ubad$0NqQa>Q_s+Gf(ZR&f(Nh0UEa2mj(NRjYjdAG zN&YPnuP;Xv>N3cnRI(uCvys?>7S_ZCag^5J)&0~b{RCc@V0TshPS`3Q#H0qCTz%^n z_?AMZw)XCCO2PMbCl^4!`Qnu$Ru^P#x94q$ke@Xx(olH7tY~dIDSdlL5o+TLP$H{B zjKTSH^?tCnY(^7DATUw&#&yO3XY0=#bvUjPxcSG?0$F&E$B&!=ZRj39wf!*Q1V}SR zx!xG4qVd%$!|fsX7#AG~3pbk1o;UBIOv34;L+vJddMD5^w@y9)0hB~R2$nG2d2Sky zu4rJ*Z4wrFRydVb;g-B@t>wPj%eV)N)QJE4a!!jocnr#@pa`r$>U<3ixn;~UQ$RBS z^{p}ee#@|)ryuEb^|7_HJGYaZ+Mi17${Wn^v~)Uz1!r-w8h2R0f^CsgQK@_*2bD3f zNcxn`dFcvTT6cd-Qd3ta1rc6VQS#$oyAFT`62%aSCfPrbMC=BazB=@~n*$cDA61MD zOSpaI+$gE8HUsv1f@X~y$oDw~0=DmB|B*TRb<4p?0R^x1?cn+U%y+ zNXKL=P81-V!PFHidH;pTQ*#pVP z|2ersHs!RNr9iT<;0=Nq(tL9F!Pux+SN}OVeqj!Z+GeuVP9_Mj=iDzxRU;2w3Q~+p zHd1497PbSTgGdU-#zr_IRP2p~5nHj?74@zHCGJd7eCmyNfK|Hztsy;Nh(ss!1Hg~B zU~m)+khMwX>^m;2y7Wq_kBoCmdHwpeu2yUEpVM7I4QxDCGjpGt+v}w`#LL_Qd>E4B zz|*IrkQah@S%3}>pz?=bAPC-})_2QcteokMO2gvzH*f5WEbf(;D?i#kR;>zeK4`bT z#GcZNI*d9qm?@m>vyxu((jK5@a1Gduus3EZQNkql`>yiD^UIwf_Ol6OQ5S20nCLX0 zARZwjwI9)|_+)o+>dyPg?G9BTpQqe{%Or-L4FGTNF>$X#$23GJ%AIR z8*qo3@9;-~8)n4jfZ*Vr5=MXF<6e5%|mB$VxOIz-T96gKyi&| zIz&7;QIK9j>j|~?6^|UHE&5hxU}YT#)W2`wXV8>J?nO# zuRc2^uhqCbV{3@{+XY&(@w2ze&c1ln;&t6mHw;endw$q`Zd%&CMfVr=HFY1$4$09< z?OxQmKdd~&E4!()x2SU!V}JDZqW4wvQCn^8pX=!Ri^sSn?wRfT|9t!G*|U;vME!6u zuvzac9336swzeMk86#kk5PJHnR8&;hz^x!W7sa7R=)yut^Tj3;6MIn2AX7FpMXdq} zul~LYe_hgT8Ti8rlg-xFkw=dnt*EMsNl8%|KYskKojaL4#FRu#TJSw@)@!G{`|xY- zNBota&pH8n{rdHi(myZWvi~_Lf!9!~~RMIOTgbad+!vP`*t*-T<7~V z4CxiJY+_4!4+Le$OIi|`e znw^+GFSG$|14S{~11%SmXpAube7hfVI3ed)%Q>(20<=`OqWx?G!^dNC@-i0rZs%ET*S!UTyYbLv@_R8xt3^0u%UU*qHGZYvU zMAJw|+tT+R%$9u<=F-sNDR11k5zg3SXy_G!4On+=(_8O5Kn5kR{bygqotE^Ws`$lY z#5n2h_P6ST{>@V*ugAxiz*kg#$fsrM`etawzjVKkL=s7FZrUPdYtDiDW2Bqlo{TBS z^Y-lmbAZ{@zkzK)75UNcqi34BY^*ZI3_*UiSkl5}WeQ|&7ZD}s%eT8kGtXaP* z1)LSFjK{~^8pW~zJTc}ej7%M;PoJKMU1N!g%9RTjX5%k+?b(C3;*ba=fxq;uv#;$_ z#eXNP#7IdDRMaIw0&p2ci_YQCV25`Z>)2VXVG-sh->deJ`s!a;cYgt2Ff_;v7#SNU z*BydLoF+S;-U2GNofnon*Wa&N?EU!Jv-24l8T^(R$~$-OUbkx1D%J*XU0^j(SBp$g z&F6IgA-xMbdVUr(`Tn|N#}1T5@6_!;Gqd}0Ph7Cc;Gd&Tv_voyXx!hvefz2Xxhhh_ zHcc25GNV8?Rj9>(cy{Wp=l*TA@5^}~^ey()dW0<6_A6@>>X?#`Q9nZ~$bze6l5gg z-+F3AT;H<%FC+eU^Uw(o?G0?nV!wSI%krfru5%y9nB^QSUA*@3w%XZ8CP?r2kF-^v z){fejhbPeN&ea|r-xnX4kDsN3-q$|I$ArXdYu#6`>@r zp1o$=jAe_Ajg4<=RH20hAvzlI1H~tw`)*)k))^Z=Incz5+IiV<0kAML^FT9mfuGL| zHZgBBUi9kUGp3Qn*Q_y@IB-2Hg;wzKg(+(h{xK~A(Ec|s)G1?EshLz}B*-$gmw!xB zx89qcn5cxcDX=>9pbJU88fY>FrCa9vTP_DBBIAg;c5MMVKMFmcJDRxv^>& zgG1{9=KgqTNvE3!Lk+K0c)fJb9a$=UQ{lzlKL5U(2WNZc9R(=z~&+XiSgY zllk#sIvN9%_dHEF9P8U%cxRgHPG{#07=AVyn}FL4``WRk8b{}BF*i3tOtKjm`&whv zymglip<6%8508{;-U6ssd8fy<_Ty=fHNUEl8e33Us0`NtJ~L}>%T!&{ZqHW#0C!Eh z+_k@w77$iLryJgn_(giv*jAj7_~D*LXOVJeq4l7WiItbFCv5h_v5vQV+-h!%^-F-V zde7a@mDs=6Vw3Mc52Srkv5K5}toeQ<`btO(#VzKu@D>#vT zb|600&`qZ?6qw-%I8&%hq)#^VrvwFw4sf&Kb**xJ)uJn;n(g(WVg(Eca&72=^t<8;( zi_=@bemxf5A0IY9n#5~1Lr5rxIOoV(;X7ZkVLOM|qGXWl6wM8?VEMgoRI@{74SqO= zcRmXsg4;*Yuh=D_!U%3Eh?Mt-S?DSQ0*G-7`v()*G%+Fwd9CS{!L>gv+N$-sASA?5Zf zF8~>KNjc7N6>Hio4yI0o4dSxH0tB^m=e@bZcHJpi_GW=vwsH$RndwVrl)Z-;Wj-7kr z{o4Ke%&u*cooq>o+he1JcxZH=dSL$v6+N`d|2_;$13%+hv2l$pPzZ-7u|4tyyLaza zf^jj=$KnRQqt+E|FTzq6@_c4A~FkD z7~wX+V4w)CL#Et3T2V#zS79#y{J;P;7UHCdL|C_73=rLcN8njRv{FvguT=|CTHuC5 zZ;jAj?*V|=aZI~^Hjvgr_1Nclkp;+mlN!&N&l4zQ^}OVdh7S$6^iSN*No zgAP`3MBwM+ej}|1-kMC)-0vbBaZ3abM%N#ujT%g50gDI#)$)AW8HGQ9)R|q-s3S?Y zb(%&P%U!SIt>U(!TpaU*(J)6(r^e`MtPT7PMop_*bEiGUoJ<8ac)yw?V8M(&;*`oc zp;xb8FK|q)JqPx9t^i?Ukn<&8fM?C*{bxGfiM7G@7b-guB%Y$AFyn|*9?SaeD)S;-t^*z-6 zj=I8fTK4D9pGPv0-4B_|BbuS2>;?y=-w?mRge;7tg?p(^crK3VDM1!9R(?3;-@g0o zutXmPW%py208_i78-I$0XdQm26RyMN!(EUbz=C3)AQ8Kt9#kduztH3Dw@0O%6~1t4 zgtOP~GrOll|AMdg5yF_P@b=jMS_Im7?I3XRy+wXjWleW2UG-HpobHz*cn0ZQCLnXD=(;S$i^XyIUxdaRf3LT@q3y|(}jQY7BvdcE}&q`aj(7 z%+(&eC7yjWf>FnmIk^A$K{%qtG`JYCHb`%K0p7Q`Y<#p7XS7&gc`rbZD~s)dg6o_V zbiS)VBD)^gfY23CgB`pFfi(2HYTO=a;&XK7(`dMR$>D77JoMcgyNjv=?z*BGp9bh| z;aP1PzKSdjGvM;EC7AgsmnGR}Ut=2E*0 zK}?vTDD-y_P@o||eJg90Mj&>Z%zBZIvw4bmzs=D|DqStr2yeVL&E*g*YUJ`{ z9@5l4Zr%S2s5}9S)DCcBo&e3^p->bMO2(eP8If5TB!8HDnE=UZ(m;Vp7K=SzwDlzu z&<@dNe3+D4&*%JnOo6j_BX{d%Y0~j{&C7-BZqEvs@J1q#D+*}Q_J260EwA#|4EM(? zL*V0y2W9!lds@dUd*kdh)rgM@I1i`+^vbRhfSnRI?C9IFx{_~RzI*p>4n3Q1ye)uo zAy&ib8>w~b8f{F%Dgp)KtuN}i38}nx@{z-v$A4T*m&%jj?b1YwRQI zDgW<0bILnfyyTh{z7D+h8|BDeJz>Z}CSS7b^$Chu5FTq#E{AlJd-iKrudZbYrLCE< z_cu(7<9anw;d>uWOMiJ(N`1{#C!`77jo!Se$Zd5|txsJG7mgRb=p3dmE^d?TSwDYz z;QCqRz^u~yY8IpNtH(AxlyWZmVSbo_%Va?Zq$l?O{`KRv8#jvKzC+j+2|7Rj%$k`Y zSlCl6(zbnpzI9h?=Se6%h#0^e9!S7^_#fTAJNrZDf zKcCRGGt>8cq>1Q;-IatPT^ezdnYXWI?;6$yt$OK5g~lq-1+XLI9;y+>X{SI`Y1)_F zXSsZ!WrJk6hz2^%|N1KL-htXwnW(LNwDc&!AN}xgy7l_-YCx_I=bUyyiWrEDdTv}xj0A$6_hkvpgDrH}pI!j$H zL6fGpp3_I#oB~X|U}zNi0;jZ>M^8wp#@92lbtMZ^YiP{Fkgo_>@Ibv|KhgkySlRn> zBEwA`0Si(?v$r9Dk`*ya^?2@Ah}r#u9J$IP2XA-woKP;wBWh~+@%B9B?ES_;E$`o94TmoJuyXkv@xU zK^f+aa_A)nxY6xl-tEf{h2&=6pdN?A5%tdsF{4u(PSJZb#eB~h;SS6V?Z2VDb=_G( zH(}SnrF~pF>(lrAg&p+yK=86kH_gE+8Is(B2+jh;uTbn765%u{9?xyNU9PXgbrWLV z2N`XaWzMeCSwj#1;T~}kgbXktV!#*oKDp_0*3z^3a@x zWSKxd4Gp^6LjhV2$L^og*w~m3rMKJDZO7&cfK>5uK+S#Ck2=ENd&lHuI_(HpuaXb5 z6YMTTdJ)LNh9oCWr09d1cMGG|Cms?69}4QiEgoJ)N%gPG z6K~gm@m7?*VLoliP6G()vuJ$@XrqCXVG|(o|NR6}T>Qj?a0{{BoRVLDesYx+4U(kA zgV1ZJI{>34mg{TWeYb;%BKSg#z53P%Wg#~-;4}vB@E5Sm5>-`u$j_->Ggjd{f+Isj z!D{_mY3@ur6*ekF|J@OYA(+~7kieE;FAlwt@@)6rPQ*1eqynA)PTC-`sq!vrG;*mm zuDbeg{@~vn?2W*x4bk*+ zCr`$cRPRYDcEuRlzCNjH_1@dk(vk>Gh+Hm4CBD0S@!~H?xHhkq?Dh_ohia0CC3gLbKj+{+jDp=ifBX=>V;pGi5?H2LtDc|>O1#FRE7aF$4@4<%uD}O~u<5Tk z{xZVQjY)asE4Di3d`hb;uRE*zF2fgKZTX`Gh7z!mCLcHZtG~V4Uq4#AS@Yf0ZFC!1 zeYNkWoPDy81dP}U-p<&T25{qw^>GownS+IQYc4TT;e{wH+9;&)NoMvxx>7+^lgaSs0h5U5d z2eF$UZp{e7a8LqzNnLOZkuxexI5$izgP25%e;@%u*nh@yH-F5YrJN)SO* z#1Iezg9;i`m@t>-1!hLE2u5kuklP-X+5F<-;suI|kFajt@@5~C2!EjQsYv;e4Q8t> zH=rIP{;n0DI95CCcuZc;>nY_0_<~XHSXo>@!LYwi|f~^8uhmlm9%1FD&F54 zxV-qV(F|zN`C%@<4t}Z1o?`Jp!xaz!Xq1Q74A}j`sCke%yA& zQ^#W2Q4J)VghD1if4pkVX;6`00R{Kde+C;`xpQ&HGFN+`T)VMMmr-ojT;6`WyaU6L zmGcYtW5PdV{Dl{;bi=8R_E>NN78N%ebUz^nSSVgp+|Mre7EbH^;lyu6_SH18d~C9X z$Q@1t5V>$9wdm5`X~J;#K^@s)emj%;A_Bb#d;xYdYa19$eT9eg^Et-~xs9LaypsrQ zq^MyX9VW~l?j>qXHZYnUX>#xA1>B8lt-BF1mJ385Lce-&D_0}%RrSE!X}I%ZzzKxE zlJ=Y69UT=$8xU8ucf)^Vm{Nho&?Xnc>+p6C+RPn4GMl_&__>V*B#>1=fNSn z7mf^(_a|t`6Hkd2P5+DK8OiYJ(ZY;7Na+%v*axD4sfx-)spjfB z3o0y7-2jpj0Sn=#YIe^9MG6Lo3-LKo_PZP%iwR+XvDTsj9%O#X@Ta+$w6<&%5)9-9 zXN`xiVsNByIC#$Cg&El(B$iN?3tdY9AH<*{tsVCXp>*4``~gWgzTFUl7J@-O3S}d2 zr=-xXFb&xuWx*o0%~#b{I1Bc*MID(y;WU7cm_hERC~R+Zne1#2W~#R`H7mI;XwItc zGY&LgAVQ-l9mEyoSL+^)(6(Zj+mLJ*QFkczm`#5R!+rpQkh91K}%IZO6 zTmw^!Qy3G0XQqTd*`Wnvn`-FGsqW0r-#Y@EM9`Sw!x#Mg>aRISXDo30wEe5T@OE3? zb?cNzgoCs5Bebj}1{n<>esaS^ZB6(HC?s5O+K=8ZW$&I3$YhzdTt0%Tu(M~KSCi-s z?Zg`*#_W{Or%O69AIMI*k0LZS7m8j5!$!|A2vQs|dKLg*x)7U*nd> zHtZtA$N)agF~Vk;mr2|pi)Q;)X>pdf^}Z+iT;G6=T+aex5CX*ctpdEuaO@2VG_jjL zf}(bN_4oXTM_Z5WfTV~RrwI(U7=Au1$s@2KWQaEdx_*7K=>8+Z-AdG^X?X78>NarT zC_odynD|#W#?91RM(+w*7xhQv{E&|DFgEV384#&5TC2Q2$66NySSKP#OyLp+(S2vG z255Rg!}Nh!z7xt5ypGd20-#F3K05Ty&8zvsF+-q2mub$8c%ovC>qiG-IPx0%=Z>ih z6GRCrT`o#DSXeyu?JlO$4K|cUR7Bk%`oDClx(|w9CI+reU*WpFEe3yyWKBBBZk?AcIu(USA)Ra+tT=0K(u)15~GR0#p z_*c#*YM(`NsYTq858jJ-BNfH#_hG`#aXE*WiKFC`Ob8t+XSIO@AFTfFsd6-IgfVEM z9^4t?p-`XU{Q=Yhkwxfk-AUA!kiE*h3vI^&P&*6NkEptUDQ-Yze*;yegscl-J0l#S zStw{-r*$&)X~R$;-|J0?y%NqR1qQn!obda1czAfwB&75kTZsbk)aFJ$@Iy~WP1MT>Kxy-2QCdg{VJV(jr1(q>$S zZv@HR1XzmaY+yAyQu03C zWb69?+ixF^-Fn|0k&3_uX$(=UCyE}~ssxld!u%sAB^Zg;^J|UU(O&M*M{86jLQ%o571*x)cRzALPsR& z))uNCt0H}iEF&x1ft_EQP_HPbdOV=Fm?QL7%%)Np;d231D#(%*n;xhul^N(Quz?~V zf>JI43qnl+oxThuEkZ)Rwvej44auzM~MbdWo&?}!PDiiC@v z`}6%IAx@LH0!b8!{JSM}`pfL5w3;`RB8HKqPR~ zejD(Vz)ITgDM%=)cVGCri_rSV;hf*-PDD4#oAXljB+!Olyw2CjDYfF;Za2DIGzW9ygZHQMKgzT6xtz?ofarn;vq zN=TmmOSn-I{xOvHZL;!&WxXJ#Hz>2e1CbJy@gR^`hRqHOpzh zW;Zr=S>N$Pu5F9D01o5c$l8E1rCrjq)}xAh>7ozYdIY@FI@&QF+j^s{k&ldyQ#*#> z;B%_BVGG8VlkpB<6ACT9fD{8^a6ox5>Q)4Vl#RsK0y>=Go53{3mUu4U10jAAeZkCHSnD6u;kOxf3I;0CrktrIc5591`Hd@)&7B`&=tF?sMzi72x%4x zZK(VJRdqHeEx_}s@PxTRa~TzZpdV5bl-1QmAXTvi55QnHZNv4r5-b6{Jg{D4I38c7 zl42$#B#F)A>iz`*d>)7(hLj33y=-t+xrO^r!r`;D_(m1;oG*h)$M#%I(T^{{?4lD= z?9icw8O>M2i;JG@JrrLi}g=PO}g}O>Ph8|kMn!dp0ik(Y*Ye&ba7rVXT2SJJ5WR7)Dcy~4^s4CMM&BHZe*mp-Vk{@>g5xIhvx`JA& zfW^F>Pn3d~C$F&k4vcuuB;_S&HCK%3yxt!3Hkww@eWceb5m#ahBr3qM&gUBdV0F<@ zA*?)^rV{VimOV5``6{P(A*Jv=WAF0}0kr2Y20f_^ggxm^j6Us-{vdB zv#tQ@cY^|~Nr+Ry6?@_Tm7|T}aG{C?ygjy|RC`PwCyYUf^$3D4Lx_aMk4C)5tLno z%8JqVT#Ur4e?`q*b+AZc0Kk=U3e!BY!(MjX9(|8QZ{JBq2DvDO3E-}GA#eYSSH zH&RkUC?euIXn4E_LIjd?U>lK$CifdNq#p_x#_m)qrRQ%wExLv-3`O!##Vto5;^+VI!=#S;vj!JI zB|#tSh5&S9anU009+&&1Im;Q2zQP1*w3$u*G}abKTzP{OS!989tE3c3x#GO zsn-behij-hXMS8w6#xjZCx?$_Jl4UPifW5CFl`X92zFOsMoMM1wcRx_Xju=JGoBtO}#R_{9k>W0+l>(^zmc}14tMZj0_Re)DTUQOE%nr;+23`_zG%#RN6w$uE$#GVt+!RcRpW#-t*bI<{ zsy*QHEx;fX<4yI&i{&c+CGk3CYW#e3MR%Z+Dz*NXEI){4mcmuC0J7jSciu4>a#-<@ z5@CejhM^zD7O`Bw0^e(k1&J``m#}zf4*Vmm`oJrPW{mY>kYP(`WRxB?+e)!q-R>=A$tTHy4>ZrQ4R0Jl(+Fc=KjL3;CRS zNv$<5iBeN>PUbu0y3MPU%V!hf2T*6pgUNU}ZL~9`P!G|E-_27+%R3-`Ak=wY5q1h9 z;hgDN3Zv-S^|7E61hg~5_i3Jh67xnzd+@dP7*3yopBM}k&M}p>+CBFsx)v|3su-ti z;hH}e(zW1iN0ZwSRNsD_{8 zNhJ{G!r??*Oby%r>Th(nY4`*ILL4rv+tyn(OE)6Wx z?uWzUvv6V$Jqqfj3Tb-=k{(Pkk7kAjmo|Z)@FF2LRGMGEdfbH}wmzF0T|j0fZ9D3* z6&p8)KIP3*v7f^Oq+&eUL*VX|Q*gjf#HJ%mj-El^6 zg4~XBm^L(`!W|<=Qpn`+2K~{M*aQ|Zewtlc*E0t}~0|g>s-m{;X3= z+~3z}K~XR==!qlhz4XplK10El^r9|5w*CV*hE51wmT(jc%H(!XPH{D>6}331Dw-!2S?WV6pw=eRzQ$DP;nRT#?|>e0c0*i! zVuk~_g7nXojJX@K#4m?xe6dCJBd;%*4E;Hu#sk(Dr|(bH77917))K{o57Ne-iuYWC z-jdDkKUwDxw}YUMedwYXW(?jMdy>-t<^+9!72p-~R3|UgI|KL#?i)yr1g3V1#V8M5 z)WsOkFJTB$d;yt65jcRWt~{sr*nRWhyY%_gc6V*lI?NnSvjmI2!0gzA<6*f;`=NXj(NS+ zk9^KGFoW^~FJ4~|&flCWQH9W3fIG2AN8JSi0s@wb%y1?y!1$X1G%f@IO=|*S8O27@ z^<&sH0mS81D6H>d*NWPz+ z{3*sgU!ril2=Oi?ZszyykA6w2IqZMtG)i#Od_=cMg+)OJ;MNYswp&rW10{HV zmxm-|?MH%K29Y;4ROquDLWE?w^M>=GSh`tALNCEEkV7L!CrEGMC|%)=@uaSZr(sna zlzSUsQBQgnvKXK6{6y}g#$l12jTz*n*Vs@NNF#)^0R1$A`YC=cB;z4 z7V8Zvhw|#&89i4=5LTXrpuzS1cE7h44NlIen9OGGlT0HW8M+Cmhl-C~h6zUqdd}aV zH(V|)?j5zYplk;nBmHcg8;b6_7(xOb_JAq$n#C|iD(ryH=!C%x*R}%Y#7^oWW&f;Al1mULignK7kCef;g z9kv_<0~J`niLQm>fm6gItSG7rS>>%qwgvv4d<7_KS6!x;XvCs0L^|`mZ6Rk*e{q3k(DE|24ni*%v zKQm`Km*JXr6U77W!xlQM_1xkHR(T~lpbCWK}Nsldxq!gGSi=2I_3Y3{KZu500_765N$Nv;Zr_?K``*c8Sa zxvVkJ$hEOK@WE$smHMp7mCXLvsm~L@tZC zs%8a`U5b}m$0yg~-MJDuz|F2?(%(^FAm#;CTniXdpA-kYD1Gr0(STFOE@~uLtcRlpP!>@K;9G-$ZP)nY0rxWkPWIj9r1J1cI%RLGK*Cz#3rK1^Zt z(G8gdQvz|y5!EH^Vd0Hz#YX!M#2TG5DNq93qpCgadt()1rBiSuBqah%o2XI^<(k>n zACsS;>ca$)*nyMxQ~wfI&RWq<69`G53U^$BCMxy4sZ3w$|MQv2E6Ak}T9?~62~R-5 zEgF3|1a9dw@h*}c_96qaH25KzEM?q4G(HK71hF_x*-r0w9n4h&=FT2iM@-Z9@uvNk z+n-P|77~NBH5Lo;V#QC0YRtCcdLhQG^n`Gyfz}!keoatd zetbb?+kQprbHq(-7WV;F{2;{cQRj9&XHpzVEC4O+T&_sx-K6n?dlJT(j zq-ve1!4DqyNa{|M4F}G!)7wz>Q7n6>p{x?k>n%HxhZG~shhde?7k~>7TQxzAk1YUU zby;nOhxzM%a_T@ee#~{|fKkNr*!UJaOFcgOf2PCYVS-SL9kEU}4=x`Ytc@z>N{hQK{w>6)##Y z$);#1b0z&MMjll&ASLYd1vG_RsxzuUn9gv?8Rk&aMP;ktsc&(Jj~_|lS|H8ix zS=q_GC}6%pCL^Qtw>k*hQDhvUj#Fj0AXj$7Je9~4VUa)165h^-i1w**xj$a4V5=|_ zz32(B>f*Qnk|2@}J&-DyRETFVLGO9AmVmep$1dFno$$@{b~LBRC;XMMon(@5H7B0~ zyhwS|(}Z16G;|UwpB-`$ggRg-@`!jAPUs1)R@i?AP$o(dqx{K3f@qf$Gt?5_q|lZS zIn*1O1U_XrQ&<^Jm<`jM>wQ5GB#s=U{Q~vl>KD;on7&jNod9wi1+?`BB!jwTI4+K#j|Q)De3OZ|AfBrg3&q2& zjvi{IZU6j12So^6@ibRp`_zlIiWGAnWh}sf8O`I~MmaJNY@la8vNI`iAdX8Qe3P$( z(?Yh$DMa{0EUDDGmy3hn+bP1N(qN7B-hY22KLswmB0Y#J(o`z#0x!2$5yv-)n}vTP z_%~69G!h_Xutncs17T$7Du|Vzl_JxM zgovH6q@t|FyVd-~IyUojr8(&WXm8qnpFZ(wQal6I9@T3EO3?K7BfKi6am}Mxy$gi{ zoiMW+v?rl0>VeT`snHN*t||2l35#A=j}xNC%#7J~|EPdHu8<)^l&Ux`hH}E0#o>9jhTpJ-H z!_HR(VJlQh^jIFHNUfCsLG;cqSpdU8F&W9LoF=+N#C602V{k)}%s`v{(o`hjvN2(< z>uBqfr@bgj;Vlpq^2V=ejt6xV;VUi|e@pT;BL0@n3eIjzEL4KvFb?k5_3Bs6DclpY zA)|RQ@ECthi~dEg-QbL!tB+p8;4X$`eS48~J z85OS+7Kq_LR(*atE&*W3-3PJG|?tPP1 zpK}l}rWxfU(dxvF^@6VuXiSIS=xr%z;8?g&4Olvz%+7JhS0!o<> z6YWC(GC8S?Rm@DtQL0=+-5Jz(aK4k^01Zh3P0_9GQ$?EexdI6Jju16uI z!Pc{$^@hqE`hgA9%XqkOc{UW^a|9h&jrRW^U2g(cbKd^{9}H$JV-`DE4??!cT7+So zY#~bv8Oc%#MUnE!7>u1Gdx(ncYbsIp6v>vTh>?;cR8m_0&+AlX?(gsM?{SaYa_W4} zd%2d^>vdfhIFbB=R=N%8dZEugcBnHFe4!UKT1uy-gPK$Ylm(4nJm}u55ArfoB%o4F zD@w!)y|eybWf1z4%T#x(k$WiOBVekx^`^%$VzMwc&0*+30sWYmg#hSG88LM- zK}vXtE-Nm)dBZXrW2z0njY&B@uIJ*eB$4lNT>`0ux_l;1s`BHT4xmQK6;5U}{nAji zZ|@RE@vyl?&3(BCryZ@U6SYjU|M(?sYh-QIdnGXW2Oiu6iX<3x;jhkog0NH7<`b9R~g_a3dwBd^u&Ma$VjjTRpJ zAOM@?g$2;s`%v5Z&!OXhrhhYg{jFvS0>9z9SyVd>?ajEfH4LVE+^gURF03)UUUIwV2So z0GMqxkhwm(^I9gLRN0QY1OPDA>on9Qk=`d)+Z3W!;YN+pg+h4_v|)5Y$PB=Yvy`RU zLu4t@Y;#H2zDRDDczkq#k*R`Kk!hcyzQdl+Q1mEsJK9!haAK4$oB}uH)8T~v#kmiq z9x!$>$45enrSE&Hre2e{OOj$C=){g3aP{sI1bw87=rB8Sd4#_d&DM;!%A>{^VgIb@C~CDH_{j=>^vfXp&E*6MJP}}CPmfp%j8&ps*S|(OT(EPa6cr1Z z$B#(wev*T4CF(74iTgt5vI=)ZX^YUqaeWf2aekhz*SS%?3qz}l11&u%S$wCF7h`?EL3EW)^Z~!x@^AS^QJImPF zDG_agW;UC57UwLj~WXiiwt55AYX4dj5Utjqas|X+N==K%bH!Z?6U5ETPsy?dpc0gw4Ahgg0*>Zsm9#h-fX(KI zNVR=B=<8i6NtvRcu|-*jAY)aXe+KEDl0)$M-O~&%*px*tS4iESu>Er315qFiN%xxO z(^4aWf=ro{2S#lu${T{n^FhC12E1!OKzlfj#1lbug=p=?a1jDHUiq=#yG=+Cl*3Wm zwr^M7+RY^Y*a{{y9m=T>W$X{(XxfU`1fcr@Gu~sdFK1ccMYDbY8D;I&Yb&AO2 z-EM7C3&l z{9aPAGm7b7#;A#~j6)7p+BZ!=13ndocDp*7^yUHQOK{c|?oMZv=(ZZIXdR3~nv-o9 zMFAj9PS>@=lFxy_=-CWx>f24E_z6u*3CN1B7fTtCHh?^q#;KPN^^fWn)ih^yX6Z@X z;6aY3bSxxmc5t@kBnl)Z`mMC)Q!A-F_eenslu-})pyC7-Uyrwk1(8^Dw{pow(WYS! zb=B0mN>woZU4TUa_fANk4pC1EGV7WI0JRf1*-&E$`X!-vkg+`MlNOQ-?iST2QU#$Q>%(MiP z5ozafU}>B*>HattQ`;qfFwSn2#CDw-ek8D-u0tH>I-R-eo%`1O(na8I;&T_u+u8{R zCej1Ru0(JjFdHq+Vpum9!cHY2c>-5OT8rb;<3Q;+*OtSY$fud}y>Lmw21|z_QQ`uK zR1|M~*a}nc-B+Z1NBcNGzgW;XU%azX6k6$PGy9JYcgxax7L9x~e49bRBS&GwAa-8M z;^MP6sYOepEp0T1_tzWbDk>ze{cbAnleeWr81&C@rtkcri6L|M(f_KgY%Ha=Yep@v z{n&D*vS+JsT=dWzBemNEE#iISfTtwCI*{WqM3?o=`Cc%3fYn0+uu4S)CQrJZA_M=> zUr%-maf<9n@C$^3BB|-0vun$!z8P>((vBw@7Ji>vcE04bZHfK*uLFomcD-7!^h|uy zeP5=c+}nTV%%|)rBD(;}Xe;;!7nTODLdG3BaNr+T_bRGmozR9YCh*GMr}_6`%th#bH~W;^w8poR-{iEP38yFhU)H#$6+ba*QE#b?_9nQXpt~Xt~rk zUrF7Gs|^<>&NB;d_oyj`t0;Sl3>p;*RX8ny)6|WW-t%6}?B6R>`u?mL1p@4Z_i-#X z5xn)V!$uH*7D zZ%K5{-}M7=xXpo*+_Q}MK(G?XektC;QzezKoAh!m;iyC=?VRTVgo&=Lh>m}w2r~xE zS17e|*f8<9Ro>2%r-A>C_U31cn2Jt0pU}tll*N;`Vgl~cz0H>gzr!kYrVQWVxF~4Z zg4m<&d#W{2{t|4MQ#1_MBKejS=*9|H%u>xD0_f1Ujzgdzx0qYr zv5PL+f8i($Xh%t%FM0GoScZ^1a_ax`2UV~X0!hjt8utQGQiy;;SaC5X++T_D_7#S* z7@=v4o+@YT_Z2iI^*Gtb`9VPR%qjgifV*D#Z|K{B_+(#t6hjwmFagpS`J7`yXQ>IY zSn%TSn~qj}GNRBy`b)?@2>h0Ve&rxnMOhjszb@`C*7P=D`fg5XhDG)GjhitN}*9c;#p zuNSkHnyCK!a1~|E#st75qqclif|ax)hu*E2$oz z0od&;RcaEiN%wZjZ+hHP1>gII{C&+IT}CXhF;qG$H9Id)d?{Kjsg=?Bd;tU?hr-Iz zy_;yBHc1Le!X~8!xJs*v(gJdO?+d9!e4`~#_AL6thj7p6{Q#uSTD6qUu&Mp?bON4T zuP7pR>9lp}C22T-QAAgu&5)2dy9@zer!fjgH+M! zFxO+&k{XL{mH-clGmC}2s_^-jS4vBeN8RMmi6K!(3w|cLD;N|@ znkh~7I#_h=+t|C+sZ*!SO-;QS89aUV&aaPC;q-dUdtu5%p(egRcHPFTmYzIMxO zXdd8s@*r87Fpiur?+5uPn?vKZHxU)$OG#GR_CyI%(SjS!2^_TQg(4!};sM)}0=gz|{d~^PoEl>pdq`s@6e=KY>v^s;aa4O6J8iW z=FiT#c=b+XUZL4oGRT@25we7yWj}5)`3=7?ft#>S*-*vLhmt=F`cFKeO*%jPJZ^R_Nc%9ary9KAS$onAP`&C(vTzZyp#3*HP0%&mhN16 zWb|e+oz^XSak-Mr9VE8=Krqmi_63RVNsoJB; zAVhORV`Jmg)Ks-5@wDA4G9>%*(;{n?I<1#Z3&rI!ymJ5k7}P5X_?eEHm|M5*Aw8_y zw5g5XLTBfxad+=d7(4cXUKE3hhG|dOMxH>8R==f@`5DGqpPLce*wWhC5l~5{R^C@O zm+gbubu!!|R1(~N{(Oq|97Ax07=B?<_OJX}u}3QNl~0nA97&oP3F#A%e6%KzHU;&XAGom?wYL;OfXbz->jg2o!K1~)fluj|`q#8*# zi-Ps9L7%P7{z28o5C1b%Ck*Ej)FbJ80Fcv}?l~iO^@#r0U@8CkV4o*Vn)C+L%z}N( z1LzpcI}u`!V@e*ebLS>ac6+8jxAJM(t=q>9c6%57^TX4`#11AVleqvl_F`-0%kXx{S8`C&~R&>36A^;idAsllkz_Mn-bW>_*K4pfSX) zfuuLjdY*??N&Ae7GiP$n3@7lavPKM_tzceS?ZIlq%zSzP`Ta?4o`I!W+kr>4I`zYCpDKx^!uAa&dk8O#yG- zywUzp){nyb7yCc{*bl6C1l6N9fLL4K*f{w=m-as!`(hdMTJ61nFy!8=g@&L8;eaBc zezPrFtxJMYVzA>(=TeY`HoundV4D-{c_F!6E&zZ73Q*A)-Y+*9Jh}O(N#gCxzG@t0 z+N%|lT*W%ZG0c=B(cCL{b6Dw=A&r_TMf0&cL{n=I)Y6~GYMhNHhV$ETj*mtYkI+c0^vV)it1!w@V;-t-Mgc(Ra?|x zM^@y%ZMOZY-w9Xs81UyX_Wss%!$x}5Amgj%);DOJ!ZEgvRbJF7y5aeUo9rxBW4q`s04BwSO#n|wBKupsXsO870 zyno4;d8nM%LevuLHhi;aF)b?wo2m4B(MjzvQ#`RBikX&(B!H~61D`vk4+Fv`5Uhl) zZ56XZ;vr^oZrc_%x3rX6jD5K1Z?3{c3_G_e1F(Hp--Sm84IccT+Y!^~-*=PYJSRzQ zkC<)kpyl|?BAw459IV!^idmEbhkcxAl7^>e&ZK5$9_6q#Dhp9ne$y=uj~#-#OlLco ztjZXyn14^#fA57$6hkKaxxe=zU{Yt0*Dh?;THba)9$pjP_`a2e1zaB#kW1^$G^{wW;%7*acoYz-36XBn<0Zn>2OH2$Mx${c2 zw>F=iJ>w`e@KN80igKi&iU^}?`9oB^tLpOdD~WG-cN->#CSF*0^y;;13mE`S0Cj}< z@JMv_lgImk>LxENAv`*_wAf=0qnR5uYIKC2VKVofkn8AJCD5#3?^z^)1ypqD^`2)+iR?-ezI7Y^k54vF zv)qBrxZl!zDIDu0npA1k?YX~VX0NRFu6_;2pNB1d%U&03fIXVnLn%>;k~Xw!+tz`E zJrTfJFP$jV)K)1DCW>8TZOg#Urh0nWJL9fi9Rvh9mZ{e7NPeQYYR7Kf8iu*!!Io;n z!3q88YJ04&W8 zxAO?HV7k*tJ!>CZEbjzv<<@Q6+G?1UzNerVl{;U}CdkE24n2*7`;S>vyw7A*8Sek9 z>hd*bgn^P_otSdz2!PTouyV}(Lw2OO7g?>t!-MP9?c26ZJ0C_ih%@chU^*k;_JKhh zCvJ7b>zoM*Nnkkae$bntJVUtS2Ch>_Tw)Q1qiyVtZiAV?YPfTR)tTw5r2=i{-6EwR zhg?690p6ED`wC9%ArFNv)g_Ui?rP*)7;}`15e?p~s&f^Wr^2I1ror$7Ex;{V~pJXplxNANC`mF!2Nqu-~`(-+rOG2uiUGSn8ZW|NvJ%(r+fnU3AN zk-5vC%a_LoXntiJwQg15Gt0v=eHqvIkcYH@{Lwss!45z-^dE<+vj3TI^KuIbTmpwH z+tnIqn~4~S?7k%@cNZdnaxX5*iPHvPwRiQ}ShRKH#*JZ@Fa?9`^B7bfGTOKI=0kB@ zW}Bb>!O=4Z`|iU5>}bxO4e4?FY)=eG9YAFWu4i{7GMO3g`2f~?`*UBgoh?#=fe<_L z2f1x&cGc@B%^p$caFA)6v#9iyux@g%AN2ZrPNFs8c?h09dGDT%1FU%r9<&g1d$}h<`#bbUr zm3Zope1F%Qw=SxUtW@8Ej zVjvaVx@S)(P2w5*wLdoRZ$N8fnZTa*F*D($#MB~i%XTGh4{bspvypp^EqThv2&C#Y zZhVAO{e~UX2An(B!|C}&%iX=^ZUd_E$XDKXvbD7}fk=FP;Y9=Q2MK^$So-)d8&0p7t7Q{zR6aj&*&fo#APAam7S*yI(CYtl{ck?dE%^n@ z7FU2cG2#Xvib%qDBnd) z@YE}nYd7p_3d5El@E1_$5z=!esT(yI6&jtQq@>tvy28bl7XaFx@c!W!zjGJWX4+NZ{q`yh@e*+L$J zGnqYpBe{9=<|j98`YA6fX(Au|9(_cDXT~+F@@o}Ee=h1fh5B70Q3nB%PJX5WK3N>f- zYA$?VUBl+aht9Xlc?H(ynsod2QS*d8J$p`&IF)X-N2ORXgOMEpa`e)l$m)TWXdV#q za^@U5$F$V69=dK*NJz+!O?&$xA07?zO=SAgDcuWNPY0PQ+FiZoo*+!!k>E=frIz{K zZyz7uEx2<)fVxlb-m~PXa~eouQLlde2`?!3^Yc7&?wrr6RjYvOatXjV0XEQ0_Xzs8 z`uZAZ3Q?Zn-2?Faf}z7(3syz|R%=upzhje#J_zE*`wy+XFk5+E9FacxlE--RqKD|l zWdp#w#p;zp2;_nFZ=E`=Y%dz#086E+Rh&Y3euggnOX83Jmnqhn&s-~|@fFdU{OY^NLjuZulAI`r#j zjpXFM)Yc{?0?VYi5j9PpJ^Qe)Z`0se_pf>u&8Sp^@Udot&OyABB3F*2Z2XIK{uEm| zj!#sldO7I=NjniQO3`{h&igEm!v)NhF7r#kniwtlJjZD6a~oW>2P(AGw{IQhEK;p( z+|TV8ix5vvUAdn97dV&LA#VlG#)x)k-Fh%l#Qn&Uye{h-bw3!Ov?@o#^zC6!8uxPc zMU=mvCa>r(tw3{WW zLJeMomr`7^8X(Ije0~s7g1hg-!{gd9QauxFChep0n(W=#+R7^C%9VlamkZ$=8`(hi4c64dh{J zs#ao$Os(OF`cAsgZGQa1BlRDTqB;q$l+b2jq0Mew16&c8h!hm;<0wwUfrAIsW-)5i z23<>mU42{+R<2S-Vq&s}+H~%Si=3t zqK^j%+tw)EJC`ryWgeU+>;Qv@Z})1k>gbX9MTHhVszblA99walMmF((4|g zhEKn((l%{_mw>#0TiJaFv~4?-;}K@JkAMm!^=qq{27lx3gYdKYMm^iN*C40drsmKY zr_6^{aY#IO7`=)=p6>blJGN`p$)28>AFY$fa)Eqs8S*&>Oy;?JXJ$9#eEHMav;B`m zN`{R;kf=G3Ws zNO<*$+(+&-90G_5BsPB9G`=aDYQ4wjpSV<*jLCQJ-nD{);)DRpy=aFL7N(ZJx? zq_aNm?unjOiPR~~-M*LCC=EK>+#?&POArRn_586dX+hqb`;y;Cd&*@covQkI%IE+6 z_ut8@+v8u`jo#a%xdKu={?I5rXdW{xqondBMLDA-cpEip)QxTBae!%tTx^&fe?s_M z_$Rd70skVzrrS+yE7wp6McQcSVR?zHmXA#`*(*dr{pcCR_PpER8Y7r`gV_;UH@x=7wKE_}l!%Ud{+#y57o1fjqZPF$8Aj|5^8g>&AYTKDfig$R1=uwj1- z_px)tLE~a@p||(;FZcBFGD3x3S!vL(ih2KffzVS%&dHMi!Y<5v`6uA+^qDi;X&U$U z40T92(TNyoa^%vdRlB(=j7gbMDv$(#GoIekm~Ys;?j{s&yN9zMRIMt`{-PU z=NH3@ii=;v001INIwF^h;=u`@1W-Psk=8$an zsHr1TBU_CBR<(eu(xl*C5j?7K)pn#ZSK}1uj2rs6emK*KWOt8G+MeW}lWwA`mgqss zOt7?bOwCkv*8HO%Gw&e~P)=RQ7O;7JnCH<0iqD}d>oxh}qA^6^g48)FyUWc$#YI;w zI%V0IYtd}RKBXiKwzIwFQ+|E|C8NqQAD_Ce3b6Ni=VwO*v}buPIF(TAc)Uiv4#Tu) zk`|NnK1p`R#IA4HZ4mV*TA%P`$ycuISMU2V#+&pToK>UkG2>1{q(ts%{RyE6=wa)J zX$j5Sa(G)$tN#ScA}YO{i5?yv%MyB%1uR~=V1cq=Svv8`>*$|;{dL?6Bhuk1TyH{e#+M9p;+hUacrdfSCJ4zLTF>9)Yx51vlo#awM62yh}pwuvzmRpm7 zzd;W`cz5^+?ws-vT#ewR*#v-=x0B$OFq>y@|l z-Me=}#Zyf1hAN(M;CACfoF@xw(Ov02PUIb|)14tEJeb)-(o!S)n7SY%qxbIJTcbve z2mCg+m|vu7jz$OAa z_8^|7i9oa`$8I2Eii_vxJUu{O>PX3suqWV|wMNu@F_2(pn)5_(h|4+8l9TUbh1atW z4J)Bxn?K7sToMqhY;T-r;N|qHF$ZckP&&V1XUB8?$vyI@oo-mJvA8BklZ(510$cMR zP}qB{`G=(Il~il?T)K3Ln|_e&P-tEG1mKkxxFtDII0*MWe8?q2_uvH>ZN7@rWpQ*u zCs6bkCvxP|073MPGM9 z!QlC?h6Wxy2&OzPEf~{apdmw)VDw@* z*Cc^GyEgLvqy_~{?mS{lwd&QYQh{#&MF z+9&GJvPypI4nx!67AFUVu_{#)ug_%;S?9*)zS8&Qon!yM83OV8?OUtuzs{^|pcpww z5?;?^siyd%KwPKKoEeKWJvBYO9hQlOh7zvBW&=~-1c~r^eXmGVIG1P=gdGV0tGNsB6=aN7Z~X7Q?2?h)6xzC zP&sl+K(7zDELx>Xyge0GRA4kun*VdXCZCn0`#NlN}j zL7;o|uwdqjRz$Oh@hMPMR{LF4_ROd9Y^hW7;Xs3t(Ufa*ruuSKY;eU})q>-H%E|dJ zzo}Afcb!%FXN7fkKyxG9*Zuh__*Ct>Y;&ZxZ``>M}PGsF;NnQw8|@q(iT78^6x$t3@RtK z%@1@?EP|#oJF?DByxWVnZ#%Tlcb`N)zbxH7hdSo< zQCI(_!@*gG*%Vup3v@2`ratO#s8C3Q-w+gDv+?`auD!1FP0ZlakvuwqgLZC)65AMX zpuR(Gz`n_$^9C}mi-qd=s4-Qh=F_JKaEn@L58~WDL_vIZO0K2sT*RTJ!zL|*? zsd5FY9yk+wCN_WUxN&(@`*i5rH<6oD{^j>wg*XNuwgcPPbIub#+Pe%BtpeF!0BGcy zoqBZyc#22Xrxhg$H!1mvVp{ei0)vS91RznjN4P|0f1kpc3QYkoF{|-y_x$Qkf?XsA zyGK3BohTU3TuKi0Nj5J}H@(uQZ(kh9khCblL`RHRw`_IU8<n(pa6Iwt`;dmTlU!Y2^E({tltEIE&lWK$sbj zyGG@-y=exf;XZ-rYfj~YSaKvRo|F=mo&WO3D*r#z)BY7|B~`;)puyxb4)My7t-_rS zoib&L@OG$s|IgPqHJPf`1l+5zO~CG2A79^Na$L!<`9%Ncdzv|aQ#E)SUKUB>d8cuJ zO==AY$31i@d1WV_Z<7en z-@Q8qj!~weB(L|Pnpda9)kT1$Z^i%ltYZLNB>6f*x}xbj?}axBkRRN{zdtUIz9zHC zn}8%XslDhcEVRMDn3(GQifrjYx3p<>u|T@lE#QyPb7Wa)CSVWU$Q5(N6I$2!-r{z3dFSjkDcXj*jp%7?BpRT z9ngVQTfK2(0;-yI8!GOa>iw_ZuCh-PiI&d$``c|!aCLL*8DeK=xAx+cG2FJ_H*el7 z*?Aic$eWO4T)UaG%8|vo>jDE682)$s*yVuyX2i~qs9gNd8T_Aj`S|f;n%n%k_VYkj zL)LdjK1oRSm@pKUwhKC`vuA}?aXgJ9}BruJM3Y~(X64HH=mJVf%oy;ZALb&>CfhK5eF26*th80L|8 zM0Deb3~y`?n^##+>1-2(qH4#+s#U9sq5xM~t!-&r03>B}dRgkd8_^oZ+6YzbuHTy5ju zf8UdQI2Vxca&o=``=1(q6GSogl~${LU%I+-n->VMClK#R29cw*-M($zx>}eJ$)T$5 zn#qGu&tu7Hvbwlv{$|(fsETOZz`5Ur5o=8lP7%QnGLQQ(I%_WWCqG>*C;(W?AOPVp z$Q7h;1>(P*U(mE{k*f~+;hyt5Pm({zzmT%J|*=Y%XM8sj5gZLwta=_tQ zX>wxhKB?0>YOzLPZ4?XgDA4*YqeqWM$%hK0W8AX|T$oMP_Z(=kCY$&YePbOidJJnRukj}*<02y4b8@YB}S(>spaN}llH*REDDNO-)Xxw#8O z@q&)qux}Zy?fr*jVt-syXz>L3E}?LRza;1i2Ldx!fxdK~dyqaVXDM0`T@6;|jhqbt zI+(3-|MWpJTl3=mrms^VLmfNc`pAMe*85$^e#7qF;)Bd5e3qg#ff zWH@f9T;G14n_K5?c`+VHn%~j}0jv^Tl!-=~Dm)5vommV( zVbr_PhXE3TgFh;5UO%toLjM4SMESnkwb}!qLl(HC&KZA~9h<I@c)t8H zAmyX-Dr$V>QZM>nS>q03-IKZ3@u&iqEd4tLVHY-it6CCO@S+tlB93%Uu@R_pv55J~ z&PO~3O)bf;Rp>ma6gtS;9wn(w`$+0Lj)gu)`s?4SivBB>Cp37+pfOLLK8=P7dQjZp zJEg}U0Jge_B+t}05_Pv>TaSTOO9NZg7fY^%sQMaS62Edrc?j-4@PoXfEQ zPOJA+*Iz@hOKCt5_zHgCZfqDFu~W~ zvIh%B!tkW%3pKgs0C#)4VKL&d=Xtc)c|)&=aRk`vjG}^^z+Lrpp;Xy9lV4x7#YEk> zjJ81ln}Yej{r21HJ;Ggz3{;&Vr|5K!&v;~{4DALwE@2|r-_HX5Gpap}zL=dYF=D;=<5z<{KNJG+gzeYnwmC1G7tml?HR z`;PnZb=`=bKEK)iykc%2iw+-;&uDGi=eJP>)0QxAviWFz4#*UD79?#9y1r<45dkSzZsCCdCW*0 zK6cySMoZCu)H+#!Ok};{L$&S0bZ!$KOyZL-loihbPE~3TK|RcSRmfR%#@SaU|Ga$ z*RRh){CUJC?2H(HqLNxs%Fht7|MPWckbae^t#R)|zos2%wPZ1>pf-VBOhxTBIo;!) zCqb}ec_aizR2y#XVmpOnFd@w4b&rK_EhM!8bKb(E3y{g@n$7aN>&+$o5*4wriHYF2 z!rLHj?O+?BTm;j1iP}==yavr3CRTht-xj_h<+yb(0CCa9T(V07cUad=YQ8|Mj{+Qv z+!p{SfZ*XMY&c2?w}W;HLWi3lfu1~wMAtcyRK%X@L_k1h9Ok&^yyVZV=lGF_EjCOa zNpy@LRe8upE!43zT)AT$iA;KrVv>tfQ+@K9d-cjp37KoV?P_1lOdR)*YH&H-Ior^3pUiSei2vq!S;!oPz5)4hiXk|AV z0Z242oJ8d)&j^5TR46rbWZE6OcKxt7%46~3*}Tpw!ZYl&X(6USGHizJERMU^QB_r0 zerZr6Gv3et#fm9YY(C zZ6u7N$h?lwb%2*=Gi~Y|fpf7G_6?vV6)+HlPdp?N*W+Tjg~Ok9|G=8{>*H}G-6!|x zO4zXQ-IM#18rs5ni|Cp{_KDb8!mUoh&d;pK0^A1d_Wnc zs$(vMBas$Byl&8S8;8~`Kcj00wsKsLY1+5ET}CoB>-i@c;GA4r^I++GvFec@k)Z9@6F7A?Z=z#R6YYLd7OL`@2JsnI|;bdX5z z?~BkBIK^P2AlT&3hxx%nQy?d$mV^VBc+pL@;T0m^p-J2J*FvS&BauGyr=Ny?o_>0g z=+R^M^h=mjL^qKd7|$u-I+o4Q!w;nn21^!RaHAVv^A zWENnepkmY>Wc5#94)ksbt7!qA!5Qhs?7zVv8s-)jJ_yN_7U*0A&XFn<)*=vDkwA+# zKT1U=9S78brCuTnCGOkm)9h3?xDamBaOkhc0&at2hVP9-foDXaxcmd~{}#2xvhtZ( zJz+CK=r91}R*|jK($;hMoR>Oz}DN;(GJ0a z6}cOaEF@g|(AV3Ep#ZASM1?C%G(h*h*_-YRu458_2vHb!2}%Vx zI;O`v@SgIo@IX=2k;;=?FdZ`>y(91ODz`^2N@sT4ijG{R^Pcv>fO$)muYFP{P|S|oJyU#$gOJ};O5V0%>^4ml@f^2x zR!*A3tLQcrG=Hqf4(YAmPR!BR!dAZYL~&rbgz-dR-Gk}z2J5YZ%v-)n?2QQE+1tqN zz)u}uK&ip+R=H6?Zo9po(X4PM@BE~E?mF{sQ@fwXLgdfaOEEr3;v%4xS}NfRBTK(V zj!1Wc+dQp0fN*E1LiDum)OrE|Gb!j2v6JR=_z9NPPU?JswF$J_y>fS;*jz-0a4CI$ zoE|#<|0lG{{HRpwz0G=XBj<>AQ)`B-GVHxe$Oxw7b+A4Us*b}3n0O%ZuEz+XED5(H zzJ3#q;sh{rdcMcz0TKlQ#EBwM^du0CW@bv4_iRK0Knqm$v3`R_JJ*3Z=gZHp4?WL+ zb`G8Qa!~AElO!c|jqdN|ml}J0P0lpY_Y*k>k@~Al@PSz@tk8pGxE%h292NUXlje3I zwgxoVCj=~_iy4!8O#Uf3V$p4`iPj>*v!GTnFN}u<1ueAn@50h!&9Ar;K$ z7&%@?QX568ljzIlaC_XXsR;XATrPBPcL+x{^ea>&=tJ;RUt}+vBO{gb8^aSGt^~%O zVK6HC_Mu;TAt4a}7=3|*LD#U!9LPN4Ob4v5xX81{pJUq{Eg_ljdDti%uX2D4DLL<} z8P3tr^AePsCL{khCI9`i?j|$Hk|B+>cux`iqLP*RhObwTpjOrcRxD;^ulp z*C=$}zsUy8XoiNq>Bim23)*0RlkcaXN^V`$?$9;^OS)71Rm+ReR9LtZH=m)7M|jgc z7sdMsqR8m@P_RjiX`EiM?vL5V2%jb%TpP&EL2|UO>Bf%pmE`NiAA7yu82S2j@qBtP zbVAo9lN|~#xM!cBlJ?z+$g{6$3%^}-R#YsdI}(%={XRrZ6dyRe0jf)Lfg8=X8O(mz zr;_oG&QmVVI_K0VW?pySQD&v?yIMOyGm1J2k333PKGAF|iXtp43lVNtDnkMqf7h5& z1K5hzEq)E%M=Y5^2c?JL+}(5OxH*B{lHrqz#ev^ehc_EthKu|8<>ckXQ~s5UQdz9O zqzeFhAG`ib2|j7Qzm{pwcPWv!1lsPo1F-RLF}=G+6?Xvop55<7g^x{_BVHsM? z($^MArOYwvSLho4#uNT;XGDEcjWDYPOLL;%hkPZ}aJ}@#lUIwbyt|T%e6MxF+Pi}X z4-O^5Xmi$4A{^y`d<`&&z5 zDC%s%Zz)N%z#$5MCl%U_mgev1HFvKJ*T`r{J}laD)K9x6+Fu(8G9>g>BE;;<5-OA| zbUj6CHx3~7NOo^Zv@c(qh%FB?*l;j)XX7xX0+|3*DKi(Pd;^PAp9kw#*;HHN%B$6+ zA{v1-xaHlNHXKpv$XiHRjVAOJ-M?ZaL~cg_7RiG zSn|lu5OH(y&~81Qym6tM+uaYrYuDa3`AS2$KXRWnZfcIDRKo9$IXI#bi$08?@YaU~ zwo5{j6m80dp#DZi@h6O)Ts_{hiDiNLB@+%y44OccvoZ0#d&OOI2=2fuLJxj2BjFv7 zJvYc&f^LJ@eb|Nk;7%F>r*UNm=vcB{NJRaR=XNYZNI0dldso;qmq@Umq z(o;yJILW={%q4oWy5teedT;xE{I&Zb;}Ixnl}$N=21v#I~&ad==AUKl;Mf`Down@m@{Hy2gG##+i`1j`1yD*VZJVnNN z!n`I~=$X<(ds=_U&9wk4I}sIz=pZ0B=WeEHH33*|eTsN`>3o}~?-C;xZ+!X+uR+kf z^mkiyC@v=NDOF#fK*-hd3VEK?C!XF^LlwOM7EfANkudhFh|C8Ep7d^#y>rSeu2nDPu%{Rx|1>`DgMKs7gd5AHbo`VZm zBpE$HPok#{0vz(zC!RKELn%fGUiSxwwn)Q z_A{kH3Ae_vXlZ(8=I$)hYVC`?ZYjI39R7>aSH5Z((j=r4wa$dr!O=^;&Koh|=y(#f zt>NL}PXl!?;TLnipJG(0f5OzDzxEFe?Jebw-2D*ATd9NSbZzO7HFdishODfT{Qj{M zV1VV3{iJ88w|qqpinW=uci>%V7bB(kN_<05>a$~vp1Ni#W>bKXPJHDGHC!9!Txazy zhrh2LUz^&z2g_fG;XLb=OhXqTjAvYk1zAC82-6#8H)gN@g+?(Sz*lz|DsF$aIOE|o zd@LPh)JfI-p2)5{+uF?O2+nZsb|^mceDt4LN#ldeuXH{+L?H&x`rJ6lZnNL?-8T|< z9_ETn?*C{!?ajk4-Arzt5jvpcX7R_6ixC5GF9&g~vmivySRGHixcDsSL=TFEYd12y zKI7$dsCl#=^H!Rlzz=3(+$REOjZ9N{7u}q8`VG!^oPcB$2-C)BHg31}28Nwp(_%V!KjX3fdyxQPL4yNrn(Oz*`oyMYLOTSQ#6*?(?Pk8MfhW zogUh~ht?H~A~%eN_$Z`DF~HiM%Yd=$lXp8k9Fht2!2!9;^es$(A|I({@*# zTA?ZMUeem9(y2;oZybP=IKuH33RlW6iR_vrb+ia-K`mvQcI`4LOmNprCo<})B!*P0 zo;;djV|g8*T_y}s>{CtM7W``{Prb0DXt5Vgkm47sgQWw*h@1VNVLEChNj|{r=wUpa zKm*Nq;HIlbrw>y6q60_GW;xQLmqJ5PE;y z*uuq_zFUv&XJ`P;xTb0icw(=>mG4B24*w7+85}?kPA%MSA1joJiN0BKe)`GGxBRZJ z=a*9-7fN7nlViC#DO0qFaCe3gP!v(L9NKyu9$P<%HjIsi3?1r#_)NsOJo}KUH=gcj!@i$-c*N$bh>?e8>wf*EOMJ& z6MlDNuKI)dY2e{JG3nsKiQ%3bk0WE zFjAhuwdrH>v{@CE_n&uA8ntZMvPPXc6TwE*8WwB}N-(fc5VX?<>+OQAqz)J(&?qwB z%AkJLYSpCp;X6Z>Y0r^VbVHEXZ`fFhutG5RCKLWD3L>SHN@?&7bztPGr-1uhgqa!2 z7qV|0qy6^AWyoH$1;WhI+L)Re1C6YQ1Q}b7RwyC;rncaY?T2u-)>6Wh*ROtRB&{a} ze1LV~38gKgBj%MY+(Q4Vnl?iwE;~{;!m~k&=tl|`K@_? zCRIWJp($AhsW%e_g8<_;o5^C38RsO~BM@t&;c7jSeniq12~=K+Rphl_zAV~mU}+e4 z_b>pW2YsE}1lq2}tKZ|KNEb(PdO;c&kNsVWpFabLewwa~TJki^^^p3oMBz`aFXu1a zM4hdcIW;2cW(<3foLQbgT10-lG=tQ^*b){$HB35VkdGo=@VPN)}H00 z^bn;O#XLYF2BX2mgI2-NRS(90c_}gT60#nsucXT4>83 zXRyq=ThW3i%E~1&q*<)7u?4~MmX7~tGo)S5i)zt0bpT6?u4n&@=Zc35(qvp| z2;X7O#tRlHC89V@X(RJ-UoI=QIQIWs9_+Ie{)rluaM9pe31JnYQ$bjTe^qO+0~8EM z!3=luQ(;}z<(C2=lb8laPZG`%b3ezh{-+S%YTXIGbYCAr2sB0BR!FPJM`!GZpS@jy zjQg3+>lUthF)*DAq$rAJEQcaHyp)*%QmHXaD}*BqO2Z^R?E%4m#$zPnK^b$oCT|G4 zrQ4Gn@8_46WXfoX1<44yZhxk?p5*QoiCV;LZ zl_`uGP_lV{B=qdk_lITMADsQ0uEQE<#=*f*cyYTtvQw&gpDjha< zLxaPMu*2!|bK2jt6cq^(g?+}?yIE`Mw&z&ZpN&c8&~6n<3ZyxPh*WTS;note+t5x@ z8jgB;dK%`YT-?@taq7?h)O|+x9#E&w#yJT)_3UGOx}_;vAOB73lBY~0mTlzFH8b%$ zRf_@Al3pYk|A{uS5yAXF!n707$HNnq+5 z(}Jyz&uFl$$c}bf5`n2_6Zsa>$9J~N5Y?S|v}mCrzg4Ry6K5?JuFD31w~_8~cY=g#RT) z1=;|Dq1y=w2>_%~3($lMMMg?_w%`eRyfZX*DkTX@9Su;?rJbXh&esGFrE%Y-AI%-4Q6Y_`%kXiNIe@s z4$|P6##H>2}eboLC@rr}b-k8ELP z$j5PG$L8Xc6P-L$2kflcok>aG8$F!5l4~)1d##<7)k?VgNn~1gI=3G#fGvg4V%McX z^_|W%-SaaQn=2=i(jY0pDC7#RKV(-$@PyM3HY0Dd^70HB_k`iNwUwAcG5#o<~^Mo71@V^x>s4*QsyY zPa(&?>z-VT@)e@r?s1M~&+&67W%30J{LVwpX>8}J6G6`8w`d7bYO3|z49_Z*XR_)0$lmyYt+1S{e zF&V{^LKi!(sYaWy-^B5dNEAZS)txYuxGl?T)V+@=b-l z%8m5SKPw|+_VzwCT=bP|BM}(DJp@a01#k}Ofrw(^Qo8ibv!JH4+4#SU4#HbfwCex! zoq4~Oog3A3{N8ptvD&*e>Uy7W!2*%Zjhi=jaU06Ne*E_U{Mf0>phj%6`eTo<|BtUX zfvYip|Nqa74~DVKVrMYpWGQQ9DO)pUa6%EuQW%o#lr7ucj2S*-%?VjbC|jv0Teg}R zLq$cFWQ)-vktK;z{hrr-&WX?W@qhf!<1uC?opbK{eZSw=^15Ev>)IjcB1Hla2a*!y zwZEVoRCqm3Jtv+53EtZ)@{4yr35yWml4E%JaVgNokVA%srT8OEFQ_iwLG)>8|7-a8 zf$5}ZyjEB4IrS^4$D8ShtT^f{_%Jy~BK{(%5Y;fzNL8dt{syw8@UdHu{A@rjXfFw0 zqd#jA&i!YZ;a=RYdZ+f@Pl{l#hmhHoCfW}XY5Yr5grtezZNRvDZu;K7u19>(>INOc zAj?lBjVl2J5eYS+)#ycXo&H#Vi{Dx)NPoLvM!_aJ${2)AiS-Rntw%NJ7i4+%;8}VR z%QLsOzMd6kTHHD9jgKEDjwiZV-EaGRndB#%$>xH>AAt|DzG>8F>osh+AFC^cv|HT~Gm(V#i#eZUi4Kmm zcXALfJ$2G3JLb-x@BGJCW+RyOw3*oq^ovOg9C)ea`&w$$-*SKXyHw=~BXz>y`cVP- zS@rSiRrRRX=O3fhjNg5)Nzwy#U(c%j)xaS`j$LZD{$Gxo)Z39clGZr#rIPDVJqs1>ng8STis{Xh9|D1si9x`}AeZ|gw_*Yee z@*3oJY^!O&SQ$e;GvDd!7 z(5Z;X?FTD^txsELzH897}#)5X#KEuzl{8Sp7`3PP`KIj*iI;d4@a%iDB;cfPGib(1VztPfYE z)01u;fQ^zzI&c-$&fT!;FWtD&HFf&`sAc;8y6Zu&wAzMCu{-DgqRLyMg_auKkKd1g zfJAI;A3eKmKj=JH7kDLQ&JOS2zrTnalek)c+Bh&UP^`6RQFNyT{RVbNg6&TmYg*QQ z`|h34L~Iwm*tOvK{RoclK?)ao>F<=Pscb=%#Yj#&KNK7ULKmnTN^}DQLvl~2YA)S>0Wr!9XqL?!m z=nTutpZh==4gcf&w{%%v?dTk~>0XYa8#Ku3cK2ZWo#UUPxyFs(041@|nNu>6#w&2Y za4^8uF4Z+B0|*zUQhFD?AI?T=T?t*|LPSTE?0(i0`Sc!ph=cR-(*c{BkDz~TJOe5n zTf0=Q$b`uEMR`9Pu0vBbsb5{CcPK!B2qFm9a4#nh&N2~%h~FelJaTL1&NJW-w!Qfn zTko8;+JopzuZB!atVQ+gXaDdj8u#BhAmf%5Im`aX&m&oT;X)X-CP51mZ^%v0`B60D zI4$Y~4MO7>+{fTZadQUa8Q-hp2issHnkPi*Hwp@CF{Zx2JL?!E1W!b8sT*w6%C(db zdpfJ21__jy!y=-rsSvg#|A)3IBss2ND!7+NfCx{={|TPgMn*Y^RuBl~rGI|k&;;9P z_;Jq?!r%LI+4P9)}@QJ7I80HM=^a!@uB`mDc&K*qCf2{w0Vq!O;3itz0AzDgg*p8xS(`WICP zaAA?GXfJt-U}-FBd()SkIdev;7hocU34>)I1$^=Ef%>|fu8iLXe+X6TeL-DmC?)1i zdOb;ybn^mhUrPMr#OhC4#x=I_QbGlFf0R!VXb7Bj%Ol^Z7u%`^C?bf93JeuSx8WqZTtnV z_%9)Ax^bo$z6@*1f!R7+OkSHY&drUHA?7HU((5S+Q7J_X)MK>w!@amP@yJ$Jw7~95 zX7x#xY6JCOef5>FAMz-68h87mE6$OwquB(T$>+6Gse8$$8KQ1Fxde^38gm$)|GT@8 z29F}gr^{=xZyeHUFMpnVLDGCK<3lL_&C&y0X65#g%DI^hEKp%@R;gDxAR zll3yeZW#3iVFOS+_<$3|*Q*R#?D_~%l-3I&FX_dC(QuqDpWyN0<;$z&e`}*w5g200 zw~taI>iA**sb1ZJ=#+5R(y(9D`yc#$Wo(-@oB;MFO<)ug$f+Qk437LvGhv??ad>h1 zP}0k3NL(9lcg}wt9F|@;?YyCfUpfeOAg`=`(m_G&V5S2bBB9<%@G571!f_EBgbAz< z#fmC@5h3UjtL{3(8~Jn5;SU7&cUMF|l>}v=UWi=1U!MP3zI)LZpgj4432TXu^5`x7 z|NW4`0Gg8J>YR4}@Vjtcy+8j{(`)F3bxPCyUzynt)c2uZ5+3{o1G6iuAqg4asz-p) zX0s`Wf3)z-yK}v>2CV|wyAjxtUcd3n0#PKjwd&~n{E>=zcKlQkMIg59E1Ln4({=++lB~~u#S82SS>7 z!bhSTaHFZvWo^mk@8ulXJhtJD`%6DX33D5+Qov&~oiOB7BuJA`mr1-~Fa!tbVQm~D zjsB2$fGG9;=*Ea`L(mtRdpg9%65MYj_zHStxF z>9Hd2_0vk{o1`XkPB{D_-T*m*I-@%vWOIoUGR!KsQC1UU@<+rVG%4lk@hPMJig;KT zit2(L7aIurasB!T2ExY0$@Mq;Q#9s|)@{~f!nkkcfV`@WTulcc4;sk2ovybvi)Sl} z%c2YyGsyHk@vS8SekW{t65Y#2gfOcL{&#jhtb2S8 z&Lo7`XXe)ypJ;llCfI2Fj1&tqaOwOs`D)Gj0x}5cL=b@-V$YFdv_(q zSzv`8vkZ87YH9efpCd`d(9nTjo=*N6mFc7C)jaLCp~KxK`X?X=%0x4bX29Gf#wQ*- zx9Ev8S2Jh+pgqdTjfUOP!~_u=Y5O5BG9(0|$1#&EAf=)9d&PdO;e` zB=sw-$$hLPQc{nSuqyq-)dlKt3IW?6NFFYvD>M+YBnHUf zkn-|fdKpQXr(eDB9q8cD?KZ{QBX@e8SZR5ZdiBiaV^0EAU(Ff!3DM9$6NizD0pgme zxqKaYcgq%!ukGI;+5|F)0knQG((|uL$uB91f8_qx^wrHk1`Zzt7?!SKlkLI9Vn6!d zVW#)2!n-Wxp;28;K1lfzK9YnA2Qr9(b*_YFBs%u1!h z@*Zv2P3Z>Am2X&3nEN>5mjQB%nyi)3ON5O{X*nG8?h#X^Gk*O5PPC)%%`i>80q`v@l(2w==<*?~mBolF7*>a@m?Y`BkelQlI5O213mF2DBqBNSw(nVH& z?Nzcb33;hiQbfD{^h3s3VE4h4LZK{lZ_c@(P94s2`B=6v`k$eGfXdEl6NK z!6W;w{WzJP$a2y1y7`}pNAi22T5}}-t=;$1flDJq9vnjEAe>HOlqpX2*3z9Kh(HJ+ zZD?5K?7X~#l$|olmKboTlP7HFD^EOm_yS-8VOs-gP_7z67e=>_QhHsb=bbYY)Z8OL z)3PB6gp@ZuxoC7uC$5}Hx-Fv_=+pnlH8qBu1>(FYx7*v>Cml{kNCQUORl*rSwtGF- zgV?1wb!Kv_QZ(YsqL;On!NIXbYkFU4Lr8@LI>{Fdh?rj-DcKN$o|~-kr5lzVa7;$3C(kXoYa^vI_}xnP5=tN0 zj~;~}eu9+$Y_8&r^k;~gHcnP*2-*L>DprC5`zqsL5EKVZC!`eptCifz5QQc!RGDL1&ABw3FDGxGbtDOIC*x*7zT}oJxnppZL zJoYi@QK;}Cr;A!+fL5f&;7nWPoXqH?BjSwP{Nt;P zK5f*ieAN?X4q(`Jw`zkacXs3J(*oK=BPs<;gWb@$!G4r$!syXUyY9gdc zmiPM7w+$#`KIZ5FloGT$b^#@qw{E#4Z!39T`zQqs(`T_WAbtKz z>)8B0+@oFF!d;h}FIuj=EsXU#cIlIppMxJ3l}vvg^QOGdY}ZTwd{sBtyY!h2(x zin7d#yq5j+fn>#vkszJyx4YwVs&ne!>IZdvw|_+~o!6D`t7jknyH5Ds@rxe*zKyQb zDaS{^g({#o@`-JcqK+sJwDC^E4L4i-c=!Cpl4%&}dxfU~>j9i!b?GFs${o5A( z4O`E*TKC!a&)i*bXXfV8+THJV9?)#=dHc+sV9ELK8_V5L03H|n9583SP0rTM=ZX`9 z&z!`gy29HJUc8U;zRd$)rHXm6Oe-b_j82Q`+kE!hobGYkz{jFq`*0n;TRdqPPF;Ae zFi6{??A>K*;^X%BU7{L(-(!Co&Ek~@)HXIE;=0SLFFIgM(4MyUdF|%CPNoFEq^W8$ z5b);GsO3fBv{yZ|&tw$xJtsFex7fJS0_Aky!dQq}&XKgRd*7R%UKX6&htZu<2TlV< ztiKZTs?oeXN=lFwKE7<4-kndgVHSHdeDGhxiID~4p-fy>g8NnsPkjbYD1-RVqL`>>*|h1? zEAnDp+~zy$-Q&vM45EYP$#uyOt~h}Pn@hta3hz6qzI4hgyJy@ppz@`)g|qQ4E#Tbi zgSifr0|L2!ogSt4PW&^wPT%)09D8@4k`l=9dP|)pR6X0Sa^pwmF$^vV9Zi|YjbL)0 zpK|5L8oRHUk}I$HG4vP()-UJ@T_bm4I^&#edw0j-)Gz3C^1-jR$_1sO8;vZf?@PL$ zd%5oYgX(<~2t53?RB?)?Fm4yDBL)8)Epyl@bB5oOyStJ$#y}kb$<1JIyYz?OhyK zz0vYl{=r%CRq7rp%)JhR__uiohYzkyE+FTjyM6gOpGWJuOB-=B^c9QB->Dz1W{&Z< zhda+ReAAWpKrr)vFQv|L^WD9=ectYJNS(=(w78qPqkB|J@DcL`w|-vr^yoE*&7}yi zbOnu#y31nS(CIO|mR-Act(_*Hv^3PUW5XcBXIe8YO8qZREW{dNW0v2tj!VfME`q(D=L@D%&^|*%P7JUn< znF>snzGt}M(>*R%p2u^qHb)(rQvns_h{N+T@{3e5%KX<)>Faee-)vqCHS2(PCQnwe zobw=_M&0H%96Fbh_g@ZZwvp}XQlJ!tk2ljr(k<7XSrc-0Vwk^v^Iai3z}MM|mrgp-U&--@wXw~1{YD`v@%R2(^cL-9ygQS_ zkE&z2dH%a>EDbx$S_fIf`wa-#)6>t-`B49VQudx7Spl5;EPox+A{+A;SzNf=%od7- zd3v^&jZ6ryNX-Q}FLPaxF**EooQ2a~gHY;t}ONm!b_>@fyOa`Wopn|Ej?rhEKw_N4R#v zN8C>hhR~Z`@+kOt^+QUm?BH#T2Z6B4EQW<~rT3UvUq#HrHNKJoWIcp3I*Z}jgM)0l z28>%gwSbM+i{;bxX);*sX!aQyQx5_1Khl@A0FuC#3K7Y-0j(&@6=oKEy=Dx>4}%|k+i7z{Yi=Yrcavmr`&2KjZ$;| zton}1;m+H)Z{x`Oz#<#}Fcu@0le0saoig|0Hvy*$PKCcJ;-YO?g#p>i5tGTg8(X-) zk6ch*W}I@a^xpHFKicd}!02p-9rM;|$8kFRUFZH&KZ6l|F7MK<+!Orho3W4y>w0(2 zUP^|W={DaqTNpXVyQl7Gx)MJp07&+aHEQgQhC%QpRtMcvj{S+3vL zH3|3JRQv8g9OH_|M^@eCfR5G-emIItnvGTy3Zch?e3hvKb@}eGzXsvIN!F4_}?LSuq$r! zB*J1hj&L@a3>o36(^F2~>)J8>PVA8Rn|ANpx6i9AaeNicdn=NaWY`+|LvKqBiLku8 zg)vC59^GPt<@K941rRP;Fl+cXlW$3QmrjGG(=TQk>-x;>`62rhV|&k4-6C&PU)Bx= zNA24>-#fU}HM@j*Si}kM9Z%Z!qa6P_WiIr^#rzO(!K zXG4L~GrC)byMIy<#x(!g-F>!6FE9Y-yvndC^+$bbH~b|TsY_&E8>N74c_mOZ5p6Jg z-Y7jZ3^J^Q^(UsQxzzUW)!31Z9gg4nPWOr(oyEp%d-5~i3y`BG`#j#gO0N|oEuo3} zpqt#6ppDZzg-Wpvt3F z$SYJq>$w{8CTkDV&4d%C+B`dYEt=S7|5)Y3tfIvWAU;JWce~>rH2>zobnm9ODQu68 zJx9Dzv;8N#j-2L3EYewXmTFh*=?A}l?sd7#MXipuo?s@+dEj!4M^CfP`JijZfHmvY zJKeP$H*dXT=JC;B*QDRKvjCVKxXLvtpYGZ?8Z&rmq4Inuxq2v2IHW{-dd)`7Yan5B z4591O*xa4ldd!_j;cO=q&9-m}mR%jKeze^qt21Gb4bSO`BYX<4zjDnTW|%7fHQH$3 zdG2}$=eWH*rN<}Y%4Rb2jaCG*&e{?$cSw;uMDm|bkSFE{LwbcVWbf;29&jmpPb>9{ z4)hhN?8U_eL6Rhdtm~GvWA-^Q9QA^#+O;~u^JlPfCz^xp_2LpGg&1KpAiUo$#QMW! zrtjM^YkR-Cy{&bYG;q5zx3g-)Pn0WL?{YFWf4}nHf;-77p4lPIBI=A@qSD8PE86bs z6iYN~7^GlVwwP=wn%~}g{CGFwT_0+z!+Ei8RmTGRAJ6dE@t%m&QNOln_m1L!*Q{fq ze2+b)q?wBO-$7K}QbMYf9*@>4#$D-oW~EDg11N1*#^i3t^}XNR(k7A^d*KM`x&Fs) z(wvw7G$gftgPBy9>gXtfA4h%QvNstL1spNxMOshW>&4aDs?Wx^q&rmO8K<8e--hk6 z-|Ms-DbXoJD(!7Q_x<(E+ke+68Mxm<;<|m1Hht(76(_Yv+|aHs`o4cyxA0nUxwS6( z;mW2Yq&sy4I_ll<*Ik;7bDn1iZfb{+OgHW_@kc#Te{QJ1LlQi+nvtLG`b^UB8f^mo z@P?gruUM5#!xG>AsiE5@Xu(=VM>%9OCDDeiDn4sB>EoyG*@zeumDi*exDhjXv@cAA_Uv@nJAb_YeK z`ay>8cw}|gU;^x{lw#(?w}RB4K&4Z!4RH0+ohv^3yuNGI_}gBWI-JO&5WiTjo)1m} zg*q2?cUjk=!0EuTU7ezC(_Jq=T8-&3$=xo2l3yc#*Cy6o{$d4bYc%2Z&6SehY#Q!x z@b38eH=L`H6)BnrU)M4ukcTY|CPFjDm#o90q3fLQ_3rfi*%X|7^osbULDQ;sE2N#* z>%lM8Jz(JL^Z*ChO*7y2KQc_c3i)HmT7%p4ZSe1z!jwx;$pW>b0ru+HWcj@>Rw@?$ z>8!E-jWHjojV;{=-dFIA+Qjdxf3wLV^5h;#34C+J6&kZ!&Ge0?kK02;=n_}ryuQc3 z|J5easVo84U7y^bssW!24MwcyrNlPd`YcYJ0QJ_1E^rM<+fph43ZrX72G%KK-tl}ob;Q+mC=($9xD?#YqJmj}Yak~*BY zt6x#awyiE25U1Jv>%-ngwV|4Pl-f^WRe&K zl3Q8PM&cn0(SCGv=wq#1h@zR$p|dgl`Mml!^M=atZ3dddjWNJ`tt2=CSW1hT+-#BS ztwoG$>9+9Rd>04Xyijc-1Pw2ehg^v;f35m?&B!Vf=Hq#Dh#QlMIDR9uI$e{aCr$Dsh0aWzpTa7xD59P2R%VfM3|f90tvGiG(Yeal>{J z74yycG*(2+9X^RUrs1wjo*jGlXWiv~P6sz-g#<^QyXMd{8>4M;vW~CwOUAuPuqx*= zdwQFoc}@j1qKaHnJp0WzM|3MP$w^;|!TuMivQ z;yU*yTKx8#xa@W;7j1fhH1uRlh z5zjGZ%#^_lrSQ5(&eL_)H1jt^cw+rCeqQyBe(CFgrOL}S}L3UwEufogiT~Bt_MJs~rkki;aT5DBPy_Wk(qHH&%P3MXtX4enmdGWG7{E-_N z+M?TS&k!>ER+Jf1fT@R@ZyuRf=C&Zi4CUv+2S+3kfuv!Ms+rFW1W@y)Jt=3;nvO}* z;xzs_m_1m)nP!WX$GbZe1lO>e6pbre+oY>`-_nfKSNr>x9eh>v z6YDjqGS0j5v{sb)DfOy$of9xdrzalCK^MR+yO)Q;Sc=>_nlVm1OS#`P(>Wp1-R;u>;KpBf#x{hCHd! z-24@~O?JeI`{uUlqWB(5Fa}dImcF|*KGJnVi@wV@-|rA`Ne~ZxD0{5v_KoYTPa9yi zMBwu1{K(TI&-LN|);8+$-5rIPLcWll+aV6LfVyuer>DWFT342eU6A20lJ73f?U|`# z8btJskz=!gb>_Bq-0A95@cH<+KY-Y8rV{%da8Reijkw4$>&c&b0%0t9LvnFBsOO2t z2ASZS`u}-?JHf4$r&vjy9@M1UiMw8df--|h#pTgz6D7_k>Z~?(#%f9GrbouMF23vS zNOAJjwp6hJ=)?>nbCK;YkZ%Wlu` zg*}a8_2O?^dtOm?y=OY4qy8ySmz*WG5sp)y?56=f#7}KLv0g&mSJKXCc=b64 zLe|?u09@n1_b9#%@`(ylkd?6X&bh2j6lm)Gi&cHwpOkDz<6C+B#tASapp~vqNwo}G zjIQUMC3Ndtz1bW)eU@x(_E_eGb;+Q+pId^Doq6%s=RQboy@D|{h3sux&xNyCJg0fh z?fhk~@%8K1-Tzi|I&UJ%#e!70>G`Z~M{hBxiVUSeQBeLrLu7 z)hUJ*dRQx}qwy5oj)eF=zJXXk!(mwh>;ua_A}X`R05W zMMqOYY9YXN(6#C0QC5_&1_2%zBLdGaGMW2%4cc4ayfod)25O`<#y4z_!I(U`HtY1W zEp@(8Y>AXN?Iyv?JBwtLOXNC3Fz_N31X)mU2@-gX`OM@I(&(6~#dJm;(l46^Z?`_^ z@Z4ys)nC+ayoh}A|l)wIi}=LNMJzE7)@e6Cu)y6|K6Vaq)L6%%;me%wetpZA#G@?jam!GhSwpPo}=u=p^SzEs?Q=@xJ;}#)EK@G z9q_ktE!5w(-o;0C+#l#=uMee|N<+^)h#jU_e`xi;!&i4ye^LFzGsOW~Q{eoYqjh?T zJ!uhIG!AcRESUJ7GWvW2bLvWb$J0-v1rO2bCG1k)7ZV>dRBJ*QuS{)$!ZYh*}8+{IJOp#&E#!%ay;| zvgwS&-xG=Mlx{wq7ok#4lu|aG&{AI4jS#ru*FEikqh`)J`EaFp0aR_=kKdZO#^~J- zC?WrMeKpDVfHlN*?y*uo;lSyK&OcRh{@GrGCaAthvn$Vtb{uU>jWs!S`J0$CB=bM* zSKoH%I8@N`N4>&B+4O3tP zH@5a1TK@K8-v-7wpo7y#IrgZg0^}6jJ3R%nmq+6eg+x86e0!*7gMr)&sgyMcQuor( zxfBa>k;QS)&!cqDLrmB*R+%;0xZjVqt$ci=#&+YdQqKHRrzaB1k?>YcPsEySoT}bl z@T*5HDtaJU>7c(HXk!IsKN+DyqwnzD@{iFi?~&r{=6r(t?{H0j7$LQn2XRRf@hw-% zO3ZZSl(};77M%<>Uo)nG(S)ptW)Dn8&|Iencs}@QzVYXijxzN*{zvq4A|yMj_+Vfw9vyboU=OeSYXlj`1XB7Fl-Ocvvf-Z9&(pk! zGg`ATq2hP(f167MHW+0%jlpj?7xjXS=l|cz-`&1_JAP?I0e-RooK9X36sZ77I#%-L z*$ynpNkcf{+aThjBU*1a5YaXpZx6lu-}p2!{v3|o#`81g)Yw_EW&-v5Yb9WmTGx>7 z_l`bL%;hV58%(XNu>UlQYz_4Sx7gsW(?>>9CI8fbT|Q`avH6l2a*DNy(51bhhDgQ2 z2>l!#Y+AsnqzO`d(T9#OH`8@Qq=%u3aS83^i!=j?O!?wX1jfB|#Fe1A;jpFeF~@wZ$EXCMC-#0T?j)(rjAV`8GIs>aqGQzTZjp zWOF_o^uEw}7R&F!xNbPg0EShB;Ob-Fzqz({Uz;+-595q4RH+E&G*{Rp~oLY@K z?|N*Y7<)!nzNNcD*>@?Ng;c`D9W?bgoD@Ic$32e?%jl`M z$8owciL3HeG>P96TOq-smJW#77dEsAWc!gq-6g?rtKo*kz4E^KkN7Dj zclzO{FB%&cS^bfn)yvbFWgQpZ_nBfmz{xAst!DB+p8v{%QSUTkVhtBrYE9%_ z!NRz)Mi;v@snc>YMRrC@-7ETS1i3Od`2AgZ7Nm<_R}C$s>+T^pcMZfxIv(3?wimcf z_LxAWybe9!G+kcbvtTR6y0xzVhUP(})CL?%bH%-OWvbRr21+e2xfa}7$K9TLmpt1; zM5v4>IDU1v%%z2bMiLhd=@^^!)e6Hdn10|RJ7Zmy5GoZO=1!U-+UQbx7vHVbqR$zF zo$)Pk=@>u?u{vc!ZStC)c{FPq*L%pl7h!uoSFiWxKGJiwE){fExg8c3=1Kr8%D?)o z=t~_hG2ziI^AGhme_sq9FQ?}WqOm{iKU}SrAv(QY*W@59y|NNyJ5uLJ6wt>R@)k_z>H8(><*h!7rwt{5gy>A}C}lY-rTo1y$lKM!WD&Fd z``(BE_z0gbH6jA<GY&@N^8*A31?rcS66SURGlAR4T%8+GSi`em?2L1U_?X* zg6?A!5oMl1RJ^`Lx$+x zF%P4`hmEFoXf1UK75wF^#9=Eb9YkIoEhtT$o=*>J6Y=F<2q^7$?HF}+oBC<%zQ^Et zgNNlvNsqBdX>2lM7`5w*z0uFVkb0(Oj2~%UHvsV@VsA0JQ7?Ugq zGq_>IOBIMSo9qm@6De$`sB+`ZyN}-3Mcydi-{At0VGKkTZBfr8KNW)| zoCwe+f+D}7RqAH`M{kS2J;k_5x{wHsjE#IgY^6j1^wo-+D5Q+)fxC)G7orAP^ta1; zO;e9i{A$yM%GX(>7xm`5b|^$tn1;;Bp!iY6F^2c-7UcfUjF=&kwHVnYO};U0`7UZr zub=VL?1oEi=_v@a57Bnvv8ZJoklZjg?C4#?Oc83M;AWF=d4{Z`pd**%d*NrvG>_j= zX6?}W0PK5TXgmd!U;ggONmFRE)^WWhhwE7USMqa>XEVyy0X+*&wvy_sJ`0nT3MlGi z2q5hqnf9i-u>5}f{+Y*ajEsE}7H(e5E^K#L-5|T>9_(mHz$VP>l4N-PR}KSF6%rA- zb9md?{_Z>vELUK$JoLU&Kvws7t5?vZwrLUO!WR(2(l%n0oA7yzzp#BXrX_3o_|h@r z{OI$Pr2eNiM^;0A8(-hR2fFfj6V@;Gu^JQ`_uuy8-YGV10^Ami*1JRcOQw-3jdNu@ z$rIN7Piht4WGo5exsn@E9`uUq^s?bOdOuQPuW9H}@V(KkvhJf^`1W&jvE&ldT@vP+ z7ObFLPsWm8HK6d$r!5Cv>UdhK+HJ5-<5SRt&pc@ehI~I2@LDLc1~gE0EkkFEb--bE zp&sRPwVnDchg+|72^e?x(JT9PeoRQBB7Ono#4ptsic~mzWU2=w;C_>S1<$W%e6BUcq)1=9j=#Xmn?(; zR!>lP(@Sfxi<+83TaZvr$eVh1AQ}l@WK5MYWHG$K@W?e)F`#bE@qkaE>*(}g0`4Lz zM%DgvhH%cV)nQYjn~@b4_H~b2)gTtYxG*0c)Wmf{@uS;{k1Cx+K)NfRtkwIYy{o) zuAbUk%DQU;-rxJQg#tPsLzFcI4I63|a%J%YKc@%HY zBt~Y$9XVHCGF_pucpQ9#Uo$ouujs)aAqE>xws?GD{>=`s&cBDwl?J|fd41tfq5gAA zm*Cq|T6Ur`eR;hp(ssOyPm@^$Xc}-iD#xhjSBFHkKv?F{$FgUdZalh8`J1ne5Qp9I zhE1AGYnFQ{AoaX<14+822p5`*T?4aY zdX{_T?P#>R;~s)!s3z8Te~0hB`>V$SZ*QwEHH_cavQ<6P9ce7TuKYOnCAvYP5i)qN zabuV+Pt($3_eamIxWMgE4|;}Mw^q*-zjg3HGuGSC{BL@b-)3cKlIaWY3>uU4;NO2I zPOQ8s)31%LvDXmnYBbSjL-*D~FV(X0LQTKqd+7Yf>xZ6XlZ&Zi+x<-m?loXSB3(33 z+nRn^{kNalKNppY`4nSI$jJA@JQD#|17$#%#WLyK0^#}&M zd7K~gCt9+jF4r+Fw&OJ@FOm4?L81}qK0COqbQ%)(Ax?Yk8+Uk$RT1q$=RH5xQ2hN( z@7d{srrZ`LH1SpQUOyg&I7Qmif$CL-&f))uz}J#_ckb@)+Y4#B*xbgkIYLhPjrUdY zefllxv?Phc*c#itYc^12oRpSZZ$aIFVBy55Ysjoc%`SOf6pFe4`*6LoL!L` zxu#C6I$j`%i9x zsMU39+K=>DjNtl2$AO2J{M#mt-&v%QhFsjxA{_vxuW0G*P*BY#me403Id)7m(CdHk z#V*jS#I;*14|GB~m`Ml;o;m3QdRkq2rdZXdXPXEU;`VU8mNACXc7mv37aP;D>t!R- zkjz2iE;EMtt*6V8o+#;5(uZP$WzM#0vN?afp2Ncmksorki7BT~of2sSF_|B$uZJo;uHZC|lsg*4WnXBTGo%crT94L%%`IG7XCI)gn-XPYIn z$WoD#5sWB6S?Ijx%~MC`DU8La+ye2BKCe?}3&L?pTRMcJbyWDtAA<{+w}CGIak`5N0UunBCk z^d9VZ7xM$9zc$sp=3d2guwS73yV{8-u&&#}rcIm5gfEggf6iB8p7F#yxJnydIQ?z5 zOWqvP9wd0&ZaH~%;z7TSe4r;_tjK<#iz-cX+rLo_*^59Mx)EjME{zF|azz@KUJ20- zW=v-1F5hUBmNNVfvDa~;9|8KdwoulHR3+rosx^B(`>b*s#20=?PaS*{`!_4IaB(uz zkX5wqQu)YFA#-_vu!VN>joM|z?IK~vzL*8GIlm55rci|cn_#6cMg zYC!{^|FusvheXd9Xw?N8$O$cNi8`9pM#Az5zS&{GojwskCd;wyT3mTiClUFCF1AVg z43!ajLDGQ8hd7vGkq!Gmp+&Sr6}lBoNE^XV*#8NGkyf}K?$T))X;%Ml`%yI&`ejXY zUe-)S=@c%oIv_yLc)So(Bw3gS1mL^MZQ zn+96@4ghs;CgBp2iXHP(|ZXQo4f@+SW=nN_T`Uxw>k)QgT< zn$I>93b#__`1UBsn4X$HU5?<+O%%?iv5oBq58k9jV%rX~z*7B*Ml(ybDpg-T#JvOZ z4r#RL$n>}D%iD(Y)uD!phiNVBwgeCXVSgL_Ud!iImP`b0$IZ;}fbSuKDKE>@X(-7P zPUN0q_rpg1m)xGW~e_HwH-7^vTWdh?*YfW2Amls{wT+bOSZ$ zLJ;&|Dh1p~j8QujML50~Vn+_s@~_5j*thAEfbXAtTF*w4ae=KVcZ&i5-gqkbv`Cs| z9MNbTGw6L}K3hwjxCJDAI&|t3b@ASUqt7O_siMBUlg9EF3ONU{G9|g0)-V;m`n{Is zMPMUFNs2^*j%f!YfvIqqZ*G-3C8=++)cCfSDL2H6F2t-_dAE$rOgyO3q^8k#=X`eY ze4P+Vu7hJ;yzlQ+O?_W?jk^ycR-Nhg)Ji|8sQxDY!F#rcgoMZd%Ra4UPC59N_PC=2V#(IJpp=l|D&G zGX;B^oSkX8HH&^lJxyeeVMVI78q}y0MQ(tNX_408-(!Dc} zHGT#P`$mQSO%^U4<-G(!i*L>Bf2Ys7Ih7x7_zydeDMv=VeyLn+F&$6kG2r_q&`Lwa zaiXp3#(+cGH;rNbwkx>nY0ZH7awW{v>gYus0Scwn0oXB#Eu{I>7EO`s^3nfk{F0Vd zq!??u&A7FTW*ziDU1Xe0#>VKNVIlq7;h5#IW(<>GmXJ1n8r|RWBix`F4TYk}iTI<( z#lNaxC#4sbnSb_rc&}lE2?-Z2T;RO(^1b{o5==7iReMm<0Q6L`2}6i86510MWj%Z- zYR%aR8~cg$HFH9md#t&7+|j}~X*&e=@kID*RX%-QtZB9K*q6lMNYHf|lvquL)r7 z_2_q_Yc<{LJB^*po{ZCS};okTG>> zKrjfx@5LGI2sFh6mRrQi$dElIO=;=ccc#y__5L5GiD(zk%@d*u(>yx&RLg`d{YM?2 zv8~{qa=ML96jxvNX|2)d&1q&rWTB-_#_Awsw0qT<)Sf1T={{YzV@Ac)lmGZ<8$`MC zAOmjF(p~+NNYhZ-`QX4Bdy_|DOQpqJd<*O2G2n+LY$5X{OO}{U=*Ao2ic8ar7@Y28 zC@o`{o78oE`k~Q& z>$e{@l+0Jg8(m&+$hQv3bZo5api>o7mWZGJ`V)<13}xn~&&?dhJegG6zOkCpu~Xcq zZ?f#wQ=)0g<)W3XD~>B3zx3trqT_?&)V4rFbycHDn=<{j zua$wmqMJ&1yBQ@GF9Ygnye-Y>3~}Z6l<}i*w}Tz3XcB7bx4Z>&Z!t3JBY1*Ojg@&7 zJnnGv6MaEk^GO$!wO8q|99$XssGr%=;`16cHfk~UN#urE)?DZ=K$#0KVJRN^KX+az z_62R)?R}P=BFouCK0`DR|8$h;>ePQnA4aTnpdNnzB(JYDb|W9=>cu=}AKZHDl1U?^%n07! zGmZdEboVag)wk9h9><`Fb^reU;K2jy&ztxi`+!D_1&w&&XjQo_9RH$RCK)q%QR>sS zi|mLSMYfZpx5E%7`A(Tg@OI&dg07D9-dZRebiKeOI`-!)ZbVNWq$~d1sqN&1<};E_ z=T773f3~b*{O?kyMva*`2*Yc1ZAoIHDh>6$c$u11aTOG-PCQq?ewLFL+O^r7TpB9{2-)hoT= zKy&?mB#c!SQvgjS#ia&IBCYKMp5~d5kZlW zN+i11F1%kx_+3nDhDLYS_@YC7bB47*1G~)1t@7YZGN0f8r z_z#(WNq7A)?v^U>Zdw8au@%THBH~%A=VeDRz@F?>30;Xf_5!gTK*AH^`W93UU;X%e zJ~SwSXeccBlM~3GQ!GxOm|z#H%2TRIcujz}m$+q+9%=c5GG1pm(&?3Z!?#pqBV$^% ztW906OHYFj)rUThs~ATsi^D<(vVK-O!abLnzq{Bd`(pPg73Jom4nO!haVyT;C7%rZ z;JADgsRWW|j^{GG00F$yY8%;CGqgBPb9au{Hm{H#e1R*F11cr3LoU}lutJGfxg{N-WxRc^%|n52oLwZQG}$;b#if;+Kxt=%vtBqD|kmgG9P#M1lZ*=E0;~p$d;cZ zC=~<>E{~weZ2oJ!3z=gx6^=t;8_CMEwc=K;Dk3?TIZ^$O{5h!8*--X<5=+565CEJj z{NCES<#%q~lDR1*ge2GQdV?IwbeJO-$Tk&(u?6_de*9Q8W@O%;DE=JDk#GkfD(k@m zzn^p1sRzIuDHF%7^%jGJ4c`O_yK437(?JTQt&elxns^S7+4{y=m}OPaTx<8wJ8CNo zV!ifu)h0DJ7SPQrHzfl5GaX%qQ$9-JzR=QtY6w(PUKFM5T^vk6mPa&qa2(UIcOo6O zSAa6=zhTT{VvOoBL;}C*yZr7}7bKCNWyB017HQIXf-fxoz^67nQB=jz&bZuZy%Weo*@Ln~%=iH@y~JIXUj2lUan@>5=r=I4+!> zw>)oPn$t!?E)o1r%z01cOGqqMcP*Ic%N9vY;TCm(!*T1r`vdU)5@!biASnqPpPjk$ z=@J6=B{HiA;BW8|rGMW{KG=ojnvWf^w&GcLN&ZUlyPWmqa%HW<=vBrbOk>v?B6wrM z4@Yr`S3m0wq-XRgx|e;8TH?lS-O`FetKY25a}mJz(+R>a_>i@4W;CmwQGyZm^4U$T z*^0X2zE{pNMG0;1QM4>>)RHZ{XdD_BrQjv;00CMVhs`GM8p#bi$o8fb0{oKip3l(r zaDdXbefo@JMB|ftr%w>Z><2PZMHjnH+a&SSJ8@Q6C}Hr3-YzjDGNmJ;|NZa9`3$r~ zrs`^2_pKl!_rVA26$9U<30gT>Nvb!x{TBBqw`mBPPOE3hdhAW)e(hK7u^|EVQJnX14tMi*_3$Vh>@rDE$|h?=`h*pcZLLYP?)!Ie>++ZtM$ zdUJ(DE_;6ZT+(EHfwHJLQE z$RMvjWCCi+!q?^AFC3L``{AgMG3#DNum5vK%C3M(;=l#(k)a6yLXN}qFWSOPoAQ1w zObIs}i2uB}2fOHAyU=S%Y1T0fnaa&J!AmEu6@gW}!oIeypVg{0f;JbyLD_e*XXD<# z8nn7uUfI%%=g%LYDtHRXDMbYBQRW!V&XwaWInzCA952t+811|6pY4k&FdJT$e}FI? zVBK;3o3xU3fH=y8^AB0C`il=n`~0Tl^%~v-ES*4V%398GBv9|DV{hn)7S88u0s~Jx z#nhC-Vx?FH9CkUL6LMa6=`&JV3~6w|djKGrbeVXQE}(G&qt9QCwXa$Fb zkm>OV;Ruk3eEeF`!Ggu_{&xN-i@ajo*u}Qr4jjZ0ILgujz-6{=G;nq%al>Pt8Fi)+ z+r3?g9UCYe#x!~a+@EmmnpN^U{J!o@nRs5|E{w<});>V$ZoB;b$@22j*^k~cddQj9 z5$oq)e({Ch(_`Z)OfOxw>{?(Z6XYNLT}o~%D&vwp<-CVkV3`p!efI1q#MVMf=C^6x zI!rWnM7O#f8yV#GdP8OW(v3OZHxZElG_luvxlJ?&jV_4@Xi$b1?r{3#Aj&> z0)iN`y|7A^DmVDfNxhV1CG$dVnRTsOzO?l1TN!oUgqbIZC+s}b@?W%2WyCb@{`!$g z1CZitnY_$u&49OMU2eaby2@13eW0VV25i=NA*D7HpgU}l)JMaJ#& z%!|7gKE3tw?YI6i8-8x!xVeAz#pSG6bz9W&8W)t(5%-uS=z2kppmza4sJ_0XsQD%T zP)!#d6LSqT*T24{dsdVZSDprfxOOgNyR;7OexWc*`tultF0s{pa?6{Z3@+OKQ@!KI zkINvmE_G_x-t6x`{8VZNG~emXGEXmbq!4=Cd;oJK)%O!Qw<*oo0c_^HWb7moZjs5S zX2E=*L-RgwOiTjb)3cuC$&)8Nsp~0u1(}TN_VMv)Jc&75gw3OaX7dzEviuzd6*RNp znt+ztSS(G5Y-<9AH10NpR5#d-{#|!7t9rbQyCeP@^1qRcCNDoTE^cnK`t|n`pnIJ3 zpf;&52&m8eo4&_pYzQL=R9N+ag^S5m^y^4k4l?u4EcqScuVOO0^Z~Jdv}*8of+F)} zC%BKapy$t?b-<+%IJCd#L+Tk292_k72^{U>a;J{j#B%+h1*%GOTf`v9_jcTYK6_WnJ7ZhCm}q39yFZs)VRpNB}D&5LT-cIboM zIfF#+dF`XahYt_Ta(4!Q9T0KbCdap*Jm1K0lT2ot5x%XkiQ2 zufO@G%=6E*E5OCX({3Qx`cP)hNFkBX_6El>T*l6nemifvR;zuCQk_gPlR7d$h6M%o zj6f#mHM+=j3N#=6SFGq4ZQ~N9jJtuWUV8-9UeuUpIIkr99NqOf5soO+c3Do>;h}SK zxb^EWXW>-G#~ypb($-B;ZsoS7W)8?K16qkX>Njn=<7vr?e;Vi2_X)+=_vh;vs&^@? zpdb>nl!x~_GVK|k;J+@yE#P*(dk6ND5}xbfQi)AzqR-&v1$8M&RmwxiQ0?_1VM|mn z4Nps;*4?HuqfyuNJubfFF@4Z7Xk9A}8wa;oYFoFKg0sG$VGZ|@@Lv}HTd3wFYyQ#a zoZ--$S#%hnzx^f=CD`vkzTN^t(}-a*6qJ8 zHPB|#4coj#!_F+r&1q3XsZm4Z=!xDBWfB?ge${JK^v~-^ab*A#J-?f^u_sR^Aa(qt zsP4Iuz5iLo~i5MNzpPCjm_F?2-$^V^Jzh}?W z`)&L7X0N6(r0!bKqpYk=hcQwrdd?uy8D8fH-D5(u)#u+2#hh>{D3g+PBgyZZ@7BW%e%Mx)@U z_YIr#;OWyt{3slTNc52-y7|e-p=|rP-QO#wcW7{_-_qZ2`n+gP5+Q`b=2N3so<4sJ zbPPfh8*4~RP)lLRC*j}VVLM-LAP(t0vRyyUxd>sMr9^saS?O`K#oOz~oSd^;DgpvN zo$hli_c5&vo6nYoI9CfTa{bP%a*f=lGpLg}J|jW0P_SpZv{_86hm9A(?>I&Q*jV@h zHVotVNF_iSq(_OkE2+|bO;C;V=g&U|{4yVZ?u?;4sY(5VsP+KRpHRQFIyRmjcMH(3 z)j=stXG56X>omvi2`1AFWMy8i`|I8Fd-m)}HZhcqcU|U4C>6_-w^SqY4fXt|!_VfDHe8Q%>q#IhI$HppPEtmnO4A=(eIGQT9XX4E z0XfiZEiKb}bazkqB3ro>@cdv-wMuw5I`;8nPa4ez=|i#R({N>#tWvm72r7i3_79@( zz-bG>Xj8~;8Z)VPx+nn?tmu$rNGHW`TKa|Rq33wF)XB1bO|@rc|s%vX=O^=X&5{V>!DFc{_1i3#0eMnNbtZd z66-Y`k3RU5V--su^lgPW_`q%sV>)hhGh236D>ITD%XYFV0{jVQhgF$Qi16grk&w{R z$xWGybz6bE%|=PP8-m8LRFJ!zo1YIgE7dom(#<}YnzYSo|J5c8=9uNx#CYU1p&WV( z>b{gx^r77H6N>v?ZEh4kW24-p{4gKo`toeG$Q^Gb!9MHW=n=MYpdNabHG>><|E7j1 zj(>7-f?i?u_VGkR{? ztMv-=LIk|bD|k8K+kt7nu3UTf?2`0FK};<6EV8Y{ZXekA0XOMBI7-X9;2eU;ck9+o z5NsfoAP6t>lkiFW@3Rjb*C5<-08UpH|54^g+=DU1Q!9Ep8-uD(-l#EDu3pn00Pn z<*n`L()I(>;K_KLalqNs4Jz{asFbpylAmqoah;lrgg)REPEfAlLb_UT^;A-4t%cE) z&=eXLuhh%X*ZpSnnw(TmSeSZYOWD}kZvQer_At&RJjZNVZLo&4kZiC^6*bxYjyB!ZR_;^Le>HQjTEHTu@Gu-ycjNIomcj z5GIr~nw)+PV^4;WgnKPsyjY;s3yUyBC5*AU=4K@@&H3}Aj>TyOObb@ZW4qjEjw16R z{0f>t-XlhDyjQb)`SR#v$xT{}cPv?~OQW4^|8KwjW}TY}trA|`a`VxmZBA>j(lO(# zl+-+S60szvq&TRP1ktl{U$lXJG>y1}jIB?vQTD&Qp1f|q>GU`Mn<^~q7iZjc;#A}E zXzVDldN;(+UspnPU(3XhgX2h@ zw_qE&Nkyj@jFtv_?TuJC76DXLx-_SZ0DK?tq@k0z#92blSfb&rb9tIxV$!JPySYsSuHvzw)AxbE>@ii-X^I8 zL2%PYAe!Mo^!@+*(TP~R34`t;cdQz~Nl+@C* zeoS~OSf7xGu3fwKf0*D-4~aQwrXV|$2^Jm~eP%GmL>O8~mHZsYfeC_gGVw-# zMDVE|H;TOhYXpa)(~J+J~6UD(5d>dRn*yv*PRg;P2lf= zEO-Q3j5y#Y@fc$x+Cj7>mQ2GMtJM*q#$fq74H$5-Ua6FS<0{Hehn7B1zH@4R_}rP` zj>P|)Un_eXV4$hL(75k!FxQdlji4*jF!33of9b-Zl+f*lAeqi^7tM6=a01Ab)f`Qb zS5BYx&wnZZnRG5FC`cwzw`G=~K9t>h0Dks`A`qJ}VwVerTj0{2rE^j$>Vyh1##(x0 z-#)howUI6vW?owVlkfg4^~Ai2BG8r}R`-rxEhydgO6QTW6RQ5z^TdsZn1r!4@Uv~B z;n!h|4laP{>wElw>ItLP%2mzxO%S%=_>2 zzVn%}oaZ^u^ZVVu`@Zh$x^A`H1N*uytj(yqRHk=~dknh?(7Bu4oll3)mc+<3q(Y`4 z8XFJO^rRa^sc|#k=jS0qIzjnwa-%|KzOge#XwIHHxA3F#I@>FkOxBrr(B##&4J4^< z@TukazOi$_&YZK->Ls}s00Zd6B%?Q<3pqHI*KL2!10I^lY}i8bH)JS8{#iJLOMwxG=io#3q|$5MN|oY5>3*AhZffL8l!5D!47m(d(3bq> ztYc}WB9xsRTxztu!K|gkm1j9Q;nL|9t0T|y`2Mqa7q?05>_ZxS^_6gW7(;t;#>}Q_ z+S?--Vt;i*+f{;(=7-RK6gmO1K?7X2W$F&%dsAga2N`J(SY_h0;O$d2U_`g{)9`c? zz_mDCP3!(_D3e*l))eCQu>F^sXxctt0Tf(^E^+0ng`ACm(*b8RW`|b>Ss1$<1128@LZC&?*wzzFFv+X9=E?? zG?^&<@DVcVkPKiq=~u$ve-8w_w`cV}%~SGR&vv`JZ^H5YLjQzu>q;Ff!GIi}b}u6O**zMUX}(Yr=R^<2PS<@Zdq8SD>$) z2^+G5K|90xIPM?Qk>W_88Qw@xH>Xpf0ZPC(=?@-6;W)6LXF%hi%fyKf^oz!AoH)GH zXY2O=ZDK!irTe^(Qy5DnK7e5=P-JGfaQaBvO8vemAfS~RxZ*YoJPKgQGNs~q6;(*P zt6zIrUX1hl+H$b~E`*xK3Rq;1J>S$^GP&QyLjT)&juR?(zm-|QJEfJV)BZ6^sN1|n z3m|4=A?w2=+29q|^(?VTNT?`wXU_lWKOuvEH3@vDm&<#I2&l2-=jxn2;e$w;ny>of z5BN*tWt<-~f;_^2D-t<^G{>gK*QPI{X6^U+@m{fNk{m(82Vd zpWy#EmLX{z|}1%y4tj_p^k)3?UQOiWCkdRerS^RIWAMa<{Am6^8R#od%@ubyN@^w&GNA8zx6 zGhUJXKRs_-zlq7ugh|kkvM@I6KByN$cZVVIi&y)~MbO|iH>*7|$ZtII9%N{L-ProK z-+uGrpU(%m3eDU8Myymo7bHlt@pVTHT(Yu&#dX zx_?F0Dd|xqpHm&QqfcJ^A3iv+`9UnYN3xV_r_@{_Pv>d>^Pfg!mDl~x%Yf5m@J||f<|3AnmNMXT#JhKj*Vd8k zp>60ZQNfx0@J?8aSMgFC%Z| zOe24oR_O$z@|$)LDuJBBOxHRP64G3%`c4y191vQ!SMboQNl6UpnraNyhBK!z<1=c% zn!eKHLb7!HIrYpe^o-|=w7&n>_3FjurEM*0Tg}R`8NKVLT0-rn{4=wtl$rRg0#yB- zva6Nw)Fm*?YeCM)J@)d(FSLa_cFJEW^C@4cLic?v^H!!F9!IiWw#|2Y zBK84qJJ;1Mjp~0|m2aU?WOK+2*CyK^&$ABcvkNLp$IhKckZRmNI?JCRKT~JgvczCj z{4UO<0-ThXMV$gGpE+B<@SC9la_aGfJF2BvWxseG?Y@1=4drB~>PuDrONkK4^k?)K z#-W%3hfr+dBqTQVr(dk%0haZ<)l?Y_ zA;a)$IRFEZlb@Gv#9Zgu%x@~iG`8OeAV?l!(=BKOVcKraoc#pub-4H#d?!OQaju;_ zbJi>Y&N^6!*jia#h<5++?-1uIBwO_;W~Pga%oSgX`_DRS?GCGY5UWEdG@s*5jP(}L zl~8Je$&c+8hATLrQPL=3R6tqzR<+d{-Gi(S&KV)B_dINGHyQ~_S$MW+JutC}JequW zN9hPVp%3=MO0za&9oCvq6ZEV2T%-w;pbxsXIVPs?rF*ZxLn5l%+m8Z;KR`_$O~Km! zXvxWznF^c+nOzDs;UFnOW2gUFqy`=$QwGe~n4uWL24i@iewUi6#xrT9X=l48x|_KX zHBa^p$UfQcZKr=<%z1i<0;*AXIsh-UQ=dM4T1KnM&~M`B#p+}I(%&p>KnfpWpH=#Q zy>;UAN%T9b?~v!FDgdVD+pT58*6P@{?I1W+quIwsu0*Oca4!mH+pMhCZQ1_W6LOD~ z&0^62cTgR7PJM<9_(;0D0B8E*AuK<(W6E&UlL#WsDT=Id5P8PhE6UB}{~^tE8|x z2!37F;%qOVTQh8^jlZP9l7fV*lbX)RU5WvE`HTPOP7XbIxgVAQJN12BMQPlST$R$1 zL3yBZoXu+tzWc6?T3CrfyQ*Wj^buuw}L8+Os=QyF@J~_g|8S$|ITPM@KdDbcF`?qI~@G?bE6bL6!e# zNJlLB;Tmaw{qE+sEsfnHq&z8`(?37({{S%hOd%%XBCh@M+Z|3TS&|XpF=I7b`f6h0 zkCj%qXM0ck=Sxt;VVdS&ufjZ7O^tFL<1H)SB|JFYK=;HMcPL#`fIBQ7?5v)RpOoY$M%cV1LN&E(;aE+h(`!0Z{teu z`tw_QO#Q7K3z*>=C^5qLCqDVe9(}u-=L58rsn^8Fm4jZIZaW@c(Z+&m@uz{Exw*N3 z{PEegU~|V7CQww~Ol5qLb2lZzrlmG{nPIY3Fa)7$#LOi`q*b!AIr(ImB?;zmnS?&a zv%Y8PvSrG$k1MLQj0Qhau^pr>FG5j)qjbiAuNy-u1R;>|nVGp#1<8hjBGQvg+B5ci z*@ual^r2tjttR*NZ#-^tBU#}V@ zC^hBJIvsHB59x3ZarP>Sl;$1mLo3PUbOxV!@rXhv01I@wATF2ssih_BLrmXrXA1)9a9Cjh)>Q_cue?R@X*4Bv0)>7~Z;WCUyLRi9sm5yENj zyXe5k_8S38qbP)%=@ztMqynzW)i%q#(tv<(v`G}eRjhD#R zWIM`f{T0oAHRJ7ibGF-0S1( zse2aq)$HZ_dC57t{jUZL{aN|>`S!&RM-zVQAE=B>x<^}gm4MCAO+Hb!5L<7@IXNRl z`K|o&B9-x-Zj;W0ffRp!c7|d`TXfJz;)chnW-5RF8xn11%m~w|z`K*XnM5`>e1_Eq zl#CF7o3;_j9n}ieQvUtpQ4g0qsXrF&U-hA`y^BW`qPL((3}&G87(g1<^?tauucm&< z8vqjANFTGNo}S1)(Xj0oJ{5)=0BIap!b{_6Y#kZC?MzH^UJ6y{rY&2HV>1sz+Q})( zGEiA=OIO?-2$3_6rl;&0REN_2*7LBeAFKCRuJ_}=Ivksk_?`NW;5K;Ar9EmB`UuB3 zN5OlT6PuCCok^Z1opF7g0isIfISBM?Gng_--gMnUyD(@OHd=%VhaE3-pFDYD`1%p| zXQ%8P-jR+Z4QJX8pW?Hs$c5mM!})+pFuUX7;t2t)sG%k=Jj z`%Wawrbdq2n&_H*{#B^4lHnUd_rgI2IKV9uXeibX?8G*WAqjJx5HIc8cVZ z)QWj0r?w-^ZBBAYh9oxI-AH>emETW$h#jw<5Q3N-pssya`^%PpCApZscnGW_CRD7a z_@Hk(=@}@kki+bah!7#C*b+!G@AcgjV?|L!s$;fo6s+wuZ~&vcliB($m%aXV{`~p$ zR@!ka{yw5Im^JRmR;04OB`gj#GJ9K8>(cf3om`rqOWRp@fSZnwihrR@m(1q*^XE0= zdaFW}Tlc0BqPp$rto@Vx!z%mH{L?*=3P}4;62#Ka7n`D!5U(t^pHLa5qm$4uF8y8d#f!OKxZZ5QlOW&?+vltI0R!rq;gLS zma9~QgD!N|Gitka>sG0wx;dW@$@TFe$tA$Z(Q(~n!H&WP1eo@(gX(nAGit=}ft$tA zP#opR1Q)DL!szj5(7-Nw4HXkANvKk`z)+jFZgthUOgC=dwd*M0d3%ae_!HrlnXzQ< zqRf*%QbcJ@6-wVdRiZ-eEo=XgCh>ezM|&eRU|g`J_mRH`%dgvdidF@MK>Eu?%h8ws zmLw%3t1as_RsZ4_G$AZ)ifgG1Y=W}70AJEcyLRmw6`>lx$PKzVS^?d81;@RMTtx7S zQbWje_0webC!aNT!i4+3WJPgs$|5G}D-U45Szi6(Xyq*Ce{WFb=O&E{S~^Fm6=L2noX4`93-sj+pKpjG!krENa*UX}hHp}t&i%<`q zA%Hufo>q0ikd|)sA%5`)7|WgtU%pnYT4mWgSUp5zF-e)bzcu?EgN0`wI&rSeweJ>qA`@2OCk7< zN5>;FIx-=eDp+3N%Ti2JCh(Rj;tK>UrW{m+RQupO&!ghUU#qhKRj5^;?bhVN!HODL)Zu;yvoacKQomB7BeMoO6i@j(R84<9}ha2u>-hL^*J>BlF~`!lAdrcEd; zR~D9ug}foZRTiF){9Y`I{Z9GrJMGMt=TG+2-&+dIIrC^bXZslXla{B8!7}B*#E3Ba zjGKSI4_lyMM$u7vns`p7GW7q?4NsXlW5#SajYv&S9xQSAgRhmZl;`&GM9WqOw|!aBm9i={$mY^P?y z-9I$-cjSBZyLGmJv9OPfve{%1nbXoE#ChpWwl*|)=fIwoKT4c2z)y)ICd+dYDS$j; zwiU0xrczQUlQ?X(IyRvEXreVqUc>KD)ny_)y5%qX|UbdOzwGyzNM*ep_~tCP<-1jyH?N@om$l`w^Ba z14Vm55bp_rf-~=Va4(i@(|${jQQ~wtn(l7fQP#`u7gW%g9B?G2QN(ZBxN$*dQ*;y3 zrkry;pG?S+Ot$V>Q%(}T#7ThxgwTe$O_$fTsl29C!w)wDH*DAi5(u^gz@vE{`=Q>ccT0iM?$Xeif~|V!lIkvk3SQ_$mLj>dJ#m1}Kee&teX2j{o47*6lKfws zU@B7)Lh~LDlmwLPnB4$%_Oytw#U5qsqD3bZe2(5jQ*vJL3xFN+=Iaaf_L1Zrq%@mN z6NQU(PH>VNUvZCqlJphX40QMks0!E&O`CR#O6{~QDO{Ys^_T0Zmb9s^@2xmDgs7{w zquB;S1H8GFT28e=4f_4ly-1t5DfBu#U>hwZKX0c7ceJJc5Oy`)kxnKib^yE^(%sEY z-B~hz{CI)PjPh`;z61I(FMi>~6Yhe^&p7!Y##|v)FF@myiV`xjRBrfnm!bZKsku9q zgL1yn9|Oic?Xiwp5#p}9**2JB`z(}&`HgMss;qWhtX}FOtem_gyOvrD83CSDX3+AH z&7mz4Pn9L-U?WR$D6}sCTihF@Vby8;kr_1{nij&F9!*=YomxVc!$GH#hysqx{a~JA z$mAF|^DG@f2@*XOuq)2CXKto)%TGkRzbJMRWEh#v;PGz59m1~E%0zH!65)iTZ(NJ) z@n1O2J>%49k_U}ZKQMgJato}l`?;_HJBJ~VL3)3la;xQ!+meuzFKpoYWrGG%unL~6 zD7$Kp7&U4v3$~qF8n8%j=jJb@m==#NR%9o(NpT6=MKTg%KZI%L^F1XOGhRa7b5|-f zRx$uh0U!_y8Cx$wr+G+1E8|?Nd&+<`&rQ0y1>>@8VQuBC4)8*l|Q}_`a(~exn zD34bn>uSeBd9B8cAvTT1dMl{ zN^Fca1qbh^wImM{zBrRX7C#3(p0D(2PPw(I(kYu-K^id`gPKq$A8dBjmYFAn*v?4S zmfy(p$|t|EXD{PpSzvhI6X>ZpI4mq|n(3z2ZQI5#cYmP92s;2{JrpWzzoOI)qBhcTfpT_B~SnH}sdtPDVi`Z%z_GnhEa$NB@s83=4hj7in z>E2$&eV=TvK9cBsPC|ff-OtB{g!%<7JG5(;nc{Zc<*1-h!fwG$fFt7;&l)?$s}mna z87D;T(swK*_2~%2(Dy#62uAeU#sd(NqFFnuxakYj@ZY432s$6f5! zbIZy*uV|;R-h|k6_3D`?1MAAAf#T@6gX?V=y5jORQk=zSBpLNPTTCxsG^O&L0O~ex z-@ZJvDeI48W+dyCgxY~^T%Yi8VZoMlRedbKi>)jzGx|00r4Lv;O?Aqi1t_dOl2oJ9 zt4tL!8kgD$dgE-*$^_LfN>u%L6llU2y0drh-pmh^nMD!AXemSW9r(PQzZ*NO8=^as zY%PfqTzLf48EK6GnfIcoNq_mlt+q+V!n8@^z5t0ZhEC&b$~>yJsWNW1mLI!Hi}C?s zvsgqNtdR^UkI(}Gcd8+gOQh&__V&G|Tu{+iQEMb;0@M&o9xj)5_OGfA!C{D}?bVbX zP?Kiucp*CZyhRP_KS{`f%|=2$W6*e^f*-26%PbnuDeMRUHprw*>xM+;T2RFXI5b_hN+* z+|NgJF3U1KS7O*e%V1-6WrxhF^_$XGwba9h5AW2|=le#E70rtj>y;gyu-WbwRIt8S z_N%9xZ`pfCUXITkkL@%!4)9{#rtmyYHLYf5JQ}cj_wE^H6(6Q2 zv$2~v>9rd0xqnDDhqCxe$f8@RJ?XgMih+b!#fxA+Ld*bK2VGnEzz2EZHL|XAj`MWz zNa*WWGGigLl3+za6o8d-o`)=D+2o?EK2i?))zFjlR_qQb)X=Zma`x6gKux3kbM3i& zXZbPi^4ssfGtp;m9@8h!T@*83u;nOEx3U5qH_n213 zQ2*ou<4q;)fyO*scVNzSYso_wGaa^;{2H({x&Ov1$7^*Su+n5xr?L6#gXPM@TwPs1 zFF4pGWj3EYV@Iz6)E-N#-ZQdyY-_?UP-x0R!GnW@2})=ZY^ zFG{id(4Uxnt0YW2oG`=0l>u9o#1+aR@u7M31omvk?a6TsrYt(`B&X1;*)bO%(%cTkF=_4?SLc62@|9(kV-WSJJOyVFso0Ihb%Dq5wDEGV4qID9FSI+nE zTeL{BqD`~9@xReerCRGidskMh9ALn2;+$Q@WO=mVpyAbkO`F2NOP(c8CfT;-X%+5E zn*H^v_~mDpC~u~99e?C!QFoGjr;^QWd#F2YFGSNq7rSXrXn4Z!=@VplP7+WLSrjRu z&^I7>Xdhfa%m2V#uUitN#}X*lRV{f-2k@quhsV+I-?V9y*HE9bV!u_pJ*fGD6_lCM zxI%g`9&|xuy#T%`*jmt0q{b_I)?c@wJgf=n=RZEKCmLC{1^kSv^mjjpO(g9Pcu%#Q zNH5+tvso^6hr9pzB79hnD`vp#{xu1 z%p|Ivi4p%QAw{z>Ddo!F6<*aQtB?Gpmo1zD3OX;Qt3(n!5!Yq+(i3Ss5~E#YoFZjP zC4j{6ra8!i<@7SJx6IZp?8^8nw`$N-D+7jYgcD5e`VT@7FcKq8*7mt4CoW#V`V*?d z9Skn^pkOOt8!dD$UEW93OTrvl6+bVtK-b3jJb*?VeJoMO!KD%bA1qzL@u3G6pCO&( z6^)`EO;Z3ij?KK*h^OsTR0wgzc3Cx+U;X^7cHL(86~CcieFxzE(DT%Vq>sxl{BG2a zBcBNWnC>O(OG>HL{{D_E!;Cb`H=cz1Po6eG-Fd>w@C=umUc8YPPnjCrJC}^#)5pMUG)t< zT2G}tbU@j=jnow!;I|2i@?*7Oqeg?b*lV{N_!mW?W)ZsvY6gQwjlyZe)T*u2F3+1> zodH&+uSL-hX0sSsWlW<9Ui2!%JLxz*#o4S2A?=HB4@JGME6;amwm~ozgMG;nTe_a$ z+}El6ZQLM*(#sM94i5JvnN5$TA3zHXY{l*olYZ(Rl~Q-tSJfv5j?jZheyMRJ%1RB? zujJ`Qw1+y>hKm@R5FUx6q`q&0+z<|gqE)4fYa?4uDz}u#w)CsjXFRiiXpk@>*VaLI8$e9+s^yhcTG1)WO^FQk(}Uh5}o-#f<^MW z{=>&(|0ne^F_ZsMCiV!0p=~h#zqj7GP6NX+6tOYhZHKbpn64zkLe&6e7e=VG|HW~bT&RF_lkR+XzqoGS>Pr5fKrcjg9yDMRDJLpEqpV7R5&8-bDlcYU+%Nu0}>bLu%guo=Wh=r zP-32h7vJMAiphZg<`v0+tI{R}vp%)zYVgSR^e8zc8f(X9W}F8+6!@3QQtMI1Ect@F zTSm55U?LCGJymcQ`V??3DR@>_CpZu~rjoP?(Vo`(!uK)LiMe}4Cl7hk5qP_Pm&(@D z>i$qb#VZgR(pY+(#JNq}tNcpurQ6mh$hZCPV>cD_?5R$vkaC@QL1q%mDRvuqecl7hUnoZOeIQ!-57!vN;zfD zp52<{@_Sv~GPWJZAc)yKdVFY71mED@Hlb~QX)=p06;hQT(=^Ti3Ws#q_2(1suT{1D za|uZa9saXljE9L7Nf8%6@&<7UOLOAsoZ~Im?<5H!3;H$^us!S#M=WXw3t9evc{q2BA1{w6?y{@*s^c}cD^~n6lkt4zr>gsgLR{Cq;ZKc7qn2FgDDD=k){@3f!cPqZVxjvbab_ZXu=FIVO8ArTIP< zewALm8W)ZVvqr2Z9xf{ywY~rH`^(S6>5zf}HJZni7Io|G$W>Ifr`}kSbb9~G0eC%& z>R5t>T0n>1i)^qEkIN%$5=wYq>ZAo5}sM< zSsEjduRaF;(%>=+VfEMY0$MfDy#Q_gQNEU5DGBJpPw!lX{D77$yME8gOZq+dj^tj7 z5xDXjjo1fbcE|ww+F4=sK_xj{_6yIIi1um$h{9b?mkhm!+2%Y?tK0v`(#!buE0^ld zP-qX+ZAp^b8VRdVq4B4q`N-u~H|3R6UV=I%meshyd+OY=qng4ZM=wNWn*iTyHuHqV zQGo)fyIN~y_2Fcka4u@6?FyB}zI&bQv{MbMcbZ=u#<3HN->?StyVUR-u!zy+Y)~M( z;iSNdj!8v_hyZ2_0a7S+bSTtzxY!({^?;wty+;(KS0&Tjk--of0*~FJcT{q?ax$m-E5P zD(U*FCC7k>gi}juywOOQjKKVIzQCB!hv<~Co)_*4+M*>oWst24q;Y1z0sqG<B9!{4xm42i+qycuT?TGA8 zg^k!%yCkjNBdNBdE@~UtNYjq&ydU#7faCH_+)IRe5ru~_k5FDDWNbxM;x2Cz z+{`Eul^z00@r2Vqc=6?9UgD0wWH)`(Yj}M~)u?f+(NhkOqkYv*{X_T3DJ-t~-D;Nm zpX=*?Gw*uEfbtvde{1u_U(2Jq4(>8IV$6%u`}aG09b3JqNAj-wC37?m9j8Z~&<`Im z`}p2oCr*TqdZ}z`^IPbmu|NN5bRxmmSNUlE^Q-ikckb>GJQ^Uu%Htz|3$dxxeZ?r4n|RQEKT z%1s0^nlN4?gcD_6qYghU8cpO#S%5#jVVx%r?`rWeKFDum=II4PMQOu#8jET~Y%SNX zf3dt?E2R>IJPy20<83$-a@VD9sJ!@b!4z6nd|YJ|35Iwdt23yjF7sgIdXyr5m1Z$( zWR!+dKmE49^Q`37@WG>Z_U7|rc>axx#Pg*LJ3|*|hP&Ftd)g`4n zYN2~ZR_8@Tj6_VVOwb&Pq(oFo3%jWN21A^#j2-2K)?{Hj8H5Ibk{O7GxLY(uE_Q433f_da?B`_ zWCFOa@wO>7RYD(=!ioY^)bP@Fg^)9@MZG3+ojZeMl~iQu4~=SAdFxibkC~AHz9gWK zdjh}fY-gt-DT-dNOB<*BrF?$UGUp8p`Ad8UKch&JS4B(c&jTe~3J}-T`j6|jyo5Y? zJbpgS_2YXSsGe*_eN#X?^A^EJie{lVaDyi;zSG`-)xnRn5w&jXc0i_HPZPd1hhxT= zivUOT%KJ(0&S2z*Dqb9|H+QtZoIh^Zilxfessw%By=qu+_H1p3L-%!_&ZTpEn~q~o z#79D+;WGu90l%5CP?s9`4V{CfcFr5X$Q$x~>NjZ6Kr%8-7`j*#(+Q7%9@c=54rr23 z3<)r*6W>EWYEb3VA_Gyt3eps`Q_|EVdOZkX)7n3=)&=L_h0(8>QEi7@AuQ(m3vaZs=jc5f*Hpg z4pDI-Qr;1$Be`gf=#-5ikMc|%2_?tqy`iX_uthh_x>fEnbQG)Ct=nt_4Ny}nMCIot zb7NRL8@(0}%%#WN#rls@*+JTcs4|;beZdQZir&7xBit8ik38w&kfc9*@uG`*xLQ$5 zuLf{t9{_`JaZ^A(yISjx;Mbm5ZcQ<4(8S~)c7xwp52%AVZ>R@mP5mxx2rKa38os1e zcFvT>E2l23N=`}jElI%dP?hb56F82vZN!3d8|BkRQH;YKD1bwEt;|n*e~PMsF=ak( zJhD0jb_NzPw|IM?r^@ORJOnDgc1>39V)eZ7dTIF^mrM9sUgwJY`sHp_St)(PpDBIb zD;33xoKn6Yv+D?_5i5k85<-no)$%;<=^jWxkd%$uzzAretdmDMl2j*B+kSxQOv7tp z$d}*A%IPOTv2O43HUGKNCd?5T&$6HQB+WuRv&{w^pjEs;sHl(s&|M_GA!98KEAniV zBRv?l5qHY5yMFvFAndNzdVk56=&$_}KWtN~=mjjuG^`|r0VEi&XlljhwQ?jts|QeM zW*GLKlae5B;`8J3`l}0L+8+8^?sAg$F1yc)(w+Zu2ce$Az)G#WZNmgCjf`aG!&1nx zU91_Z(cqQyI|ccLQg7oUL++(j>^{;GgURb1h%5j5^L8T`kJ=931sTGyCK-V9cb0$f z6_UE6J-&J+vEqnsAzTWt>~!!62`7m?JmIcU)3u-H_frn}v?dIR`1p`7Q~}PLx=p%o zsjUC&ufzZQAzbXR zrpvy{+X1-&A=KR980^iNa8-?+KxVygz3b`h%%MXgEBTxY6RZXL`?~!Xb!+GJU7;sC`r?{VO)BrPRUJA2k{&r+ zVW+pnP=SO^7*dp{B#nE8DsZHl z6dXG|Dt?YxZID_`qjf-;5 zC3OGK^Rc3dkP^FiuKEszt{@9*8rHDN*0Su$@qh?K7a~nM`qx`>?d|MDNF=&}x9{H3 z(^%7Gq#(ueW0ft)^uB$HG27EveC}Ng8Cf8<^^L-LFIK*PSMTM3L1@LM8rG>*Ydjff zD>dKX4sH+O#L5=^mW~nL9+Mf8ieOqEcCCs*enVya8@v$hVFnd5^Q?C=Sx?;!L_rx| zK|LI}s8D$r&H9+A4Ra-`ZB^0G>T5CRfr)9P7QGc<-lf0Dt5EYgvzMjr=CIu>7>eNb zAh8G5SLMrAyEcM4(qfRgU$E$;v=HY=NI+40Y8ZbZBL|UMMnY6|7%C%ktor}=tMb}= zLX}qA!Lfk3fcGmH zO65)Y86~FY>mEB0W`M|ywB$p0KduU#??zRlo;lY37CSYLtUM5BvC5>4CH*e+fL$`n zrP_j3AgLj_RTL?TNGCAZnQwkNTJm`|oH!hTyh^$v9Wow3gPxPIeI%GMgR3>T|JB>K zM`KVf^q=6~%G`sji~gQ>ad#<@{t2s=ELZSK5joN9N|_Fc>B5ut?b^x26&X1Kjj9cm z5KB-XMnrzTkLLaQxs`>*1WM>MREuLz&z~t2O9my&yxarpS2mxIi6=O?cA6A-0|hcT zuXLG{lM*5%h_4J!sr*h6xIBp|2As@u)V4VgG?QQ4=3(lV1A0jsv% zV33LowBx!;=o`bCT~7}8-CD?~Q!$OTSo&@sy=;M806nCVk%kykb`57c{Uz}rLHWz7DuQ;I zS*Dw`AovnF!!94dsPtNfzQu78-+Vl5k3!ZHe_vtoLWeS2&QT@?tV)DQGmOwk>U$V{T_9PvY-=!p$F_ z-z2ZnQ>RbofuyGa+{@hM*vq4a55L1=k#hpFuUTe)_JR1^A_WOYbG8q=!$s>_;CUs8 zvGAoyoA11|XEB`naNkWNM43-&mZZ{!>q%=}K+V*s`i^O2;P1qu9NZ(QtCOC^a0?5I zq%CkkZ?OUo(DHxwqPJ!}D=`6@lQfXwPXrfnZ5Ds33?2m=?3DQ<3;Z}~qSsBJqN}`A zQ0cUw!%lh=S_~a3GMT&hg9K)WgkcStk8VQbubASe2N_S&>!u9Hj*saap}4pR>rt3} zmF28eH{%J%_R)ZrlT(ahThwXtkv;|`(Cd+<2Hht+ao!IvT1fH16YQGlCR_rsh;Tjs ziXdcIT3XtlnI7_s@DCO+PqUId=vGy*sC;RQ$4wbiBS+ z>Q3P9gh4TSnqlDqnkF@@*gb0So27r}%ycqmG1S2MR){OI_;Y#@btW91wgLw{(-uDD z|MI{~d>lz1MT#yAIz9<5<1yqaMa5xL6<`8v|HS()?^MFXeJvMDsfBHDgJsKkGd*c2 zNHnKcwdB#fw^whrJy}407jwcRvy)oLOi=wbdFQaD*v+x9^(9bRMClH8SUyki0CVdIFGoJ3iOuf4Fw`;Kgl+yulHVG03oMd@#*$q9S}DmdUaqi$moSRCxEq6S*#5*JM2~Go(O{d$Qe3Z@C{Km z3AXqN>+P}dsE=lmV9NVmP~^wZ1t)2$iytzTT0W4S?3~hODiucSFaOsm%R?h&>DjyRq_8rdgaEx&3u2B3_#bJ7vr*(Ph@IrjyTxbM^98vS__JnJeTz-9e!GE`>2CK zsb`+)7>{aA1m%#^)MsKlUM4lZV;TLKL{*qiVqnsu|FUuH?z>#;mmQrN7}Vzdi=vVl zE-m`Of)MJ}O8q99oUE;%cehIEtQNL4Q(!u(g=L)N^F|@YgQ30gD!3_aTF6GU*d;~UR`f?s@N0goTK!u6f&P~9WRPpX1oq#Y=c=DMLR)yKfZ6q z z=5ZvCEPYsUvbCaAbh+}OxcKDh%Z411JxWR28$)SL?2$(9=uvbs3M8lT`t|FFFm3Ei z8EO7nQtjZfo!&W3Vomz}nWBt{VygT+ss;Vhmi?BdcrI9=n3ndQee1%X{{A=Te+B@W z72LLU>%W=8up&JUH|f(p{%8TSf_8!Mqfn0vs$UdJR3 z{n83@h#{zplFG>d6MNQq+bI9aB&gcf#fA9|iu|eGnOrlJS~&qCQGfYj1sgzY!^GQt>wi2AfpA z#*>w^r{+Oj(Z8~O-MU_cF!i625)!&Ga5QxEbr_Cr&1DTS<* zkC};ZIy@AhkoU^V2Wjruf=#6RZQjg6Puw18^=!>90kUzURpC+!J8uHE)PYaVlgn-0 zu^3-GLPgRCDo4`o4IWvN!{e6X9^t#%dk#&CRC}9#n`d6e5e6oqQi*&Ex*Z53#Djaex|Bvn>=30wc-mU5+Mt=5JP?O_Ui*tP z&d}9l4lnh@ta}~$9)FGMQn}4R;eJALXE8`@vi>D*iJVP0-FL$Nl~i_8dOK4TuIA3uH^8DkC|9iD=UN3{JUiS5{6Hifi77HEY%!b)A(` z=qvsXV!6j(2=gfiv-kw)aeWck8}yUXE>}2^PK_#kIL%$PSQ1LQW4EMPmY3&RUVlKx zj*0oR$M2q;DTf+~$Gyx71*yKzYtOtTa2bCwd)K(voH2oEefsoyjzY4>0}|{brDf^J zcwLfzTYGO}Jx<$Sl6^^gjy8{lY$USe^cx<(H0k3xB|H~3{G8~S`O}d(#|)e z5T~i(D4({jpD2?@L(J`gEEo&0?9B2lg=96qT~y2qx@mV4|MfyFA6bb4%!?has6-&C zxNA!}2g>SoOwob_okjxqj?(Qp>*qEvYqLhvMlF(lq%}hQ$7^99FjJ!l9WY_s9Kohr z|9QoTnl2= zE#9st8`1V`8VCNU-!|?|VD`=!!k1VOu^qd|AI6Ss zOWwJy+PC{#HCFrRTkpKRK@5-TBL17#8Vs(fs=Gs=Y^7OGdQ|!1t3!|!U=3zzVp>{I z{Ysd3{_WS<@$ac^3P{E@uloPed(Ys5;9a9Z_>8vJn_kq^r;+;1>C-a{f>US|PI$Y= z4o-AzDcdK_Da=@nGu3F&6Vr1!Y!=bJN*WHb)tMGW(r>So(g$#F^VZv+a33!$o=-5b zyQ*ROk74Z#CqT&WIF#?_;LgliVaw)__ccvC9UNw+B%j zNhhPlo?EA3&f}66UR(1Hgyb5} z8fJ_p3_Jq}Y6KNt!n-rq8=Z{^9NTr-t@~}%Ji$Y*GH!KIQg5DF`yYM?8d(bG{c8B~ z!r}*p=3}o#Z*tD*D`I8VUy}J{0)x!^C3Bli8;O~jKEb2L9J#oReuIHx;C{7o|Ec_k z@sbyUy1Os;tEsZ>;k{&u*!L0@1=v4o9X`a(-1gWLW`A=B4M%h>JI|4U*5Es zcM7AUE8-Yf@4Zj%xxbfM(%g3)9{r>G?ukg(Ms=m!JFpmUmOT?rDjr}lDQkL~+A{}p z+XoN|lNY7z3!%SAO;j>wwhxAmvzB;3|2gAZc7{V@rf*jc1X4n==Nfa=Wt5b*q=BR1 zapxqqx- z!-hwdvpJ(v-o7wiThwCKtXU!}k^Df+&dB6{68Ccf7?^vopBhib;z3G0?UfZ|H~l<# zZ{528_8W%&DO)CzQO2W%Kb!ia?nTwzi-XSAHBNhDZCW}*PYd>y5f4V5Gu^ zD<5JM;$bqJ-k77;12!ZiB-AiGf}XtJ*>;gSxO+1u2zeTD^E@k8;q%5O*f+UIo?&r zoS3ny-|_s>>vt+=FhRA6;{0jg6+`dB^47uzoqNymo*ei?PG3T#a13Dy?E$2ABRa^z zLYh1)nIwI=Or{duJ9z%y^e@|-`+Obog_ZArTai}%r4nb)MzSq9e}e>=TJ{)U2KgzU zm7$P9w@Kr~gUdaPv%UAttcj2A(rA&oO0(8`5=}?)ot-lW_%s?cYgUbCypCV)Q-YYi zlmXHq=E?5F(ec1Z#|yrC*kCtrTzC~?(EzL`R6m;gDO7h!{l)fLq<}=1y{FG312%hE zUZ#{rQCj;Ob+)}FFgN*)h`U9RKB;j?mTipT4g$uc_xa_u9!F+vW2E8cXrJ(dVKwUQHVp76Tw>e&Pbv>$XM3=O9juiUm$Hdnun!vEfn* z!hG|%61S1>Exqrh9MJFhHTicf_m%dq;3opiCH3N8T%<6cgW~~+5wr!)k7Aa>V zDNiT6us>5b%r|(SpG|XQR_x@xzH`qL5i`@;w{M>h)NfXNa#sA>kOM19&Yc)H{+j*t z^BWx~4P;!8!q;@802D@@w+E;YrIB55@#D|sec^3*+LUK~5)G#A$M_G(I9;Wu_qQR1 z8v9~DFHDk9xf_tM13A36r_5Pt=~q?o5H`bA?T23dSJd+F`)Fr(TuQabSo z$72gAy??nM+1tDiV@*fJ0*E&-?6O(@o_O+8eTA1nF6jyRMyA}GIcMRe5`6xqWzpJE zrT8PHUj@NzDc)4%^f&J=wDQei?hb`UCswtb^E?CsOiay_t$+Xd+>zCo%1e*p#N15f z%*^5nH>nutT3uhnDKi!WkuEV1X+knfj_hC2VB*B{rLX$d0>cK(PRdc8&K`Yte&=UF zB{x7Sku`*}BWB&d$$$%BIbl>@k0C#4o0-)+r!RDb&?(h?Hc+sZ(t91?X~7ri_u=03 zoYSbVUOks_Z-nGA(G%o7k7(F_q%F{A(ncs4@>4g8l5sq+zl#r4UlaX76nlY2>u`&u zJLhnz^B7dgo#T5Pd4VUvei#c!%Sa=|Zrg0Roj)ju=GiH1g79IxaYR^o;Y3l&oZi>Y zYf>T!6q>z=(fBwCh!O*N!4KZ` zW$~cL3YC$+v?HkslX-HkX{+KYAEflXSO$|_d7uTcG@$&KtER&f>$<}4P`!amBSO7oM z50n=#uD$^P4PMiOW`b*4FDu_<8fjV5dIyR-V;HjVEg202?_N_q$Uuqcxz#|Tt1BLd4_M6S^EbiHF#)W_UB{orQ}j#2FaO4XX$ z4Gi6jnm!H8pe{(5TeKZUZhES}w(=hN--ly z&r`6?o#D=Kpq@Ve80Vjm_27EXw$#ud6X{ojQ8j==x}?is+ML65?daJnAs%rE%=g2M zaq$7p0aS4apWc#w0PRza5n#46DE5gC76IL*1y1=YEpP!i5%}%rv~jp{#Nd6WVrWcx z(I+38JdjfcoY>@?3wuTE+R}GG`)AR-JfF)wV?Gj)?d{{X%SdQalg=n!hZEnzE}prx zf==s#hJTSMiM-%YUmxqM>n3*?zt^vy7hari! zQUW-=Lyd^`emJIE8J|^C2n$`Tbu_Ul(KI3UvqW7=AGSHIl>QL+_%=Y+VVXkNWH|#F znJ1M{d`L@-ZL4|J%|KFwWXrANwf;FpMmdyE68DkkjR`)_7^PcMw6lC%`W77+e^hvb z@aMvPd`TDtqfYvOw}Oe&n8{Sl%gf%UNwdjFB!k3##eU_QSzzx`+>jvA1o2eZ(a?k< z+?57alX)-SW5lPH7OJ2yl-Wh)PWMK~96;nScOPkpiw_6ZCn~4pcc%J-K)fvaK&76MI(%w_rK)oGxCYSEoZa*xQHK4p? zVQtkS%BQz`y?=3RnzQV?129R|8i*&q^jCjGPvEkXrhJ5tKWMlZa}F;y6il{c#xr7` z4EGh9B)yf^YT>Jls<^52FTLJ;4^Yk=a`YkHhlZQ@L`$PlmKnQ7CLZ!95Adpuu_5Bd zL@)=-!AK2Cj}R{!MM|Q6;d>m=#D?TPh7lwq`qj>Yw#=(r_6tgLg1;J}@9Fzf!Xfa)nUhovXf%yWL z@R*yVanT@j`zg-JtT@m#cTb0jcF-OX&>|Fau%G>N~@IaqN*k{VT6j( zMEa2om5kzUJN4;vuYCoN5dnONduGApb0rkr2e=Kq+ze}+zGhHcEyER!UY&!vNLhQo=GtJfLqeL#8_bNwK`~Y{V)8+#e2vOql zR+Is#BYzi!I>~3oCTEQxZytsLEox0YUnJ)_)t=0I!u%?g^jzp_f_wr)0l)3Wr*9G& zmPzk#XrZKt;WMsiBb{4H3eqaeRJY*&{pUTI?GrGA!$kQf-+0L|(B&bF<<(cOG@6di ze37%v+zL*c`N0M-D#=Z{0BOA{@OZxNhKeO1+UG1#_@d0VxTC=L9tY=VQrQ3|r`Pe_ zZt4?FzqfvTkF_LR&3FvfC;bWaw?RCg0Glt$?Ed4Ax3^%D%A?zmyUZVE7Y)9b*C8)__*UDIy9fFEwNASUGCGrshq3Lj5!e)n^iiw!oF zDAv#iegxXw1D9_l)1+5UXY6V-+;n~?e!o0?yp!`1CUw^uvGprM_oVYz2cg)mzuLDHBzunT?j)*7Q8?xTZGu@6xw9Zlr z%R3B_(eHHT$4_2R82L!^(mwL#=_`PbqGy@u?f-7am>pZ$-ePb#{=*0xml$CnP)H_y z0QvO7r>H3J8aSmLdwndc$UUn!7bm2pQl9{qc7-tol(i3<4^5mvkf^X&8_&4nr_W2l zzn?x&q2;`XL!No2)1em!vpI!sJ2ra4SoV&+R6g5_4dZ*r0NNJj^M59v9}TgCBRw^A zpq@aggu(`vW(NkBB}{z#;O3Ko1!H-rv|HyoX89bS%7>sPN;;x&+{iFiwN}SZXDPOD z>RhGlv}41D-+k8;{Hlwz0&a;fS|O*WP>Ige7s}_(Jgpsq7YEPIQTq*^{)6VSgPx+dp=J2C$^- zxFLsHU7JU3E=fVSPq&CUCB#(ZS1-C89VR&zaMF*lpB~1vy0MrZ4i@&isc|Kdc5V6R@=V9#!k z|M}|^$0XDJWFqva9hT#1sL%KG>`@9q(_Y-!aNFL`NxP4}xugX$CxO;My*M4S)6i){ zo54nEj+jOrY9MW)BPs&DcX1Xq-9&wmu84ps^!2prg71(Y(-37InwsYMe!+@ht*Tx> zV*91y;4J+Z@+w;%w-Aj;uc8DYSkAhMy0qp8>2A*VdaCFIg>NHB#ndu$w+nvo3Ede8 zCiYs|NspIYwE`j~9a#@IF#iOCaovxBgHl%D{QlAp&cifP8T~MM$=|fwAS`nw8L6C8 z`)AHNQ9_8*49^9wMI>Agp39*=9kn0QAe?57*cu#kj`LV(0Bdw!#YLsjyqx+e3!a9f zqC-5~K6Amu6LsB31GO4S=@sPnyIxaKSVvO&-A4x{9JS@8i?9tv{lfK6aj42P(HCI^$%4FuEEph(UR|PB$_wnGhNzP*w3CaqR z{&ebpXy8uT+pi~GE2@G(qof`Ewn;~%)?}iTEWJv8{%hMEOCEnE7Qq_Kxmv?hy2~^2 zJNt?hIp+Loh%NW#es0V|}pn@xbWtRQ=1ugtTujN=$mI<7GFh<(YoQ#@YLluyV zXoH1bAcAOcEWtTM#4q2PZ}!t}QnH*M*YrWH(r4MnL*5x=VLBzk{Eg&I*v1_Q%KJmF ziwWg2E>JQ>S;QmK!3M??b2xNdA|#4xtF6!K9umg$71)7?f`+bu&_|p}=GcAu__2hF zqOiH_x7Rx>J`a^ChEnNE2e%b>D`&H$5abO{^YXS{>!|I{*OK{Qd_PDskPL?B0U8Ql z-=dMKmA?h`HLS#TYAwLZD_hfW;BL`CtP(NIF0=UC`Cn880o!}u!w{T)dF=)xiV^Y7 zJ=Eu0)T_wa%0zR@oyhbYO4ZFULA)RK&?l+O6XaL=rN{@U1o0&vEp`J`ja^1+VW@;c zj#ym~rw%yIkt{pv?GdVtEm!Ec9O|KbCE_;K46Xj?V$$CnnfX;`rtLkUstMAJW!SrW z)BOdn5=#P+R3tdcImB<^-Zx5GA4LBMJl^z6DRL&U6mMMmL;DJUdn&H{AelPfy10t6 zx!Rl9&rv)ZTK73k7?Vo&k$pDmaIqij(eIpAVKV1G^1io!%BXFT^%}WSSjd?yH1C8g zkB4U?Tl8HdGtq^i1hNw)id7BINTgWDW{+NDuYK=tqd<J ztiiWi*p0OyX%Ybtp_ zPb?f5E)OhjIl<{IAxDfH3)Zm>5*!O z2Flty5D4*K2+!oI8iCgi`I-953%8ShD1N2p$kfKQ_!s6@z1;)TkUeNak$OiauZs6H z=-AfD_V7?3dJKo~5d@K>Tf1631;5XQ;0xN}tu0%&go!;))CVNJcigwkT0x5(N@-_{ zPcmR}>gd2V&8`$qt&{%kx1+xrGsf0dNi^wt(sR`jCz=A&7hbj4`*LgG)|qo6Qj>SP zt_hs^%;Cnkz!4L|-*-;?!{FOm^JYl7(6rfQ-n~Ilr|z#A!9_@ zld^m6y|xK^Land`#fWubKn2vT7K8>`du>@`#nC#3a2t-Ew$sL0dL8=m+Sr_9RUwAkrS- zzC9tz8LQExnw1uwd^_h-VE_9*y32mGC#nk$h|$@lz`~L$m%HXn(W*v6mMzUE$`T>zYKc;YROeRZ0;W5)wz?@f(Nv7KaX(zHNhyCj8( z|J}NE>&C>BX=%gJlybp}X@#2L_FTCg=atY&m6NVTq(?w;z!U8PN}UBPqw!DQ>(`^z z+(+ahY*&65^~=wNjb@C8wB*Uq3YkU6zoAmO^h7`^163+|xz00-Mo6Z7sa&Nk{{GdE z%7{3TyN>OL0S~1{bUa>(&zO`#ZArpbA~-*_sGx)0TSY}RQN=H?v$Yk8Y(WUr`Q%nv zLZXkzYxuN2k7eYr6U&j)_rpMUp1^1IlUg1ZY90b6JpHp z5Nd5v0s|TP&}Ylmt@7r0%Buw1B1uPnBkC{8@!cm*oRI6+x^*;FE64){JjeKH)26NO zJ?GBRk-~j)bqz<}N@A$Q!i0zv1P7b1UekKXFsb$G=1U4cpKR-N_!t`=`Kpm@wNGgf ztz7S2qDs|#FIP_J40b6ztHglvTS(RE%1bpnsOP6^D|X9jWSSi$A}m_zD5e-RzE4bS z>{xC%!K^t%#%;+#?f?{^b^5sa-clq3((116k19_tczHMz<16BOX%947cP*N907y@jE zS>VY$VUTN%PP5_Hf1g;FFjnFSdPjnoW)7v@2DQg3<)t}Bi^8#k%qu4&br|g`Pn*1u zLQ3ZBV|P1hWZZ%PE~Pur{G`MG;8vq$T#yQ$>dJPwy202~7sh$ed;j)=SGR86Wa>FS zV_3S!{e!~%FMv6G)U3BE*_%7Fglbe7iqsR&$%teN1r_6Zp#x{`WoX4aZW*%X-_O5` zae?jw&VWP+@+BI>48U9F(u{w{Qdv3g@kKq%Bb5qVKAz)tlCzW3KH-?-LCK?1&FL5k z0)}@$hI?%f>6 zFn76n-eUND)*7v;JsLIl&*eA9lZhF--W&;Wk%EDi6lFSCPJ-p^@USq!x3Ou2d}Xy&2T7E46zU*H=*M6I;jXNES=-n-KepdSHW#78c=0c1r6Yu>f~drt><1|N+C&>U0q27P*c00 zV*2-G*tIk&RK5QCHWtT?R*eK8$k|P*oA&D6y9|zp4O9LOZ4gCZ48^I|%U@a(l4LDL z5PFrOL0t@NUK&peZq1lEbH60TOCGFhN|N-8=fGyGxP|#TB_HV2)17` zefSA=)oX0T5ePb@cLyRPBX7R}=0`s$s=P%Yudd3l8a#Lv12p}1Dm$6{<;b17I_WP@ z!UR-OC*=v5Isq=kQ}r5j=>clgu0-vUSYGZ^j#HNH5y~fU&BSvnfBWqvl9u_f>;1t< zcRft{N~V5DU&?0iBQ5UQ2+SW8FmzG5@kX z4O~*-!@q}7zic#Y?dspT4o{otPQ}cBoeG;NQ=*CIq*qHfW!+)v^Eln9i2n)dR65c( zQpgC%1m|5ZA`a5Vvt~^}>;*;Nch@e59{WCJVRnR~t4%O7wRjn|5xNS)$fKkLLoXRR z(@Vl>;PKMNBqiAk_6zVp<7Y(n0-H&b4#*+KJzQm_qjG_Kg~X4RQpt#ljyus*)TO4x zK(uLNdDWl? zuV>_+F6wKcSbPs|OG;i4jOCM;CDwHE@R%B-2d*l!e?Z6?4kFH3itiLhq=mYa5Yg+- z>|+8DfV$Y7a6W>cUnLm}J~j=Fc?uj)hN-TG5%-6)#U$_7sRRqM72JN$*ThFY6^ums zP62M=@XZsiczZ~f8Z=X-GWwaj1$vN4c-BE9Wbf!xRZCXd;1DxTk^0-c8JAr;m+Bu_ zgk%?NW;dd0QCsKCg2Oy*CjjA?rQ`#mvCkbZ$b~GNnTPD`)oq3|aN@AU{bH1um^hz3 z^k2mJom{w%xnekYty*N0r8NPF!+rUE$6om!%G&_3aB&{@?}crPdyHul117cp8sh#C z-ZSm;<#8)koV-@j&}Q7YA$8h39ZVpRe*L=Lgb88bFXs^$dNM3bh7(}^DT+*nF`$S` ze29^8@&o z32U1{H{HvQo%t1p7-*pJ-aYhUYcvrZ8jx#H(;oHS)nvh)m-qQ~Uf zhRv1`mA{Qg;UkEpge&wj3PK^}Ah=X5T75e*ncklTS1ihhDcWbxpNqM`Wz{_JrVxbE z2Eus>dv$_yNy{l$WE5%6{)|TK{&yd(;HI+t>q9B*_^Nw2B=|K6#0fk(*5W3O-O`UM zs^dU271X!9P-Q22vfbVXat^R!YETHTn|z$@Q-UP{=-8%cfv4vf&_>~~SWqqW+Ekc+ z2%n5L#GbU9jLr$CIl~1|I_39)>%YUyLUWmvRJ`=fKwg!a_kkX54YylK37#3mE|$L- z4JOtQ1%DaNsX6t1?`6fO>nG%g1~eQ6z~b}b#XQ*J4nM3_2FnCs8B6*+31rqClBQf0 zc8GeM^%V`$TXp|+EgZ!*;;Xx92y}FuQBJn*`j4=gIB`2z87H)#)O)CX5NV|gef)X? z9>SWFDH6xueDl|_?p&kfj#vmS(Q5VL`{n2rkRD_$9IWRlD&Hs7(S~ z?50bt*0~eNj>7awNr<1O+fW=1>nvCkdrX*b;C6xr*L#moU8vJEf-nUIt3c7Z)U5)e zq$=**uX6SjO^J8b^OSk9%!WXjlNp@$v#$k^YDAx`aAVg6fhQL2)s~CwV)-ub$UZ8&JO%1j*5Uvp;bu3e&NtD6v2 z`RY!9mEPHX-I?e{ge$BbPX2+u1WMjf&wDSgSRfN%;*PfVG~~|F3sxzk_BOq|Q&dN~ za}`eNy&OjtBkVF>N~Rox>IkqV;mX1(04QAg?^?8oK-{;uh%hj@dhdtqw|Rmnq>s0s zyNd4?fR6lt+(u7I6+x*_2+Oz()$@d>%H|?jk=S)3fkC(K-6LCTxh^>a6(+gLuq0Pk zSJV+EY)M}w>k%4yfNVG^>lA(sD)GR|&*uEav4o&JYr(pX`GAqq+a+UQ+}{jzc6Q!X z^p;fYiCmM?zch!Q+O8ebRjSY9s?ZY4Jz)@tTCs3Zoj zD~Kb}sNU1~Tipw?@uP^oB1+02vMpg~7n(Fex;P#!qVk1G~o?>8<}^ z?b)CsEZX{^%#@<6jRPN(Yb>!Hd%3%Oxs*z1U8locNb&uuD+A%A8sBT-GK_eNSXL&I zF1h7FwN83(F|`Fw{c{HT`TOgX0GUmx4AN*h9c0z}sED`NDY;V9Zb-07%@NK*9Ly-U z>gX9B9x=FG>9Pjmv*T3O!8G{{_;@W-{t<1#drI|kO*&i(nVq~6^j8&R&SeW5n|_sq z;x1$0OrqYv$0`s^lQLeJQtvc(%wgdzC+>P|e#+1MelVz8XA295nlrhLw#ox!e}amV zjtj+($es1p>23e`?O`36Y?%tl1WchX1;;GV8z$KfK_x5j$ z3b6`(36Z5IeAe0#Gfr0K93PDAjjtN@H^LT?nW)!^D8zCkt0(PFHJc4J>gTvp?$Gz# zMCON3;MsWUFezocbsA5s`>Y>~&y=|!d-wh*x(LkqBN=)!fjF60Ulr6Qi5YIv_z+{@Sy9 z1!sne5VmkPV3+s^?w>F?N$JaWen&|7)rXK!-Zyz6xAr;>w=ee3$J$1I)<~EwbOe^& zi6IUOXH|E4{O1!@a$5mo6kY4ukrWYO;}%UC{%O#j^Qo!U?2<@slujaCyawpMKPM+= z(y)5Xg5%>SnBMaameVK!O(YFALWt5`ucsrZPZQ3FV;3-e@7{lQn&EZ?(qNNhh@^-i z=={ScHHPB@{yxhT_bXE8U?Ii{_fDT4z<-ep*RwS<_R`LuALH!I9D4J8>((_aDByUw z61|#4^_QT0mwDXna_A+IgNI}OF{HA+^b3Xf_^8}Z%l~l1QP=>9U5A$~GCu*CA|r6J z@|7}~Wa7})v2{|+mcEJSjs*8a^(M~feNOTM) z^0(y5fxt2RMsE|+m^=?O}1ZA?s zweShTd17g*#}`8QBaA%&H^hE8gv8tN+|%hv6oJ%pMU-PJ}GKx)j;ub z#VFR-9K&^jHW5Lj%`(x=M`MfPO7#<;^&&=@fDXKrCM4N3BSwAQ<^vuf$Iz!@%kJIL zlKab?`O%}tGlz(zxeVub=SZRSpd()Ya;GvEh}?a^pxR06MMYB|y(zk1*y^jV+SadM zKVTmFe+VzF~6aHTIcJZJ1zZQ2;DJ*)gk7{+Z6Pnc{GVC)`>)RO4+LK3j$ z*fbo*5JumjA0iofF_)IDSG=AsziJY7h;jyb|BbQ8u?_MK)D07Rm*2iyw}`X?;_Iil zmBX(0iEd=|^49VAVOWgJe}Zs@fu$AY|LByiAxR~#;#V_aj|i03#lqsQc_F>glX1>; zZ`}Y@G4Qqkh>&0o#}~CWKDh&043^Xgj{VRx&gI%bQa51&pHAfoJ-ZNyUqf7`1Men= zDL(%7PP30G^n$xySyGY^=b!{>K8DlsdPa=}J)=?^b@8H=cwap|J?4|yE?JU5D$jWA zaKiik4!JOK>3(ur-@B#xf?+^En*sv^F*opA=Tl9PwmtL5d3l|{#hMq@QH5DJIXO`k zKeDyQyBlWKyI34iBQ$xWOrS|*L!16{dQ2=XFPC;u>iDxJHC4GW{Q8d{45mavFnNj* z&Xj{fk~~T@i3vo6%O{&#N7a1g-iyTl%CZH+wCMC1>BC2CLANfgFhiNYt;YP|4r9|wd5(Hrd{RS;0PNgh`wu`mE0(0xBTpAMoZ}IgF&d#>~=aC04r*T)qWvYpY|24_}vx z0K@XaLFzI~wAyb(CZEut>blo+a0LVo2_Zr+$JX`VfzucDI~fNg;RN|+B<57n(}leL zv1H^7Pne8rMFwm<{MNWqwo~SSOD+mY5V6L{JR*A|xBt55s*J6p`@Q)YpL5uCJ%eqZ zFv~2?bv<}1g3=C{At7dAPN6M5qJDclK{ZC2n)c5mr{4M3Duu@a>TQ2EFH`RA8W_Q%Icu?7V(msX~fd(hmMa!*)T zm>UXW&yRUAa&-5i*;LjrrTH8B6o!>$C-2jz_`znSqpQ(fBfi>1VHt?+YF44q;2QtM z*^ur6LeTk->0*8&MPQ`CBnK0cocJ%JuK#}(h%5u_OJ9rW98 zp8!Kr-5f#g1Vev3cm}md4wa45EXsx~b9l)~l5`PfpOsd{p^zp8lc+kX*D=rzCU*Zp zYRa%z!(U7h3NBp)Po6wc`a1+iMq0s+wM%QcVrH-HFKHclLL+tfjcm(7S|*LtdL#1! zv_h|)eJge_#J8FZVGuaDo)}!tjJzXFaB-!GBa1J@qX;TvLp@saMI@w3JdjX zNXt7*vqgEu;@6Z&5JI8_e4Ve51%CT&)+7(20|~P2SUL<=^Qg^8bYDMaevU#n?K#>N ziJB}B$a~7YdqktsLqZI8dGWis8{kh!UWWg@wEo4_*X#}*h zrb(G#L}6UrH#PSqt#N{Qsqx>?`-PQo!DC}QbF}7;q}l$4>(nmv#%ep zY*`Zbmgtl2fY7IN+I!8PFS8KjG!ihwNNv7}k-zmmauQtJzz7skOLK#MEHUaexMxfO zk`P1IlNXcq#GPnf1DYSPfT@9@|UjXh43ZG`~jzH<*plyet%vg&@X&zm-FO1eF1Dnr>6Zv6O(?TOgv$38H(o+dM( zO4rFkF4xp+`Sj^~XbYT`wh6=y+WLg4Y!7i<`Od{ifzUwv)691~YyuM%@i;cQ4RO{| zdMT@W{*?cZ!%ltw>BGFd5n3@JB);t70Sx5u%-ivMjNaEs`6_&cWpx7G zV0`n#%|ivRwoJ>BxUPBDH7a@7on zJg{ykqdciHL1;d3ByogeXDPEH(ct@$mck`e#gjWYH8y@Emw?aJ-$9@U z$x)Zm38iMiA5F78S6(|Le{3XF0XwSfjV@h9CQ@dCrja<3x8jx<2osu=)K@bWgN}`e zShtIZAX3(f<);z;j^Cch$csnY3{$HCe#PzA_nIzUx-=lGvw^{}{3Sz& zo;$U(bZM8riaz_iqpwC*ten7PR$5yPO-X34q|YK{$JA_PHe1*#DRd;0L9X}EKGCqWg~NuVQ8+$ zx5L-#j1s$W$p{n(sk@T`05Bz*~~*6c>jbo%Pq9J8yS- zJU)s|$GfTX880^u>F@QuK|f`3xfUIGrxVJ63>2&$e?oy>-%d1}IKgFL%646T674h6 zG+;vMap_WG(YHi9-(!a2Op^JmY*){4fd;LyH`1xx$Q$ECHEdMVlg+=4A@)~w-H%J7z8i1Rfc z{lBOZmBE3^hgpn{_--%AIu1lSF@QtjuIhgKN!TAUXQ)}vx{r?012S54j5W*&zfTKwO z5e)6$8{G0_QXaR~j$<3nVp3L$ZISo~-U zl&V0rhMN;T2fm_?1|!dDdA)S0-Wdrt|Ni^$u0^b!NHdP1RlDKv`pGU3Hf+gsnMA49 zaAmF`wLZj97&gsUkz>hBme>hOcWF`MuwUMu%j_XAN1%rc??7=a#Co3dX^dFsA~Qjhm$$e)4?wM)pxyf*9)ocp+YFFGaW&N4M_<=rlW5d6E?&# zZC0SGaao_KjGE7Xw(dgLAouZpwm7##y#q9G^fW>R0*F242aof%Q~MKB!Hm z3=`xbG+HU}2Fyv&;G$|A!P68X#Uozy_)nT@3M|Aay@5!{0g|N5iVFG|HvU)9Onx<} zHK4D7NN7pR2*}4ZbG&{uK%gzK%7e*fb?p7Pd_zFtw{jWOQuZhM?0Z&<$}WENm|ifi zJX~16&9Wfj$#S&3jw=2c#aI@=KW`|v7c01oWkTT`pId`^|#>*P=pVKffehCQG7^@*VPWaMe8;${A2+TI;ROLc51f92iWjP~u&Ruhf<_KlAA@u4$ zq3vw*u2%OSA)(^4ItGjq?ma_g7oK=-M&>6(d?NR?c*8S{<{~-R2>uhXn zCpz317wF-SG+ODI#@#&SE(nRdY_+Q@ zY_@P#sZprO*)-|UIU+>WCN?%*cT0R`V~5+lZ4LM>PLS~F@4$#UL0|N?oA>31m#h6R z#tRWLk{NInaG`@aV3W=*U%nh3hT7t#a&-{EsuAWt)S2TV45`RRJ!K3v>0npkHVRQF z$Qi`ji`l+|dSmIHkl)e(6SN0zaM_&3AA+a%ziZiOCmer~gm{_OEr!|AS>~kd(K3(| z!eqQKNv%E)>N5G{vg1_>MtJgCz1H42b)*BY#s%o#Bjb}!&VZ~EQ~@~40te*hccFQ! zRxJSKlA|Y!Y7heQIl+1b7Jx*CYsKRFQ7D+IRN6rW9tXRXf8zFF|EE5oXGned)G9{* z_@nZpjA9pR#g9M!Gq(GdkPqCR_iq#5jHF2T`r^+(o+h7zKl)Nw<>2}@6bx`kIi{k9 zoU*Y^3_-IdeNYn%88MQoBLF6!omiU~5m%8a+Iw#PjiecKDFBCpLg?Ci1S#Vs$QJH6 z$`%LGiuiS}V9ME2K$6lW!WSU+daeFnwL~5=z9-IE#?gwJDXkU-@f0LFT5Su0<1vX| zgL)l5^_efx%A)MKYl!I7nvH#5{6_}Cjks9`GR`RhhioMT2<_mp~qlPw`BEJ1HVq z(0L;oH(ubT)>0&$+nV4TsVic=1A8#N&nHbmY_Fx7hE4HK_tNH z%%813Ik?RoerZ{CI^&+HB2`r>2i}?xqU3GN#CMvLChI;K3AIHCIfF1Dy|Up4Bb>gZ@OT~vS|a7J z?NlCIK^8NASxtN^L%4hXBtw6v5s~x~jZs#_S=r*@bn02`1tEhzS>0mJ7^19(R-daz z-k~An2=yDWpIoA&8yMN*elKaUScs?HzWi|MeHtvH>s4YfAz6XFLH}~~5uQ7rt1hcsr%Jci--=d zoBsg0u6}E8OOB22%9SgB&1dWUNN>dMl7fzrOsp@3Q)0Bj>cfkQel^S5SY6#F%cM`U z)h%b3rS#0p-I?e2P1nOudJG&9(&X!aLE8r%z2^6)a%@mzo32BWcKx+Ax8;vxwSzkD z_d7g&;k~lAXR->i&fYX$pLMbFRaU{9q}^BY3Jwh_x>0sA`rJPB%lkh0yj|la74LV% zCp#FayEJxCH8scxZ$Gl_bmFAR#2?FNzF%ZwLdc5CIls_pn@eZ3Ec zT5bPj9kOu9zxVuBh+C>ASxD|dt7;#-3C@xf#f8CigN*V^JyqK^ss09mA~MB-VxaU& z372h&9B`};ojgcq%(Z8&mV1m7A;eHlTbGtVZ-{g;pDRA6F; z{0)-{=FTS>1xDnML;-B%@Or8y;~<>fTVC#<(VqR~nrNy2TDSfVhM1~U-ky9T6%Qi@ zbl+X#9tCvEo9sowVaLwKs%bI8RiNVlq!Wr)e7vZ=J-u_RXk~)ae`a`kdKN1y&0Fan zfLWw(y~!TYRS+d;gkGe6Pn@`;M)!e#Db|tOBf7#ygq^gJrEK%Kd2o-2al^#2qQc$V zu$`)}bRUb5j85fURIuPZAE1OE&_X=5w~E@tcVKU@S}tx|ny;XIja{d_vtwIB)h0U> zM{L0q1;-TD5fnJNt^xx9-PE6Zq1oE9al3SK1M<`hr4JWNE}}ztp#qsc0v0@bVR!jc zKIbz4F@lIvTp%o&lwt`41Ei4FY3r>a%+G-r-XSqCx6#~&;P#g<>&}n-{_#Ko zF~0fc2-cAFJs*n;6T!4k&G=Miz$-UoQasz~hK`Xr7@FZ!RNYq5n{leTG6q-O#Zgs! zvjl=Nl0u|dZVrEw+_h$I5uKu3st?^9x!pwM2a zhPY641a&rjOg0qpb^9TrHwUC+#2I+lQ1}x>tRVp&Zrf5ike1x_}&8I7}qZHzqXwTLV=j$duC^un-?8m0)ja)(8wa2m2eJ=r$vJxh}%JC8`RcUBV zb&Hc3dIE{i=QfiokX+Y9f_cC(?VKN8%%WMEqV$_%=T?6N0$wZQVVOm9x6<6=WVDqW z!7sn-Jnh2`+PIyP^Q%Qo$wfc6Ow)8O(6(*1h>Q8?12$ zji9?drzM~2cU??!b;Wx)K6%D-g5fck1(?U3*Z*XpAoKz?eIQ_?|YQu*?x9jqvePKOE}I`e&k!_ zbKH)s4tl>WlcgBgsJE&id=Xt!E1X#d&W=6Lq-}8)AO|-n^J&Hc+DmIHA#IJ-IDb=; zJxKZ*5g4KRFaxiaV<_cNk=kq4glwF9dZMWGBl6*>tcAEE$3!czk(9t5zBe25eBs^$ zsf_r(D|Cf1n@YZzNehndkr)CXa!RQ6_k<|+ur(HGN1sK}OI|){m+x#ih<5WM%rB}r zD~n{{x9o^U)G_T4c$IT-kuv6fe33F-D-=MyMF#b1*WQqsuZrBFh3j%&mJqw(Z{!2K@eECwB{~ z%5hu^Fj4cVoiXoa=N z&>+KWq&xSX!v0xTK-j*9@2wGcqTZraOOF$3J@GN~yN`>wv18Zi+8=%Y|9swxZ$~yw zM0_g9Cf6^frX__~e45Bw{y|DrBBMhnF7=pt(q8MwMaR7;lcY9gO3R6=wL9u0W3hc5 zG?>gRyd%UWM~uAXsJZB}ktaOCjgY}y#~azjdp?M4`Sm}=L4Va6==YDJdKEhM+D#8v+Lb8tK=3HX;IbX^9#L8!@v%PJeGfkUUK zp`Aw#Ca}9^?604ECUVNT*Ofnbe4jOhI^9I_Ms&VD`Rq#Vh=@RUU zT_DgCM1e~7t2s^PdT)C0@1N(t`y|q}VGZ*m9laEA?q^uS!qkzL~VxT_a;cN!-qLf4={J z66z+R8FVAT%Uuf3(mVT@$#8o_)C^BagTNMcU$uQM9v0aSSka99Ko*?cj(w+u%K6Q? zKY>v(b1SKAZ)5cCrt4mX*|*|}jl8zL8k_e2UvkYa#BcLGVbd->-DgppqnOP-lIRI`3PR|HeH0XaV1AfmPXNr5c!9Z7@X!(B z<`&wX949}W(d6G*rS2JuEu|fn8V;v?^QE;8Cz6t61S@&|MDlIpDPJ{Ec`tZ)DPTN? zc`QLp2OO6-RdnERWNNDD- z-o->YP|{4oL7j8=$({vBnBv8ywzB#}KstF7wmy~4m0M>4Y{bIJ1?Ig8kGFs+)Hv+6$0{hUtt-G; z>|lS3!-`+L=oMZ4W|3Oy{V_SN{n@sYcT!*xE&;cK6V)9O+;FYL{M08z5|TgCf*{GS zHW?^43H*@o#!@Q-gGJ1UM(Y}THZOQ`^={Wg6Kt8y>U$j%^I3KGlGNdw)gO{w|10Tr z8z_POV`y;^G(#X{dN$P>nILZjyt&V@z6|8sH5WroR@jg`jZ7;Y@8$>5V3)B6p- zCO76w;UaZYs@~~-*h*U8sX$XK>ZTEiC^9)2hEwyyrhm|`r~bb4nB<3Q8L!p9+8I@; zvxUn?HTm&_nJSOv9FkWI#$)*bSbcD3isLni3&x}>MB&Xmd^V{ejY^G;JswX`K9abk z=VV++9D^}rfu*cZUeDc<7KLzr%K03w$x`35!0d~(wrzUg=WkH=@4f6I@_<~JMHge3 zr$PRI-g6`Q57f=W#_fHw1>_S!nBgNvd|aGO^I@^gTms!v`iwa;g7G^Kt&MP}?MER^ z{sl%*UB^$XUyM=H&D{mtCX1Q=%Rn5)*pOjg%k>t*D`#JNOoF%VVG4!rIg{J#0}yK; zKKy&KQ=1NR=PGl-Xos)fh>OXN^P`!g*BJ|P&QQNPK6hb&S*Ou}_)B@es`U4FvGswP z_;$Ef5JH*0vIJ7u$WelC@yVHj>o!)ZmI?VzsjfU`7}#la>$1YTlbKw260u|_-9y@; z?lM31(#JVCjCiFhQgnv8)Zz*mD@!~4lbL$vsu@=#>{H2Ai_fMX%PV&ULCH`Qf z+IRrNL#<`SSemk+*v($p($VKbnIT!G?Di1CG6@Hfzz)$Cq&gb#J^{C^N1|usq1@{$ zJZMG%>^DR%Fi0O!0q8u$T|LRYIK#-225);Hi9B9_gC~57v_$b-=d8GZQuvvXgeX?x za6LNvJtA=t$5RFCIHf2WErmoV4blqWYx7`XW9M2Z35T_m* zK7@Z&kk%crbBM+NiQk|3U9X>N=C+%A7Y9M(wrU6IZw^i-i$7g|s@19Lszi~EMhEJ` zLr2{{Y@3Ppe2N!+0+*VZeQcocSrd&*hm5jSO^Qb2fii(koX4x*pD>%09Hd?)>^el` z@C0L*!SW;P*x_57PgJe#!~zmD*fFR=lTOTw_vuzi%u^GhtCCxy^^y{I_LoC??ejjC zpQrsvR3Cw((T3oLTmWYu*Q@Vr@bf%+D^qd4lGTcIfCOtB130|Wt0*{!(CGn; zrGraAYVx#;8{`P*7RybO}wQENlbsG9F@ITp>ngLMCHpsDC_`5KHkafHL zOgb?6SVkYlqX^W+JvAxC*x1p53dtYf%W8OimAAdPRCvNGu^C|JiYsxg5JrS4ff#2% zE5Y5opD8WLlqvWh&!fNp#gpo)qUyqJ&}I_Axq4;WaPh>`P40DjhQQTG8?4?a1w|qt zoum?wgu|&7cOTCMuaTmd;G83mpXLXoBE|yWo$Cy$E_!cBjm<1-s{Iw>*b^aMvI%}j zCapM^iyMOdCgbnyq}qQa9t~fpVY?p8Uy1+d%)pY*WtsmfOYLUKlfp|aE|7&Br+RxQ#83UQ;Dn6_e7M5qeZR|gF) zA(QB)i!)!J>;QHogW144CH%QHciwsqkn@E2;?a=2Zs6BeoJ*o9Lx1YYoey?P;>WWu zx+t7jaB5v27^dmr?(%zR9&$p~l|a~G;@|vp36C!+0o_bYOwRPCEOrC=FIX30pjIM1 z4U1TKqCD663F8Ukm^oN&gXxsHhUhjO^(bTMeprYa>#sj&KUrOWn`>!_? zU3o64Jlxq9Gw=f(?4K#Z*kC6_Q-pu4BVgJdoHjs!5xmEx;=W5?S0A3mum=%jqK4^K>rqrW*UIpIozZ^~!Wn+luv*MN zYa`_Lojbj|bWydPNid%d%_)Nf&>`M)04icxqSeIS$na5J#HDK80_$*tP{uKClF(H2 zJbD>0m_!g8O`1Y=diB!!4VwjlNC>D6Flk47m{c^k7)+glPV5qwsVmhIL18+zYAGN+ zZd69hGZy-f^i`5SwWW>r%t6_DhvW{sZt;@~#LU{C;l*Qy4e>3{c$KwaQv~ zQjnNbX9r4upo4-sNmDdGiLu~u4^GU366dR=YC8zE&u|LXVMRrLCz@$B47fk^rs{V9 zbYOMuoUO~2!A(mkdUY27Fi$G`@akI)Zj3b$6%_eV5k{s;kRTiuJe_iH?iA-Yn@cA| zR!8=IpLZtzEb)a*IoHVSI|3J{d9`3+&b9dj)WT&I)Qz5p^>3KX=HfL-fH519>&Ekm`Kd={WfzRlL7Z_Le>F_~18wzpC`jhB%v5C^I?GA%}+G3qzY z2_C?IvRB-Aq^x!?t$Le8Qy`wPokol`CZurjD}+-gv|~vvOva%nqFuCh%NDrVIm=Sdc(v1Gr?ObO$XcHQr3M@I>CNV|g)C zPo1P~dUt;7>{H0UcVVCmH!_pM_x#p3|7?2Gyw-21T5s@AeNeZVLh=moD>C5J`m$9E zsp8dC{kK}5m-pqSNZTY>bqHY6reMJ#2wS__#~)O?venWY#lWSSa5SgnI+Lez&57X! zIdFLU9E;E=@4)dr*mK6N)v#|z| zAw>0)h2Cu#jl!r!1HF)d{)sr%X1QPiY{1#*w_t^w8m^?<1hp6d|&`Nd&VKy;#L^{dgOL~EvC!AN>YS0FGJaZw^M#0T` z_NF?Yn3qcnnq;EuN=P!is@^S5ygcOcN0w&WbH$9oq7O<&HMCjrdO+jGeb?94lpG(! z;+M{GNC=`E0+_8CixZNb_5TJHlKY8cQ7}7{)YKZ0f@oln6a*GfgJ5s;hk8f*(1m6S z2sW}nC-Zhw{T1ecHU!>RzpKm23j+5w&?6C#y%dk3O7pR6tgt?d?`JsX<0fPTzdx1ZtokR_zQT>3Uuz?{o?CSOYopw0A8rJkg1I$%e11M z#4__nxXn;PMZ>BMGKmvmtG(9!KILn29?0;Rxy!N46y09zE@;Z^oroJ<3CJ04nLprM zq5=1Z9j)%({NT*xMAWlWpb(cgeWs6%^XNTC0jPC&3+ra?mHCA!si_*O`xh4M0z(%| zzK)m<+2;*?GVgj+;q01cA>N%ucs@S3Y z!b`dm!ig{?lNVkQR77h~#^9V5e3|z!Q4OKD$TBNts=r!P>PR#6qDNX;N%q=769Y)% zfACS=W}nK9@xS<}uAU-wPMX#iEh=AHC@R0Z`!0T#Fcd6mR*Fn?k*eQ1gDFWazST0{ z0F?5Xxs05T)hi+zf?d14up6ND&YgvoKU5HI-&yBTx#Ert=Fw?{Q7{#dnM?>DX5wCD zR+TqD-h08v&dv8!h2qZ%q*qHL29waB_+)Nd#270PNrB5r0+SA)t8zR=&fVf{Ltalb zqxX-WvUu?QWu@!ydVC{kv}ZN6eUa?u&4(qtNqJ4xL!#kDkq&U6FsQbG7y!go(}%`Q zmi)-;#94(D{eD%w%E`DeW8AM*(Rc<^2fzLYTWzxO)?}BqfcB#V30#-=&Gq#3mt>3g ztwTKfkTfM0Su?gKpoDbFsrN~XQ=Ajb{B!fSRD?}(IVrc|Vl03CWILzgNzu5uXHNh= z4aV?Y$K+8fnKS-}{KraptBw8s2W6foTJaLx!VxhsaJ%f=X`_(H86iMKJg;NNPgOam zh4?{$9V{Fpg47ClOfqGDIrd@uii$3*hmR6N<~XFwN;wK5X1pkJ^1*4ps`L9-zZt!u zYLQp{`fHT#QHvHVxVZIg=)QffG@EXZb|>GpPV;<}_I)_Q=4e2=)*hCQ` zuJ~@MHW01+I784?x6KZ;m%%7)XOP+S8m=cbAz&`)5KhUZ{-h0|YfPqahB{u%Z`rDq zF&aizSzCHc&WzV*T-T);9Egbtd8BW%QjP`sZ;tf)Lf-7`7su|$05mi=Ul4XTFW@=% zRgo;zP=8`<2s9*w6~=#SbGqS2xJdUl6gyQ!AToLZy61in;X8dZY21dP7>*0*xUVi5 z0AlvLWUU!EGl1(Z)J6E6!c~XkyG@tnpO1qbVkW~kb<833Ad@>QPVpE#8IYInytrznRZm;t|v6@oJmCIj< zhCW4t=D7fuW6l?E`$z8s#8~?dz9_FpP){O}Ki0(&{f70d;XVcVbg6Q^Rz#1KQhlm4 z*%|@d$^ck2GjuqjWg&IQ5JU}Iy)*+tnbl~877I>keNmR9Z(B4XZ25}yC74N>doH?Wm@A=089EdSqCnKw zBA1mgziz=^ zEfkH<;GkqhFTWMekR+2_T#RGFObTjEx;D1@*uz!3Ny&p=05OH(3f$lh&^hRGkycc9 zpnRG8fbf%xBCfqP-@V`(U0P1x^}a|GOA!*#!sw6<3iNn3nv4^fS2!>uyNwC*nRsX( zYxC!=cZ)6)&Vl7cx9$VZ=@%##$AYC&SZ70}p-+ndkeaszb#bSCat z-hF0DS|9Z^9T(Boj$S&6Ce)I<&GYnr?XlJa4{&LD`}XaLcHj7yJ+IqN0l3>j(S`{s zQoNoa1q_*VMzRzMeul=|wr$@~bg~ha9ZePxiiT??2}7D$&}BGtyT~X)2@6`};@UJv zof@5AAx-*HD_FOtQHxNvrecE^TZ4I%&Pl^^cst<`n-<=W7^19iF5sv{t)jiZsP78q zA&C|Q+)Bg8G7gyZ=GHESSCGa9NVp5n-6r;t39;$;4z*^NxtB!L(-u`dvprzjMTkAk zrY0^NDb53!bj7*bM8Ps)1h}SjkkxXOdf~6>KKl~decpxsC?e$Q9-~($zxJmpp$tcU~3T6!K=?)_!i6PU+@VOb46}o zKB0#QnmD|L9diJFa*JF%%`JO~el2o;t*g{WiB6~5L$Xph!9c_LO(%+2FLefO2scvX zXSL$-Za+I-cE3S&(m8?!sJ^ici$rl1$ZCXEq>BX$p*2)$oW0&PU-aze^kuEURz&i* zcjZaP6a?(JT6jEI&AdZod7Y!3pTMwNNN3GA@F# zGO;4)E0tJMZpKJmLz-6525`w{1xUz5b7JpoQRo4$U>v6u_vq+07#u7e}KpZSY-{v`^{W zi#!C_LGRY!t2;$m6~K3xR_f6rZA7?@riDL&yA%dRaD=04LntjAklZ&p)YSAA!&lrZ z9Zn@mNy!ew8LQ`ROnl@Yf|}F$OP8MSzCX3Ex6p-*`$+Pk`%K)-XilgpGRf^WomtEt znzc|}^JmE^tcB=8L)g2y>~;YC({gZ8AU&V@$CXT=*F(EdeiNCnc#Yuak79IuC-IbNiYm9>$uQ@*nh_5W53=_ zQ*1t6x)v|N{Wk(L;k;dm?p+Cln_h?HN(xSbEI~^_hGt(zMn<1k$f1kIIcCpw>x*h~ zK!NFn#@w)U#8`e;nka>yw#Z^X2$VS`Zi^yeKti9^B)r)<8xXJ8rE+9{T#2Ael=-Au z!kUu=fqU%~e}$s5NXjy2*_MDkAa9k&Vv$^+5c-s=h}1+wC#qfPD;03U!F_t*{{SF= z9ez{0Z9YRU`*pEF+fA)OcJQDeRqyLkDpM@Sa7Efn`P$i2!tANoHfm*VD+~lMED7Y! z-=)nviY5(0hVxKiffA(*v$emTFmjXcQj@n7P!bSBl-}?zw@zfyPC&kV$URD$x+JBe zW9Q<6<^0+@0nVndNk`tcCJ{f@Eh|dtP3OY%Ngku;4jOm2>&SRQ?iH^e)EE5&B7Es- z7V<6-!2#S>bwxjLVd+|kW~oGvLP!df!KBzMhz+eWX{yW@aIPgW@aKB)#ovH=A}|yt_YV-=r&R@U___1YJS;N zKa$FbOf`g{qz!>Z*2(|pmvAge+yvEQQ|W&WU=OgQJzLUM^!Yu30+Ol@Ul z^f-|5e4<7u9^%F0OYS-+X#nalPCD@mdVCA^o`spvl{S2yfmi?6?Woh*mCRIJY)JK~ zmvzoZr74ge@jHn1P9|20n>pe?XI3_@E!D#iz5v5t$JM7Q7Ky13&~+=Yc^e}Tg+LNz z7?B}jG~2AtZ|bkC5DqY+t-$^e>Qu#>*mvCow3MO~<-0|y`j;A%v^ea26CPns@i0Nh zL@7d(4spW!>spF@>jQY6E^yt2$lU06bj8)^yKaV<)S@m8={Op=d##Q096R& zA=TAI|FQT;=m6UHES;mLo{Hw`Y0}0XkG;O~ z%lD=@`OH^HHLhyVR5q%GsWQhFM);Ja&y5e)cKx^YX8>`g6bR8D)b$-)sm-zla4?v9 zPlC?m*4z>sqIpQ3-5}pdV@j>`YadBq3vFP;NcjkJbLm;<K?*|3@k{2Bo zhOK|{46;}3?OVI`%Ng)y=H@0JH|yMLsJo(ZLzF#4`w}sZ1K8fKla7Sx!gGG0xHF1% z*NDENWPBi^hbc*9i0@$@QLPgLWI0Y~La0MvrX+}$fr^y*iI2Cp5-76#fs$FU5(|K8 zL??g?jN8TygtXFNaOT{oz<#q!Csl%a(^P(R zX`7F$X#YUQH&CV&+MIx1Oq38Awhxc@pM#SW`t9Ae+*?{?A-&5c8-+Ol*d}}nO}?CL z5vHaL9L;SD-gX*=Uy-Y9VohCT2f zHN=sT)W3|)zE-vE)Q7G=fZreUox99Gx$Moedm}#32is;HoX}k&rk%TXb-25%-CYIZ zqE#(7%4esu^ewSNk`loEZC1*`xe7w58yJh0t6aUD>3oHRsYcV1RT|T^dejL;O}vI^ z{>qbbIYCPY4Po1L5_2(g=Tk|E*B4NEc62rP*ZJ5tDNb<{3b-WkTnfqD-jm5J8<(Kq zaQ^=NsZ#64gRXl6Th&!EG$tXD7`ch>DL#l8iH?$n&(l#lDX&R_szc|KTURf$dCvJC z37_(OJ_4|lbEQ@IO8S(wH{9Z!7q$J+5q(jn&|z#3aHgC^_L+}mV^8W0q7ESlOaG}R z`3ys)_vHyN-9~ms}HQYbOYNO582K%8(sr$FreN;o>C8GcaEF@lxs5N`HL z+Shdab=6HeeE%#M{xb`(>e2v7Zo!ftUN>+xBw;ColX70e4R?0r-(DnU7G+7%uh9x; z6lxQ6zSBC;C_X@F_=LRLkW&gZED~EC!B|a|Xpc_QgBUW+t=&p>>i?3-bE;<)UCxcv ze^R;e)XWLlUwe-3eQd1JU~4Pe-vJ>Ie(y#j8!Zcq51QhcTCZun22F=O|80c+ z-Gs|e96p^p-O{jOBhxXbTUlGrZ}9bit=s)GD(kjTe6~Cyo4fS=c>B%JAs(Zu(B(c; z47ZU~I@9y-%3b<(l`8pZXn1&+b1mmv(V_oJok~w|0G;Z6psaN_X{hos&{fMA8#iM9 z{P`kY&8R#_#-4RIZ{YJA8CN%Y*EKdaHYT@Sf|HOZs6MZ>dD?aS&oY*?_LKbQE8N8X zqeo2`p1zff&Adm0wpi;A;a z17l+zl+wHw_g@b=?Szoyam0DLB|eKCQB}MK#?6Ydh^hN9FX0U%)UJJpJP0;%@!2 zw#S#Up*$J);^(`ciP-tG$G?STN< z7|-uj-R@As@juhJPj>|pgca@HM7mm<<12#=kU8s{`+ieYs;$TP5l09Ix2CUY)eB7d zMO$fWP3QO~(Z{dH_&auG+A0?w5UGUA?=GBrH$wZ66!GGK`mOx%4&U1|e=V?2eu!^6 z4SDKRV$02(u294zbGj<3aiPEjo%_Fg(Bv>!sX7(YRK0z`YV{x1?U;+^>5s}FN$Zn z(=m}$^f>B0v2Pv+bkZP2@TzU1ST=C|@)F13WOHT779{^q`)aYjVl$DiD zWWZK+ewOjS+8rXl>DMP^jE#*iIOOqn_Ao|D8DqWgPcrmVky%Wx^=D6NsP@gvwo^7LOue===wGnQxZ zy8UR;nciJs1AKWnd&>RJHN_S8$k5PG`{iXf%;?fzJ4&wdgc-&rCZVKH;nC5TZAM$z zoh;(4EN0SMujzmN{*2rG*$7HJ=%EusNt>qQN@=6Nsy@3_BK z+U^bsM*DZp8;&5*(cdCe6Q*D{^l9to9%$D!GyL&f>!r`;RrZ5|8h*rn>esrPGwxgL zT`C-#K&tju+j_R6qkY;nqBNDteI2IpAssCjsR;I8q}-=gH!rn=e+XRE&$iT2y?c&i z(iz+FDH<)mZI7l=w|`cmBI94)P3PQ`pZO;A%|Q|Nk{J+sq!PnfZ>hoL5dE6CxqT zI4oxoQOThqB@~g$)1KLwaki*X%8(q2C?X-o5G7G6C1P}xV@cBK_r6xyWB-2t*Z03( zue}+q)_R`jzVGYsxjxtD@}HAH$!t!0JL!;iIrn6J^B-<(S1Ui%ZS2c!L?1EF_O*oKDe;DOF^R)QiWJ1zZ(;dQ)$47r-@R#W-A