Merge branch 'main' into feat-k2.5-support

Signed-off-by: Roger Wang <hey@rogerw.io>

csrc/cache_kernels.cu

@@ -24,12 +24,6 @@
 typedef __hip_bfloat16 __nv_bfloat16;
 #endif
 
-#if defined(__gfx942__)
-constexpr float kFp8ScaleDivisor = 224.f;
-#else
-constexpr float kFp8ScaleDivisor = 448.f;
-#endif
-
 void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
                  int64_t block_size_in_bytes,
                  const torch::Tensor& block_mapping) {
@@ -407,7 +401,8 @@ __global__ void concat_and_cache_ds_mla_kernel(
   }
 
   // Compute the scale for the tile
-  float tile_scale = fmaxf(max_abs / kFp8ScaleDivisor, FLT_MIN);
+  float tile_scale = max_abs / 448.f;
+  tile_scale = fmaxf(tile_scale, FLT_MIN);
 
   // The first lane of each half-warp writes the scale to kv_cache
   if ((lane_idx == 0) || (lane_idx == 16)) {
@@ -476,8 +471,11 @@ __global__ void indexer_k_quant_and_cache_kernel(
 #endif
   }
 
-  float scale = fmaxf(amax, 1e-4) / kFp8ScaleDivisor;
-
+#if defined(__gfx942__)
+  float scale = fmaxf(amax, 1e-4) / 224.0f;
+#else
+  float scale = fmaxf(amax, 1e-4) / 448.0f;
+#endif
   if (use_ue8m0) {
     scale = exp2f(ceilf(log2f(scale)));
   }

tests/models/language/pooling/test_token_classification.py

@@ -1,8 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import random
-
-import numpy as np
 import pytest
 import torch
 from transformers import AutoModelForTokenClassification
@@ -11,20 +8,6 @@ from tests.models.utils import softmax
 from vllm.platforms import current_platform
 
 
-@pytest.fixture(autouse=True)
-def seed_everything():
-    """Seed all random number generators for reproducibility."""
-    seed = 0
-    random.seed(seed)
-    np.random.seed(seed)
-    torch.manual_seed(seed)
-    if torch.cuda.is_available():
-        torch.cuda.manual_seed_all(seed)
-    torch.backends.cudnn.deterministic = True
-    torch.backends.cudnn.benchmark = False
-    yield
-
-
 @pytest.mark.parametrize("model", ["boltuix/NeuroBERT-NER"])
 # The float32 is required for this tiny model to pass the test.
 @pytest.mark.parametrize("dtype", ["float"])
@@ -68,7 +51,6 @@ def test_bert_models(
 
 @pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
 @pytest.mark.parametrize("dtype", ["float"])
-@pytest.mark.flaky(reruns=3)
 @torch.inference_mode
 def test_modernbert_models(
     hf_runner,
@@ -77,15 +59,6 @@ def test_modernbert_models(
     model: str,
     dtype: str,
 ) -> None:
-    # NOTE: https://github.com/vllm-project/vllm/pull/32403
-    # `disham993/electrical-ner-ModernBERT-base` is a randomly initialized
-    # model, which can cause numerical precision variance and edge cases.
-    # We use @flaky(reruns=3) to mitigate intermittent failures.
-    print(
-        f"\n[NOTE] Testing {model} (randomly initialized weights) - "
-        "flaky tolerance enabled due to numerical precision variance."
-    )
-
     with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
         vllm_outputs = vllm_model.token_classify(example_prompts)
 

vllm/entrypoints/openai/chat_completion/serving.py

@@ -68,7 +68,6 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
 from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
 from vllm.entrypoints.utils import get_max_tokens, should_include_usage
 from vllm.inputs.data import TokensPrompt
-from vllm.inputs.parse import get_prompt_components
 from vllm.logger import init_logger
 from vllm.logprobs import Logprob
 from vllm.outputs import CompletionOutput, RequestOutput
@@ -375,18 +374,20 @@ class OpenAIServingChat(OpenAIServing):
         generators: list[AsyncGenerator[RequestOutput, None]] = []
         try:
             for i, engine_prompt in enumerate(engine_prompts):
-                prompt_text, _, _ = get_prompt_components(engine_prompt)
-
+                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
                 # If we are creating sub requests for multiple prompts, ensure that they
                 # have unique request ids.
                 sub_request_id = (
                     request_id if len(engine_prompts) == 1 else f"{request_id}_{i}"
                 )
 
+                if self.default_sampling_params is None:
+                    self.default_sampling_params = {}
+
                 max_tokens = get_max_tokens(
                     max_model_len=self.max_model_len,
                     request=request,
-                    prompt=engine_prompt,
+                    input_length=len(engine_prompt["prompt_token_ids"]),
                     default_sampling_params=self.default_sampling_params,
                 )
 

vllm/entrypoints/openai/completion/serving.py

@@ -36,7 +36,6 @@ from vllm.entrypoints.renderer import RenderConfig
 from vllm.entrypoints.utils import get_max_tokens, should_include_usage
 from vllm.exceptions import VLLMValidationError
 from vllm.inputs.data import EmbedsPrompt, TokensPrompt, is_embeds_prompt
-from vllm.inputs.parse import get_prompt_components
 from vllm.logger import init_logger
 from vllm.logprobs import Logprob
 from vllm.outputs import RequestOutput
@@ -163,12 +162,25 @@ class OpenAIServingCompletion(OpenAIServing):
         generators: list[AsyncGenerator[RequestOutput, None]] = []
         try:
             for i, engine_prompt in enumerate(engine_prompts):
-                prompt_text, _, _ = get_prompt_components(engine_prompt)
+                prompt_text, prompt_token_ids, prompt_embeds = (
+                    self._get_prompt_components(engine_prompt)
+                )
+
+                input_length = None
+                if prompt_token_ids is not None:
+                    input_length = len(prompt_token_ids)
+                elif prompt_embeds is not None:
+                    input_length = len(prompt_embeds)
+                else:
+                    raise NotImplementedError
+
+                if self.default_sampling_params is None:
+                    self.default_sampling_params = {}
 
                 max_tokens = get_max_tokens(
                     max_model_len=self.max_model_len,
                     request=request,
-                    prompt=engine_prompt,
+                    input_length=input_length,
                     default_sampling_params=self.default_sampling_params,
                 )
 

vllm/entrypoints/openai/engine/serving.py

@@ -94,14 +94,11 @@ from vllm.entrypoints.serve.tokenize.protocol import (
     TokenizeCompletionRequest,
     TokenizeResponse,
 )
-from vllm.entrypoints.utils import (
-    _validate_truncation_size,
-    get_max_tokens,
-    sanitize_message,
-)
+from vllm.entrypoints.utils import _validate_truncation_size, sanitize_message
 from vllm.exceptions import VLLMValidationError
 from vllm.inputs.data import PromptType, TokensPrompt
 from vllm.inputs.parse import (
+    PromptComponents,
     get_prompt_components,
     is_explicit_encoder_decoder_prompt,
 )
@@ -1290,7 +1287,7 @@ class OpenAIServing:
         priority: int = 0,
         **kwargs,
     ):
-        prompt_text, _, _ = get_prompt_components(engine_prompt)
+        prompt_text, _, _ = self._get_prompt_components(engine_prompt)
 
         orig_priority = priority
         sub_request = 0
@@ -1341,12 +1338,10 @@ class OpenAIServing:
             # yield context
 
             # Create inputs for the next turn.
-            # Render the next prompt token ids and update sampling_params.
+            # Render the next prompt token ids.
             if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
-                token_ids = context.render_for_completion()
-                engine_prompt = TokensPrompt(prompt_token_ids=token_ids)
-
-                sampling_params.max_tokens = self.max_model_len - len(token_ids)
+                prompt_token_ids = context.render_for_completion()
+                engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids)
             elif isinstance(context, ParsableContext):
                 engine_prompts = await self._render_next_turn(
                     context.request,
@@ -1358,19 +1353,19 @@ class OpenAIServing:
                     context.chat_template_content_format,
                 )
                 engine_prompt = engine_prompts[0]
-                prompt_text, _, _ = get_prompt_components(engine_prompt)
-
-                sampling_params.max_tokens = get_max_tokens(
-                    self.max_model_len,
-                    context.request,
-                    engine_prompt,
-                    self.default_sampling_params,  # type: ignore
-                )
+                prompt_text, _, _ = self._get_prompt_components(engine_prompt)
 
+            # Update the sampling params.
+            sampling_params.max_tokens = self.max_model_len - len(
+                engine_prompt["prompt_token_ids"]
+            )
             # OPTIMIZATION
             priority = orig_priority - 1
             sub_request += 1
 
+    def _get_prompt_components(self, prompt: PromptType) -> PromptComponents:
+        return get_prompt_components(prompt)
+
     def _log_inputs(
         self,
         request_id: str,
@@ -1381,7 +1376,7 @@ class OpenAIServing:
         if self.request_logger is None:
             return
 
-        prompt, prompt_token_ids, prompt_embeds = get_prompt_components(inputs)
+        prompt, prompt_token_ids, prompt_embeds = self._get_prompt_components(inputs)
 
         self.request_logger.log_inputs(
             request_id,

vllm/entrypoints/openai/responses/serving.py

@@ -116,7 +116,6 @@ from vllm.entrypoints.openai.responses.utils import (
     extract_tool_types,
     should_continue_final_message,
 )
-from vllm.entrypoints.utils import get_max_tokens
 from vllm.exceptions import VLLMValidationError
 from vllm.inputs.data import TokensPrompt
 from vllm.logger import init_logger
@@ -436,11 +435,8 @@ class OpenAIServingResponses(OpenAIServing):
                 if maybe_error is not None:
                     return maybe_error
 
-                default_max_tokens = get_max_tokens(
-                    self.max_model_len,
-                    request,
-                    engine_prompt,
-                    self.default_sampling_params,
+                default_max_tokens = self.max_model_len - len(
+                    engine_prompt["prompt_token_ids"]
                 )
 
                 sampling_params = request.to_sampling_params(

vllm/entrypoints/utils.py

@@ -17,10 +17,8 @@ from starlette.background import BackgroundTask, BackgroundTasks
 
 from vllm import envs
 from vllm.engine.arg_utils import EngineArgs
-from vllm.inputs import EmbedsPrompt, TokensPrompt
 from vllm.logger import current_formatter_type, init_logger
 from vllm.platforms import current_platform
-from vllm.utils import length_from_prompt_token_ids_or_embeds
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 if TYPE_CHECKING:
@@ -34,15 +32,11 @@ if TYPE_CHECKING:
         StreamOptions,
     )
     from vllm.entrypoints.openai.models.protocol import LoRAModulePath
-    from vllm.entrypoints.openai.responses.protocol import (
-        ResponsesRequest,
-    )
 else:
     ChatCompletionRequest = object
     CompletionRequest = object
     StreamOptions = object
     LoRAModulePath = object
-    ResponsesRequest = object
 
 
 logger = init_logger(__name__)
@@ -217,26 +211,11 @@ def _validate_truncation_size(
 
 def get_max_tokens(
     max_model_len: int,
-    request: "CompletionRequest | ChatCompletionRequest | ResponsesRequest",
-    prompt: TokensPrompt | EmbedsPrompt,
+    request: "ChatCompletionRequest | CompletionRequest",
+    input_length: int,
     default_sampling_params: dict,
 ) -> int:
-    # NOTE: Avoid isinstance() for better efficiency
-    max_tokens: int | None = None
-    if max_tokens is None:
-        # ChatCompletionRequest
-        max_tokens = getattr(request, "max_completion_tokens", None)
-    if max_tokens is None:
-        # ResponsesRequest
-        max_tokens = getattr(request, "max_output_tokens", None)
-    if max_tokens is None:
-        # CompletionRequest (also a fallback for ChatCompletionRequest)
-        max_tokens = getattr(request, "max_tokens", None)
-
-    input_length = length_from_prompt_token_ids_or_embeds(
-        prompt.get("prompt_token_ids"),  # type: ignore[arg-type]
-        prompt.get("prompt_embeds"),  # type: ignore[arg-type]
-    )
+    max_tokens = getattr(request, "max_completion_tokens", None) or request.max_tokens
     default_max_tokens = max_model_len - input_length
     max_output_tokens = current_platform.get_max_output_tokens(input_length)
 

vllm/v1/structured_output/__init__.py

@@ -74,6 +74,9 @@ class StructuredOutputManager:
             self.tokenizer = cached_tokenizer_from_config(
                 model_config=self.vllm_config.model_config
             )
+            reasoning_parser = (
+                self.vllm_config.structured_outputs_config.reasoning_parser
+            )
             reasoning_parser_plugin = (
                 self.vllm_config.structured_outputs_config.reasoning_parser_plugin
             )