[Model] Add Inkling model support [1/N] (#48799)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>

.gitignore

@@ -18,6 +18,9 @@ vllm/third_party/deep_gemm/
 # fmha_sm100 vendored package built from source
 vllm/third_party/fmha_sm100/
 
+# tml-fa4 vendored package built from source
+vllm/third_party/tml_fa4/
+
 # triton jit
 .triton
 

CMakeLists.txt

@@ -1408,6 +1408,7 @@ if (VLLM_GPU_LANG STREQUAL "CUDA")
     include(cmake/external_projects/fmha_sm100.cmake)
     include(cmake/external_projects/flashmla.cmake)
     include(cmake/external_projects/qutlass.cmake)
+    include(cmake/external_projects/tml_fa4.cmake)
 
     # vllm-flash-attn should be last as it overwrites some CMake functions
     include(cmake/external_projects/vllm_flash_attn.cmake)

cmake/external_projects/tml_fa4.cmake

@@ -0,0 +1,50 @@
+include(FetchContent)
+
+if(DEFINED ENV{TML_FA4_SRC_DIR})
+  set(TML_FA4_SRC_DIR $ENV{TML_FA4_SRC_DIR})
+endif()
+
+if(TML_FA4_SRC_DIR)
+  FetchContent_Declare(
+    tml_fa4
+    SOURCE_DIR ${TML_FA4_SRC_DIR}
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND "")
+else()
+  FetchContent_Declare(
+    tml_fa4
+    GIT_REPOSITORY https://github.com/vllm-project/tml-fa4.git
+    GIT_TAG 13374f0c855acc1add1bf30444bd67aebbc24a8e
+    GIT_PROGRESS TRUE
+    CONFIGURE_COMMAND ""
+    BUILD_COMMAND "")
+endif()
+
+FetchContent_GetProperties(tml_fa4)
+if(NOT tml_fa4_POPULATED)
+  FetchContent_Populate(tml_fa4)
+endif()
+message(STATUS "tml-fa4 is available at ${tml_fa4_SOURCE_DIR}")
+
+add_custom_target(tml_fa4)
+
+# Install into a private namespace so this implementation cannot shadow the
+# flash_attn package used by vLLM's standard attention backends.
+install(CODE "
+  file(GLOB_RECURSE TML_FA4_PY_FILES
+    \"${tml_fa4_SOURCE_DIR}/flash_attn/cute/*.py\")
+  foreach(SRC_FILE \${TML_FA4_PY_FILES})
+    file(RELATIVE_PATH REL_PATH
+      \"${tml_fa4_SOURCE_DIR}/flash_attn/cute\" \${SRC_FILE})
+    set(DST_FILE
+      \"\${CMAKE_INSTALL_PREFIX}/vllm/third_party/tml_fa4/\${REL_PATH}\")
+    get_filename_component(DST_DIR \${DST_FILE} DIRECTORY)
+    file(MAKE_DIRECTORY \${DST_DIR})
+    file(READ \${SRC_FILE} FILE_CONTENTS)
+    string(REPLACE
+      \"flash_attn.cute\"
+      \"vllm.third_party.tml_fa4\"
+      FILE_CONTENTS \"\${FILE_CONTENTS}\")
+    file(WRITE \${DST_FILE} \"\${FILE_CONTENTS}\")
+  endforeach()
+" COMPONENT tml_fa4)

requirements/cuda.txt

@@ -26,7 +26,7 @@ fastsafetensors >= 0.3.2
 
 # QuACK and Cutlass DSL for FA4 (cute-DSL implementation)
 nvidia-cutlass-dsl[cu13]==4.5.2
-quack-kernels>=0.3.3
+quack-kernels>=0.4.0  # Required for tml-fa4
 
 # Tokenspeed_MLA for faster mla with spec decode
 tokenspeed-mla==0.1.8; platform_system == "Linux"

rust/Cargo.lock

@@ -2220,7 +2220,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
 [[package]]
 name = "llm-multimodal"
 version = "1.7.1"
-source = "git+https://github.com/smg-project/llm-multimodal?rev=7df38e53f99aefaebe86934f010aa8084ec99b2f#7df38e53f99aefaebe86934f010aa8084ec99b2f"
+source = "git+https://github.com/smg-project/llm-multimodal?rev=5390032d6dc8a3e6fdc83acd320260367eb4b9b5#5390032d6dc8a3e6fdc83acd320260367eb4b9b5"
 dependencies = [
  "anyhow",
  "base64 0.22.1",
@@ -2235,6 +2235,7 @@ dependencies = [
  "once_cell",
  "pkg-config",
  "rayon",
+ "realfft",
  "reqwest 0.13.4",
  "rustfft",
  "serde",

rust/Cargo.toml

@@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [
 indexmap = "2.13.0"
 itertools = "0.14.0"
 libc = "0.2.177"
-llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7df38e53f99aefaebe86934f010aa8084ec99b2f", default-features = false, features = ["native-tls"] }
+llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "5390032d6dc8a3e6fdc83acd320260367eb4b9b5", default-features = false, features = ["native-tls"] }
 mimalloc = "0.1.52"
 minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
 minijinja-contrib = { version = "2.0", features = ["pycompat"] }

rust/src/chat/src/backend/hf.rs

@@ -20,6 +20,7 @@ use crate::output::{
 use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
 use crate::renderer::{
     DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
+    InklingChatRenderer,
 };
 use crate::request::ChatRequest;
 use crate::{DynChatOutputProcessor, RendererSelection};
@@ -71,6 +72,7 @@ impl HfChatBackend {
             RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
             RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
             RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
+            RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
         };
 
         info!(

rust/src/chat/src/lib.rs

@@ -33,7 +33,7 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
 pub use renderer::hf::ChatTemplateContentFormatOption;
 pub use renderer::{
     ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
-    HarmonyChatRenderer, RenderedPrompt, RendererSelection,
+    HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
 };
 pub use request::{
     ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
@@ -294,7 +294,7 @@ mod tests {
         )
         .unwrap_err();
 
-        expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
+        expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
     }
 
     #[test]
@@ -305,6 +305,6 @@ mod tests {
         )
         .unwrap_err();
 
-        expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
+        expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
     }
 }

rust/src/chat/src/multimodal/audio.rs

@@ -79,6 +79,38 @@ mod tests {
     use super::*;
 
     const AUDIO_PAD_ID: u32 = 151_676;
+    const INKLING_AUDIO_MARKER_ID: u32 = 200_020;
+    const INKLING_AUDIO_EMBED_ID: i32 = 200_053;
+
+    fn inkling_info(decoder_dmodel: serde_json::Value) -> MultimodalModelInfo {
+        let config = serde_json::json!({
+            "model_type": "inkling_mm_model",
+            "audio_config": {
+                "decoder_dmodel": decoder_dmodel,
+                "n_mel_bins": 80,
+                "mel_vocab_size": 16,
+                "dmel_min_value": -7.0,
+                "dmel_max_value": 2.0
+            }
+        });
+        let tokenizer = TestTokenizer::new()
+            .with_regular_token("<|content_image|>", 200_005)
+            .with_regular_token("<|content_audio_input|>", INKLING_AUDIO_MARKER_ID);
+        let context = MultimodalModelContext {
+            model_id: "inkling-test".to_string(),
+            model_type: Some("inkling_mm_model".to_string()),
+            config,
+            tokenizer: TokenizerResolver(Arc::new(tokenizer)),
+        };
+
+        MultimodalModelInfo::from_loaded(
+            context,
+            PreProcessorConfig::default(),
+            PreProcessorConfig::default(),
+        )
+        .unwrap()
+        .expect("Inkling multimodal support")
+    }
 
     fn qwen3_asr_info() -> MultimodalModelInfo {
         let context = MultimodalModelContext {
@@ -120,6 +152,40 @@ mod tests {
         bytes
     }
 
+    #[test]
+    fn resolves_inkling_audio_from_model_spec() {
+        let info = inkling_info(serde_json::json!(1024));
+        let support = info.audio.as_ref().expect("audio support");
+
+        assert_eq!(
+            info.placeholder_token(Modality::Audio),
+            Some("<|content_audio_input|>")
+        );
+        assert_eq!(support.placeholder.marker_token_id, INKLING_AUDIO_MARKER_ID);
+        assert_eq!(
+            support.placeholder.embed_token_id,
+            INKLING_AUDIO_EMBED_ID as u32
+        );
+        assert_eq!(support.spec.primary_key(), AUDIO_PRIMARY_KEY);
+        assert!(matches!(
+            &support.spec.field_layouts.encoder_input,
+            llm_multimodal::FieldLayout::Flat { sizes_key }
+                if sizes_key == "num_audio_tokens"
+        ));
+        assert!(matches!(
+            support.spec.field_layouts.model_specific.get("num_audio_tokens"),
+            Some(llm_multimodal::FieldLayout::Batched)
+        ));
+    }
+
+    #[test]
+    fn inkling_decoder_config_gates_audio_capability() {
+        let info = inkling_info(serde_json::Value::Null);
+
+        assert!(info.audio.is_none());
+        assert_eq!(info.placeholder_token(Modality::Audio), None);
+    }
+
     #[test]
     fn resolves_qwen_audio_from_model_spec() {
         let info = qwen3_asr_info();
@@ -210,4 +276,50 @@ mod tests {
                 if tensor.dtype == "int64" && tensor.shape.is_empty()
         ));
     }
+
+    #[tokio::test]
+    async fn inkling_tracker_and_processor_use_standard_audio_key() {
+        let info = inkling_info(serde_json::json!(1024));
+        let wav = wav_i16_mono(16_000, &[0; 1_600]);
+        let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
+        let fetched = info
+            .fetch_media(vec![MediaContentPart::AudioData {
+                data: wav,
+                mime_type: Some("audio/wav".to_string()),
+                uuid: Some("audio-1".to_string()),
+            }])
+            .await
+            .unwrap();
+
+        let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
+
+        assert_eq!(prepared.replacements.len(), 1);
+        assert_eq!(
+            prepared.replacements[0].tokens[0],
+            INKLING_AUDIO_MARKER_ID as i32
+        );
+        assert!(
+            prepared.replacements[0].tokens[1..]
+                .iter()
+                .all(|token| *token == INKLING_AUDIO_EMBED_ID)
+        );
+        let item = &prepared.items[0];
+        assert_eq!(item.hash, expected_hash);
+        assert_eq!(item.uuid.as_deref(), Some("audio-1"));
+
+        let features = &item.data[AUDIO_PRIMARY_KEY];
+        assert!(matches!(&features.field, MmField::Flat(_)));
+        assert!(matches!(
+            features.data.as_ref(),
+            Some(MmKwargValue::Tensor(tensor))
+                if tensor.dtype == "float32" && tensor.shape.get(1) == Some(&80)
+        ));
+        let count = &item.data["num_audio_tokens"];
+        assert!(matches!(&count.field, MmField::Batched(_)));
+        assert!(matches!(
+            count.data.as_ref(),
+            Some(MmKwargValue::Tensor(tensor))
+                if tensor.dtype == "int64" && tensor.shape.is_empty()
+        ));
+    }
 }

rust/src/chat/src/parser/reasoning/mod.rs

@@ -23,6 +23,7 @@ pub mod names {
     pub const DEEPSEEK_V3: &str = "deepseek_v3";
     pub const DEEPSEEK_V4: &str = "deepseek_v4";
     pub const GEMMA4: &str = "gemma4";
+    pub const INKLING: &str = "inkling";
     pub const GLM45: &str = "glm45";
     pub const KIMI: &str = "kimi";
     pub const KIMI_K2: &str = "kimi_k2";
@@ -63,6 +64,7 @@ impl ReasoningParserFactory {
             .register_parser::<DeepSeekV3ReasoningParser>(names::DEEPSEEK_V3)
             .register_parser::<DeepSeekV4ReasoningParser>(names::DEEPSEEK_V4)
             .register_unified_dummy(names::GEMMA4)
+            .register_unified_dummy(names::INKLING)
             .register_parser::<Glm45ReasoningParser>(names::GLM45)
             .register_parser::<KimiReasoningParser>(names::KIMI)
             .register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)

rust/src/chat/src/parser/tool/mod.rs

@@ -25,6 +25,7 @@ pub mod names {
     pub const GLM45: &str = "glm45";
     pub const GLM47: &str = "glm47";
     pub const GEMMA4: &str = "gemma4";
+    pub const INKLING: &str = "inkling";
     pub const GRANITE4: &str = "granite4";
     pub const HERMES: &str = "hermes";
     pub const HY_V3: &str = "hy_v3";
@@ -70,6 +71,7 @@ impl ToolParserFactory {
             .register_parser::<Glm45MoeToolParser>(names::GLM45)
             .register_parser::<Glm47MoeToolParser>(names::GLM47)
             .register_unified_dummy(names::GEMMA4)
+            .register_unified_dummy(names::INKLING)
             .register_parser::<Granite4ToolParser>(names::GRANITE4)
             .register_parser::<HermesToolParser>(names::HERMES)
             .register_parser::<HyV3ToolParser>(names::HY_V3)

rust/src/chat/src/parser/unified.rs

@@ -5,7 +5,7 @@
 
 use std::sync::LazyLock;
 
-pub use vllm_parser::unified::{Gemma4UnifiedParser, UnifiedParser};
+pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
 use vllm_tokenizer::DynTokenizer;
 
 use crate::parser::ParserFactory;
@@ -14,6 +14,7 @@ use crate::request::ChatTool;
 /// Canonical public names for registered unified parsers.
 pub mod names {
     pub const GEMMA4: &str = "gemma4";
+    pub const INKLING: &str = "inkling";
 }
 
 /// Constructor signature for one registered unified parser implementation.
@@ -37,10 +38,12 @@ impl UnifiedParserFactory {
         let mut factory = Self::default();
 
         factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
+        factory.register_parser::<InklingUnifiedParser>(names::INKLING);
 
         factory
             .register_pattern("gemma-4", names::GEMMA4)
-            .register_pattern("gemma4", names::GEMMA4);
+            .register_pattern("gemma4", names::GEMMA4)
+            .register_pattern("inkling", names::INKLING);
 
         factory
     }
@@ -88,6 +91,13 @@ mod tests {
             .with_regular_token("<channel|>", 257)
     }
 
+    fn inkling_tokenizer() -> TestTokenizer {
+        TestTokenizer::new()
+            .with_regular_token("<|message_model|>", 200001)
+            .with_regular_token("<|content_text|>", 200004)
+            .with_regular_token("<|content_thinking|>", 200008)
+    }
+
     #[test]
     fn factory_registers_gemma4() {
         let factory = UnifiedParserFactory::new();
@@ -99,4 +109,16 @@ mod tests {
         );
         factory.create(names::GEMMA4, &[], Arc::new(tokenizer())).unwrap();
     }
+
+    #[test]
+    fn factory_registers_inkling() {
+        let factory = UnifiedParserFactory::new();
+
+        assert!(factory.contains(names::INKLING));
+        assert_eq!(
+            factory.resolve_name_for_model("thinkingmachines/Inkling"),
+            Some(names::INKLING)
+        );
+        factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
+    }
 }

rust/src/chat/src/renderer/inkling/fixtures/text_audio_input.json

@@ -0,0 +1,21 @@
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": "transcribe"
+        },
+        {
+          "type": "input_audio",
+          "data": ""
+        },
+        {
+          "type": "audio_url",
+          "audio_url": "data:audio/wav;base64,"
+        }
+      ]
+    }
+  ]
+}

rust/src/chat/src/renderer/inkling/fixtures/text_audio_output.txt

@@ -0,0 +1 @@
+<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>transcribe<|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_user|><|content_audio_input|><|audio_end|><|end_message|><|message_model|>

rust/src/chat/src/renderer/inkling/fixtures/text_image_input.json

@@ -0,0 +1,17 @@
+{
+  "messages": [
+    {
+      "role": "user",
+      "content": [
+        {
+          "type": "text",
+          "text": "look"
+        },
+        {
+          "type": "image_url",
+          "image_url": "data:image/png;base64,"
+        }
+      ]
+    }
+  ]
+}

rust/src/chat/src/renderer/inkling/fixtures/text_image_output.txt

@@ -0,0 +1 @@
+<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>look<|end_message|><|message_user|><|content_image|><|end_message|><|message_model|>

rust/src/chat/src/renderer/inkling/fixtures/tool_declare_input.json

@@ -0,0 +1,46 @@
+{
+  "tools": [
+    {
+      "type": "function",
+      "function": {
+        "name": "get_weather",
+        "description": "Get weather information",
+        "parameters": {
+          "type": "object",
+          "required": [
+            "city"
+          ],
+          "properties": {
+            "city": {
+              "type": "string"
+            }
+          }
+        }
+      }
+    }
+  ],
+  "messages": [
+    {
+      "role": "developer",
+      "content": "rules",
+      "tools": [
+        {
+          "type": "function",
+          "function": {
+            "name": "local_tool",
+            "parameters": {
+              "z": 1,
+              "a": {
+                "b": 2
+              }
+            }
+          }
+        }
+      ]
+    },
+    {
+      "role": "user",
+      "content": "hi"
+    }
+  ]
+}

rust/src/chat/src/renderer/inkling/fixtures/tool_declare_output.txt

@@ -0,0 +1 @@
+<|message_system|>tool_declare<|content_xml|>[{"description":"Get weather information","name":"get_weather","parameters":{"properties":{"city":{"type":"string"}},"required":["city"],"type":"object"},"type":"function"},{"description":"","name":"local_tool","parameters":{"a":{"b":2},"z":1},"type":"function"}]<|end_message|><|message_system|><|content_text|>rules<|end_message|><|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_user|><|content_text|>hi<|end_message|><|message_model|>

rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_input.json

@@ -0,0 +1,24 @@
+{
+  "add_generation_prompt": false,
+  "messages": [
+    {
+      "role": "assistant",
+      "reasoning_content": "think",
+      "content": "answer",
+      "tool_calls": [
+        {
+          "id": "call_1",
+          "function": {
+            "name": "get_weather",
+            "arguments": "{\"city\":\"SF\"}"
+          }
+        }
+      ]
+    },
+    {
+      "role": "tool",
+      "tool_call_id": "call_1",
+      "content": "sunny"
+    }
+  ]
+}

rust/src/chat/src/renderer/inkling/fixtures/tool_round_trip_output.txt

@@ -0,0 +1 @@
+<|message_system|><|content_text|>Thinking effort level: 0.9<|end_message|><|message_model|><|content_thinking|>think<|end_message|><|message_model|><|content_text|>answer<|end_message|><|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"SF"}}<|end_message|><|content_model_end_sampling|><|message_tool|>get_weather<|content_text|>sunny<|end_message|>

rust/src/chat/src/renderer/inkling/mod.rs

@@ -0,0 +1,442 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+use std::collections::HashMap;
+
+use serde_json::{Map, Value, json};
+use thiserror_ext::AsReport as _;
+use vllm_text::Prompt;
+use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
+
+use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
+use crate::error::{Error, Result};
+use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
+use crate::{AssistantContentBlock, AssistantToolCall};
+
+const MESSAGE_USER: &str = "<|message_user|>";
+const MESSAGE_MODEL: &str = "<|message_model|>";
+const MESSAGE_SYSTEM: &str = "<|message_system|>";
+const MESSAGE_TOOL: &str = "<|message_tool|>";
+const CONTENT_TEXT: &str = "<|content_text|>";
+const CONTENT_IMAGE: &str = "<|content_image|>";
+const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
+const CONTENT_AUDIO_INPUT: &str = "<|content_audio_input|>";
+const CONTENT_THINKING: &str = "<|content_thinking|>";
+// Inkling renderer semantics use this slot for structured payloads such as tool
+// declarations.
+const CONTENT_XML: &str = "<|content_xml|>";
+const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
+const END_MESSAGE: &str = "<|end_message|>";
+const AUDIO_END: &str = "<|audio_end|>";
+const MAX_REASONING_EFFORT: f64 = 0.99;
+
+/// Native Inkling renderer that emits token IDs directly.
+#[derive(Clone)]
+pub struct InklingChatRenderer {
+    tokenizer: DynTokenizer,
+    special: InklingSpecialTokenIds,
+}
+
+#[derive(Debug, Clone, Copy)]
+struct InklingSpecialTokenIds {
+    message_user: u32,
+    message_model: u32,
+    message_system: u32,
+    message_tool: u32,
+    content_text: u32,
+    content_image: u32,
+    content_model_end_sampling: u32,
+    content_audio_input: u32,
+    content_thinking: u32,
+    content_xml: u32,
+    content_invoke_tool_json: u32,
+    end_message: u32,
+    audio_end: u32,
+}
+
+impl InklingSpecialTokenIds {
+    fn resolve(tokenizer: &dyn Tokenizer) -> Result<Self> {
+        Ok(Self {
+            message_user: resolve_special_token(tokenizer, MESSAGE_USER)?,
+            message_model: resolve_special_token(tokenizer, MESSAGE_MODEL)?,
+            message_system: resolve_special_token(tokenizer, MESSAGE_SYSTEM)?,
+            message_tool: resolve_special_token(tokenizer, MESSAGE_TOOL)?,
+            content_text: resolve_special_token(tokenizer, CONTENT_TEXT)?,
+            content_image: resolve_special_token(tokenizer, CONTENT_IMAGE)?,
+            content_model_end_sampling: resolve_special_token(
+                tokenizer,
+                CONTENT_MODEL_END_SAMPLING,
+            )?,
+            content_audio_input: resolve_special_token(tokenizer, CONTENT_AUDIO_INPUT)?,
+            content_thinking: resolve_special_token(tokenizer, CONTENT_THINKING)?,
+            content_xml: resolve_special_token(tokenizer, CONTENT_XML)?,
+            content_invoke_tool_json: resolve_special_token(tokenizer, CONTENT_INVOKE_TOOL_JSON)?,
+            end_message: resolve_special_token(tokenizer, END_MESSAGE)?,
+            audio_end: resolve_special_token(tokenizer, AUDIO_END)?,
+        })
+    }
+}
+
+impl InklingChatRenderer {
+    pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
+        let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?;
+        Ok(Self { tokenizer, special })
+    }
+
+    fn write_text_tokens(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
+        out.extend(self.tokenizer.encode(text, false)?);
+        Ok(())
+    }
+
+    fn write_message_start(
+        &self,
+        out: &mut Vec<u32>,
+        role_token_id: u32,
+        author_name: Option<&str>,
+    ) -> Result<()> {
+        out.push(role_token_id);
+        if let Some(author_name) = author_name
+            && !author_name.is_empty()
+        {
+            self.write_text_tokens(out, author_name)?;
+        }
+        Ok(())
+    }
+
+    fn write_text_block(
+        &self,
+        out: &mut Vec<u32>,
+        role_token_id: u32,
+        author_name: Option<&str>,
+        text: &str,
+    ) -> Result<()> {
+        self.write_message_start(out, role_token_id, author_name)?;
+        out.push(self.special.content_text);
+        self.write_text_tokens(out, text)?;
+        out.push(self.special.end_message);
+        Ok(())
+    }
+
+    /// Write one image block holding only the `<|content_image|>` marker.
+    /// Multimodal preprocessing later expands the marker into per-patch
+    /// image placeholder tokens once the patch count is known, mirroring the
+    /// Python `InklingMultiModalProcessor` marker-anchored prompt updates.
+    fn write_image_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
+        out.push(role_token_id);
+        out.push(self.special.content_image);
+        out.push(self.special.end_message);
+    }
+
+    /// Write one audio block holding the `<|content_audio_input|>` marker
+    /// followed by the `<|audio_end|>` terminator. Multimodal preprocessing
+    /// later expands the marker into per-frame audio placeholder tokens
+    /// (landing before `<|audio_end|>`) once the clip length is known.
+    fn write_audio_block(&self, out: &mut Vec<u32>, role_token_id: u32) {
+        out.push(role_token_id);
+        out.push(self.special.content_audio_input);
+        out.push(self.special.audio_end);
+        out.push(self.special.end_message);
+    }
+
+    fn write_reasoning_block(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
+        if text.is_empty() {
+            return Ok(());
+        }
+        out.push(self.special.message_model);
+        out.push(self.special.content_thinking);
+        self.write_text_tokens(out, text)?;
+        out.push(self.special.end_message);
+        Ok(())
+    }
+
+    fn write_reasoning_effort(&self, out: &mut Vec<u32>, effort: f64) -> Result<()> {
+        if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
+            return Err(Error::ChatTemplate(format!(
+                "Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
+            )));
+        }
+        let formatted = format!("{effort:.2}");
+        let effort = formatted.trim_end_matches('0').trim_end_matches('.');
+        let effort = if matches!(effort, "0" | "-0") {
+            "0.0"
+        } else {
+            effort
+        };
+        self.write_text_block(
+            out,
+            self.special.message_system,
+            None,
+            &format!("Thinking effort level: {effort}"),
+        )
+    }
+
+    fn write_tool_declarations(&self, out: &mut Vec<u32>, tools: &[&ChatTool]) -> Result<()> {
+        if tools.is_empty() {
+            return Ok(());
+        }
+
+        let mut specs = Vec::with_capacity(tools.len());
+        for tool in tools {
+            specs.push(json!({
+                "description": tool.description.as_deref().unwrap_or(""),
+                "name": tool.name,
+                "parameters": sort_json(&tool.parameters),
+                "type": "function",
+            }));
+        }
+        let payload = compact_json(&sort_json(&Value::Array(specs)))?;
+
+        self.write_message_start(out, self.special.message_system, Some("tool_declare"))?;
+        out.push(self.special.content_xml);
+        self.write_text_tokens(out, &payload)?;
+        out.push(self.special.end_message);
+        Ok(())
+    }
+
+    fn write_chat_content(
+        &self,
+        out: &mut Vec<u32>,
+        role_token_id: u32,
+        content: &ChatContent,
+    ) -> Result<()> {
+        match content {
+            ChatContent::Text(text) => {
+                if !text.is_empty() {
+                    self.write_text_block(out, role_token_id, None, text)?;
+                }
+            }
+            ChatContent::Parts(parts) => {
+                for part in parts {
+                    match part {
+                        ChatContentPart::Text { text } => {
+                            self.write_text_block(out, role_token_id, None, text)?;
+                        }
+                        ChatContentPart::ImageUrl { .. } => {
+                            self.write_image_block(out, role_token_id);
+                        }
+                        ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
+                            self.write_audio_block(out, role_token_id);
+                        }
+                        // Inkling has no video modality.
+                        ChatContentPart::VideoUrl { .. } => {
+                            return Err(Error::UnsupportedMultimodalContent("video_url"));
+                        }
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    fn write_assistant_tool_call(
+        &self,
+        out: &mut Vec<u32>,
+        tool_call: &AssistantToolCall,
+    ) -> Result<()> {
+        let payload = tool_call_json(tool_call)?;
+
+        self.write_message_start(out, self.special.message_model, Some(&tool_call.name))?;
+        out.push(self.special.content_invoke_tool_json);
+        self.write_text_tokens(out, &payload)?;
+        out.push(self.special.end_message);
+        Ok(())
+    }
+
+    fn write_assistant_content(
+        &self,
+        out: &mut Vec<u32>,
+        content: &[AssistantContentBlock],
+        tool_call_id_to_name: &mut HashMap<String, String>,
+    ) -> Result<()> {
+        for block in content {
+            match block {
+                AssistantContentBlock::Reasoning { text } => {
+                    self.write_reasoning_block(out, text)?;
+                }
+                AssistantContentBlock::Text { text } => {
+                    if !text.is_empty() {
+                        self.write_text_block(out, self.special.message_model, None, text)?;
+                    }
+                }
+                AssistantContentBlock::ToolCall(tool_call) => {
+                    if !tool_call.id.is_empty() {
+                        tool_call_id_to_name.insert(tool_call.id.clone(), tool_call.name.clone());
+                    }
+                    self.write_assistant_tool_call(out, tool_call)?;
+                }
+            }
+        }
+        out.push(self.special.content_model_end_sampling);
+        Ok(())
+    }
+
+    fn write_tool_response(
+        &self,
+        out: &mut Vec<u32>,
+        content: &ChatContent,
+        tool_call_id: &str,
+        tool_call_id_to_name: &HashMap<String, String>,
+    ) -> Result<()> {
+        let text = content.try_flatten_to_text()?;
+        let tool_name = tool_call_id_to_name.get(tool_call_id).map(String::as_str).unwrap_or("");
+        self.write_text_block(out, self.special.message_tool, Some(tool_name), &text)
+    }
+}
+
+impl ChatRenderer for InklingChatRenderer {
+    fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
+        request.validate()?;
+        if request.chat_options.continue_final_message() {
+            return Err(Error::ChatTemplate(
+                "Inkling renderer does not support continue_final_message".to_string(),
+            ));
+        }
+
+        let mut out = Vec::new();
+        let mut tool_call_id_to_name = HashMap::new();
+        let effective_template_kwargs = request_template_kwargs(request);
+        let tools = rendered_tools(request);
+        self.write_tool_declarations(&mut out, &tools)?;
+        let mut reasoning_effort =
+            resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort"));
+
+        for message in &request.messages {
+            if !matches!(
+                message,
+                ChatMessage::System { .. } | ChatMessage::Developer { .. }
+            ) && let Some(effort) = reasoning_effort.take()
+            {
+                self.write_reasoning_effort(&mut out, effort)?;
+            }
+
+            match message {
+                ChatMessage::System { content } => {
+                    self.write_chat_content(&mut out, self.special.message_system, content)?;
+                }
+                ChatMessage::Developer { content, .. } => {
+                    self.write_chat_content(&mut out, self.special.message_system, content)?;
+                }
+                ChatMessage::User { content } => {
+                    self.write_chat_content(&mut out, self.special.message_user, content)?;
+                }
+                ChatMessage::Assistant { content } => {
+                    self.write_assistant_content(&mut out, content, &mut tool_call_id_to_name)?;
+                }
+                ChatMessage::ToolResponse {
+                    content,
+                    tool_call_id,
+                } => {
+                    self.write_tool_response(
+                        &mut out,
+                        content,
+                        tool_call_id,
+                        &tool_call_id_to_name,
+                    )?;
+                }
+            }
+        }
+
+        if let Some(effort) = reasoning_effort {
+            self.write_reasoning_effort(&mut out, effort)?;
+        }
+
+        if request.chat_options.add_generation_prompt() {
+            out.push(self.special.message_model);
+        }
+
+        Ok(RenderedPrompt {
+            prompt: Prompt::TokenIds(out),
+            effective_template_kwargs,
+        })
+    }
+}
+
+fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
+    let Some(value) = value else {
+        return Some(0.9);
+    };
+    match value {
+        Value::String(name) => match name.as_str() {
+            "none" => Some(0.0),
+            "minimal" => Some(0.1),
+            "low" => Some(0.2),
+            "medium" => Some(0.7),
+            "high" => Some(0.9),
+            "xhigh" | "max" => Some(0.99),
+            _ => None,
+        },
+        Value::Number(number) => number.as_f64(),
+        _ => None,
+    }
+}
+
+fn resolve_special_token(tokenizer: &dyn Tokenizer, token: &str) -> Result<u32> {
+    tokenizer.token_to_id(token).ok_or_else(|| {
+        Error::ChatTemplate(format!(
+            "Inkling tokenizer is missing special token `{token}`"
+        ))
+    })
+}
+
+fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> {
+    if !request.tool_parsing_enabled() {
+        return Vec::new();
+    }
+
+    let mut tools = Vec::with_capacity(request.tools.len());
+    tools.extend(request.tools.iter());
+    for message in &request.messages {
+        if let ChatMessage::Developer {
+            tools: Some(local_tools),
+            ..
+        } = message
+        {
+            tools.extend(local_tools.iter());
+        }
+    }
+    tools
+}
+
+fn tool_call_json(tool_call: &AssistantToolCall) -> Result<String> {
+    let name_json = serde_json::to_string(&tool_call.name)
+        .map_err(|error| Error::ChatTemplate(error.as_report().to_string()))?;
+    let arguments = if tool_call.arguments.trim().is_empty() {
+        Value::Object(Map::new())
+    } else {
+        serde_json::from_str(&tool_call.arguments).map_err(|error| {
+            Error::ChatTemplate(format!(
+                "Inkling tool call arguments must decode to a JSON object: {error}"
+            ))
+        })?
+    };
+    let Value::Object(_) = arguments else {
+        return Err(Error::ChatTemplate(
+            "Inkling tool call arguments must decode to a JSON object".to_string(),
+        ));
+    };
+    let args_json = compact_json(&sort_json(&arguments))?;
+    Ok(format!("{{\"name\":{name_json},\"args\":{args_json}}}"))
+}
+
+fn compact_json(value: &Value) -> Result<String> {
+    serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.as_report().to_string()))
+}
+
+fn sort_json(value: &Value) -> Value {
+    match value {
+        Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
+        Value::Object(map) => {
+            let mut sorted = Map::new();
+            let mut keys = map.keys().collect::<Vec<_>>();
+            keys.sort();
+            for key in keys {
+                sorted.insert(key.clone(), sort_json(&map[key]));
+            }
+            Value::Object(sorted)
+        }
+        _ => value.clone(),
+    }
+}
+
+#[cfg(test)]
+mod tests;

rust/src/chat/src/renderer/inkling/tests.rs

@@ -0,0 +1,375 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use expect_test::{ExpectFile, expect_file};
+use serde_json::json;
+use thiserror_ext::AsReport;
+use vllm_text::tokenizer::Tokenizer;
+
+use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
+
+use super::{
+    AUDIO_END, CONTENT_AUDIO_INPUT, CONTENT_IMAGE, CONTENT_INVOKE_TOOL_JSON, CONTENT_TEXT,
+    CONTENT_THINKING, CONTENT_XML, END_MESSAGE, InklingChatRenderer, MESSAGE_MODEL, MESSAGE_SYSTEM,
+    MESSAGE_TOOL, MESSAGE_USER,
+};
+use crate::event::{AssistantContentBlock, AssistantToolCall};
+use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort};
+use crate::{ChatRenderer, Error};
+
+struct FixtureTokenizer;
+
+impl Tokenizer for FixtureTokenizer {
+    fn encode(
+        &self,
+        text: &str,
+        _add_special_tokens: bool,
+    ) -> vllm_text::tokenizer::Result<Vec<u32>> {
+        Ok(text.bytes().map(u32::from).collect())
+    }
+
+    fn decode(
+        &self,
+        token_ids: &[u32],
+        _skip_special_tokens: bool,
+    ) -> vllm_text::tokenizer::Result<String> {
+        Ok(token_ids
+            .iter()
+            .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
+            .collect())
+    }
+
+    fn token_to_id(&self, token: &str) -> Option<u32> {
+        match token {
+            "<|message_user|>" => Some(200000),
+            "<|message_model|>" => Some(200001),
+            "<|message_system|>" => Some(200002),
+            "<|message_tool|>" => Some(200003),
+            "<|content_text|>" => Some(200004),
+            "<|content_image|>" => Some(200005),
+            "<|content_model_end_sampling|>" => Some(200006),
+            "<|content_thinking|>" => Some(200008),
+            "<|end_message|>" => Some(200010),
+            "<|content_audio_input|>" => Some(200020),
+            CONTENT_XML => Some(200024),
+            "<|audio_end|>" => Some(200043),
+            "<|content_invoke_tool_json|>" => Some(200049),
+            _ => None,
+        }
+    }
+
+    fn id_to_token(&self, id: u32) -> Option<String> {
+        let token = match id {
+            200000 => "<|message_user|>",
+            200001 => "<|message_model|>",
+            200002 => "<|message_system|>",
+            200003 => "<|message_tool|>",
+            200004 => "<|content_text|>",
+            200005 => "<|content_image|>",
+            200006 => "<|content_model_end_sampling|>",
+            200008 => "<|content_thinking|>",
+            200010 => "<|end_message|>",
+            200020 => "<|content_audio_input|>",
+            200024 => CONTENT_XML,
+            200043 => "<|audio_end|>",
+            200049 => "<|content_invoke_tool_json|>",
+            _ => return None,
+        };
+        Some(token.to_string())
+    }
+}
+
+fn renderer() -> InklingChatRenderer {
+    InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap()
+}
+
+fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
+    renderer()
+        .render(request)
+        .unwrap()
+        .prompt
+        .into_token_ids()
+        .expect("Inkling renderer returns token IDs")
+}
+
+fn fixture_request(name: &str) -> ChatRequest {
+    fixture_chat_request(&fixture_path(name), inkling_fixture_options())
+}
+
+fn inkling_fixture_options() -> FixtureRequestOptions {
+    FixtureRequestOptions {
+        enable_thinking: false,
+        no_generation_prompt_when_last_assistant: false,
+    }
+}
+
+fn fixture_path(name: &str) -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("src/renderer/inkling")
+        .join("fixtures")
+        .join(name)
+}
+
+fn assert_fixture(input_name: &str, expected: ExpectFile) {
+    let request = fixture_request(input_name);
+    let rendered = format!("{}\n", render_symbolic_tokens(&render_token_ids(&request)));
+
+    expected.assert_eq(&rendered);
+}
+
+fn render_symbolic_tokens(token_ids: &[u32]) -> String {
+    let mut out = String::new();
+    let mut text_bytes = Vec::new();
+    for token_id in token_ids {
+        if let Some(marker) = symbolic_tokens()
+            .iter()
+            .find_map(|(marker, marker_id)| (token_id == marker_id).then_some(*marker))
+        {
+            flush_text_bytes(&mut out, &mut text_bytes);
+            out.push_str(marker);
+            continue;
+        }
+
+        let byte = u8::try_from(*token_id)
+            .unwrap_or_else(|_| panic!("unexpected non-byte fixture text token {token_id}"));
+        text_bytes.push(byte);
+    }
+    flush_text_bytes(&mut out, &mut text_bytes);
+    out
+}
+
+fn flush_text_bytes(out: &mut String, text_bytes: &mut Vec<u8>) {
+    if text_bytes.is_empty() {
+        return;
+    }
+    out.push_str(std::str::from_utf8(text_bytes).unwrap());
+    text_bytes.clear();
+}
+
+fn symbolic_tokens() -> [(&'static str, u32); 13] {
+    [
+        (MESSAGE_USER, 200000),
+        (MESSAGE_MODEL, 200001),
+        (MESSAGE_SYSTEM, 200002),
+        (MESSAGE_TOOL, 200003),
+        (CONTENT_TEXT, 200004),
+        (CONTENT_IMAGE, 200005),
+        (super::CONTENT_MODEL_END_SAMPLING, 200006),
+        (CONTENT_THINKING, 200008),
+        (END_MESSAGE, 200010),
+        (CONTENT_AUDIO_INPUT, 200020),
+        (CONTENT_XML, 200024),
+        (AUDIO_END, 200043),
+        (CONTENT_INVOKE_TOOL_JSON, 200049),
+    ]
+}
+
+#[test]
+fn renders_text_and_image_as_inkling_tokens() {
+    assert_fixture(
+        "text_image_input.json",
+        expect_file!["fixtures/text_image_output.txt"],
+    );
+}
+
+#[test]
+fn renders_audio_marker_only_blocks() {
+    assert_fixture(
+        "text_audio_input.json",
+        expect_file!["fixtures/text_audio_output.txt"],
+    );
+}
+
+#[test]
+fn renders_reasoning_text_tool_call_and_tool_response() {
+    assert_fixture(
+        "tool_round_trip_input.json",
+        expect_file!["fixtures/tool_round_trip_output.txt"],
+    );
+}
+
+#[test]
+fn renders_request_and_developer_tools_as_tool_declare() {
+    assert_fixture(
+        "tool_declare_input.json",
+        expect_file!["fixtures/tool_declare_output.txt"],
+    );
+}
+
+#[test]
+fn renders_named_reasoning_effort_after_tool_declarations() {
+    for (effort, expected) in [
+        (ReasoningEffort::None, "0.0"),
+        (ReasoningEffort::Minimal, "0.1"),
+        (ReasoningEffort::Low, "0.2"),
+        (ReasoningEffort::Medium, "0.7"),
+        (ReasoningEffort::High, "0.9"),
+        (ReasoningEffort::XHigh, "0.99"),
+        (ReasoningEffort::Max, "0.99"),
+    ] {
+        let mut request = fixture_request("tool_declare_input.json");
+        request.chat_options.reasoning_effort = Some(effort);
+
+        let rendered = render_symbolic_tokens(&render_token_ids(&request));
+        let tool_end = rendered.find("<|end_message|>").unwrap();
+        let effort_block = format!(
+            "<|message_system|><|content_text|>Thinking effort level: \
+             {expected}<|end_message|>"
+        );
+        let effort_start = rendered.find(&effort_block).unwrap();
+        let system_end = rendered.find("rules<|end_message|>").unwrap();
+
+        assert!(effort_start > tool_end);
+        assert!(effort_start > system_end);
+    }
+}
+
+#[test]
+fn emits_one_reasoning_effort_for_multi_turn_conversation() {
+    let request = ChatRequest {
+        messages: vec![
+            ChatMessage::system("rules"),
+            ChatMessage::user("user1"),
+            ChatMessage::assistant_text("assistant1"),
+            ChatMessage::user("user2"),
+        ],
+        ..ChatRequest::for_test()
+    };
+
+    let rendered = render_symbolic_tokens(&render_token_ids(&request));
+    let effort = "<|message_system|><|content_text|>Thinking effort level: 0.9\
+                  <|end_message|>";
+
+    assert_eq!(rendered.matches(effort).count(), 1);
+    assert!(rendered.find("rules").unwrap() < rendered.find(effort).unwrap());
+    assert!(rendered.find(effort).unwrap() < rendered.find("user1").unwrap());
+    assert!(rendered.find("user1").unwrap() < rendered.find("assistant1").unwrap());
+    assert!(rendered.find("assistant1").unwrap() < rendered.find("user2").unwrap());
+}
+
+#[test]
+fn canonicalizes_numeric_zero_reasoning_effort() {
+    for value in [0.0, -0.0] {
+        let mut request = ChatRequest::for_test();
+        request
+            .chat_options
+            .template_kwargs
+            .insert("reasoning_effort".to_string(), json!(value));
+
+        assert!(
+            render_symbolic_tokens(&render_token_ids(&request)).starts_with(
+                "<|message_system|><|content_text|>Thinking effort level: 0.0\
+                 <|end_message|>"
+            )
+        );
+    }
+}
+
+#[test]
+fn defaults_reasoning_effort_to_high() {
+    let request = ChatRequest::for_test();
+
+    assert!(
+        render_symbolic_tokens(&render_token_ids(&request)).starts_with(
+            "<|message_system|><|content_text|>Thinking effort level: 0.9\
+             <|end_message|>"
+        )
+    );
+}
+
+#[test]
+fn renders_numeric_reasoning_effort_template_kwarg() {
+    let mut request = ChatRequest::for_test();
+    request
+        .chat_options
+        .template_kwargs
+        .insert("reasoning_effort".to_string(), json!(0.8));
+
+    assert_eq!(
+        render_symbolic_tokens(&render_token_ids(&request)),
+        "<|message_system|><|content_text|>Thinking effort level: 0.8\
+         <|end_message|><|message_user|><|content_text|>test\
+         <|end_message|><|message_model|>"
+    );
+}
+
+#[test]
+fn ignores_unsupported_reasoning_effort_values() {
+    for value in [json!(true), json!("invalid"), json!(null)] {
+        let mut request = ChatRequest::for_test();
+        request
+            .chat_options
+            .template_kwargs
+            .insert("reasoning_effort".to_string(), value);
+
+        assert_eq!(
+            render_symbolic_tokens(&render_token_ids(&request)),
+            "<|message_user|><|content_text|>test<|end_message|><|message_model|>"
+        );
+    }
+}
+
+#[test]
+fn rejects_out_of_range_reasoning_effort() {
+    for value in [0.990_000_1, 1.0, 1.5, -0.1] {
+        let mut request = ChatRequest::for_test();
+        request
+            .chat_options
+            .template_kwargs
+            .insert("reasoning_effort".to_string(), json!(value));
+
+        let error = renderer().render(&request).unwrap_err();
+        assert!(error.as_report().to_string().contains("must be in [0.0, 0.99]"));
+    }
+}
+
+#[test]
+fn rejects_continue_final_message() {
+    let request = ChatRequest {
+        messages: vec![ChatMessage::assistant_text("partial")],
+        chat_options: crate::ChatOptions {
+            generation_prompt_mode: GenerationPromptMode::ContinueFinalAssistant,
+            ..Default::default()
+        },
+        ..ChatRequest::for_test()
+    };
+
+    let error = renderer().render(&request).unwrap_err();
+    assert!(matches!(error, Error::ChatTemplate(_)));
+    assert!(error.as_report().to_string().contains("continue_final_message"));
+}
+
+#[test]
+fn renders_developer_messages_as_system() {
+    let request = ChatRequest {
+        messages: vec![ChatMessage::developer("rules", None)],
+        ..ChatRequest::for_test()
+    };
+
+    assert_eq!(
+        render_symbolic_tokens(&render_token_ids(&request)),
+        "<|message_system|><|content_text|>rules<|end_message|>\
+         <|message_system|><|content_text|>Thinking effort level: 0.9\
+         <|end_message|><|message_model|>"
+    );
+}
+
+#[test]
+fn rejects_non_object_tool_call_arguments() {
+    let request = ChatRequest {
+        messages: vec![ChatMessage::assistant_blocks(vec![
+            AssistantContentBlock::ToolCall(AssistantToolCall {
+                id: "call_1".to_string(),
+                name: "get_weather".to_string(),
+                arguments: "[]".to_string(),
+            }),
+        ])],
+        ..ChatRequest::for_test()
+    };
+
+    let error = renderer().render(&request).unwrap_err();
+    assert!(error.as_report().to_string().contains("JSON object"));
+}

rust/src/chat/src/renderer/mod.rs

@@ -14,6 +14,7 @@ pub mod deepseek_v32;
 pub mod deepseek_v4;
 pub mod harmony;
 pub mod hf;
+mod inkling;
 mod selection;
 #[cfg(test)]
 mod test_utils;
@@ -21,6 +22,7 @@ mod test_utils;
 pub use deepseek_v4::DeepSeekV4ChatRenderer;
 pub use deepseek_v32::DeepSeekV32ChatRenderer;
 pub use harmony::HarmonyChatRenderer;
+pub use inkling::InklingChatRenderer;
 pub use selection::RendererSelection;
 
 /// Rendered chat prompt submitted to the text backend.

rust/src/chat/src/renderer/selection.rs

@@ -24,6 +24,8 @@ pub enum RendererSelection {
     DeepSeekV4,
     /// Force the GPT-OSS Harmony renderer.
     Harmony,
+    /// Force the Inkling native token renderer.
+    Inkling,
 }
 
 impl RendererSelection {
@@ -33,6 +35,8 @@ impl RendererSelection {
     pub const GPT_OSS_MODEL_TYPE: &str = "gpt_oss";
     pub const HARMONY_LITERAL: &str = "harmony";
     pub const HF_LITERAL: &str = "hf";
+    pub const INKLING_LITERAL: &str = "inkling";
+    pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
 
     /// Resolve the renderer selection using the given model type string, if
     /// it's `Auto`.
@@ -42,6 +46,7 @@ impl RendererSelection {
                 Self::DEEPSEEK_V32_LITERAL => Self::DeepSeekV32,
                 Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
                 Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
+                Self::INKLING_MODEL_TYPE => Self::Inkling,
                 _ => Self::Hf,
             },
             selection => selection,
@@ -63,6 +68,8 @@ impl FromStr for RendererSelection {
             Ok(Self::DeepSeekV4)
         } else if value.eq_ignore_ascii_case(Self::HARMONY_LITERAL) {
             Ok(Self::Harmony)
+        } else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
+            Ok(Self::Inkling)
         } else {
             Err(format!(
                 "unknown renderer `{value}` (expected one of: {})",
@@ -80,6 +87,7 @@ impl fmt::Display for RendererSelection {
             Self::DeepSeekV32 => f.write_str(Self::DEEPSEEK_V32_LITERAL),
             Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
             Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
+            Self::Inkling => f.write_str(Self::INKLING_LITERAL),
         }
     }
 }
@@ -106,7 +114,7 @@ mod tests {
     fn renderer_selection_expected_error_message() {
         let err = RendererSelection::from_str("unknown").unwrap_err();
         expect_test::expect![
-            "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)"
+            "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
         ]
         .assert_eq(&err);
     }

rust/src/chat/src/renderer/test_utils.rs

@@ -103,6 +103,8 @@ pub(crate) enum FixtureContent {
 pub(crate) enum FixtureContentPart {
     Text { text: String },
     ImageUrl { image_url: String },
+    InputAudio { data: String },
+    AudioUrl { audio_url: String },
 }
 
 #[derive(Debug, Deserialize)]
@@ -226,6 +228,15 @@ fn to_chat_content(content: FixtureContent) -> ChatContent {
                     FixtureContentPart::ImageUrl { image_url } => {
                         ChatContentPart::image_url(image_url)
                     }
+                    FixtureContentPart::InputAudio { data } => ChatContentPart::InputAudio {
+                        data,
+                        format: None,
+                        uuid: None,
+                    },
+                    FixtureContentPart::AudioUrl { audio_url } => ChatContentPart::AudioUrl {
+                        audio_url,
+                        uuid: None,
+                    },
                 })
                 .collect(),
         ),

rust/src/chat/tests/roundtrip.rs

@@ -263,19 +263,32 @@ impl RoundtripCase {
             sort_json_keys: false,
         }
     }
+
+    /// Inkling typed content blocks with native token-id rendering.
+    fn inkling() -> Self {
+        Self {
+            model_id: "thinkingmachines/Inkling",
+            assistant_stop_suffix: "",
+            tool_call_parser: ParserSelection::Auto,
+            reasoning_parser: ParserSelection::Auto,
+            thinking_behavior: ThinkingBehavior::Always { value: true },
+            json_fmt: compact_json_fmt(),
+            sort_json_keys: true,
+        }
+    }
 }
 
 macro_rules! roundtrip_tests {
-    ($($case:ident => [$($(#[$fixture_attr:meta])* $fixture:ident),* $(,)?]),+ $(,)?) => {
+    ($($case:ident => $(#[$case_attr:meta])* [$($fixture:ident),* $(,)?]),+ $(,)?) => {
         paste::paste! {
             $(
                 #[tokio::test]
                 #[file_serial([<hf_ $case>])]
+                $(#[$case_attr])*
                 async fn [<roundtrip_ $case>]() -> Result<()> {
                     let case = RoundtripCase::$case();
                     let backends = load_roundtrip_backends(&case).await?;
                     $(
-                        $(#[$fixture_attr])*
                         [<run_roundtrip_ $fixture>](&case, &backends).await?;
                     )*
                     Ok(())
@@ -300,6 +313,7 @@ roundtrip_tests! {
     gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
     kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
     gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
+    inkling => [reasoning_and_content, tool_call_mix],
 }
 
 /// Run the fixed reasoning+content fixture for one model/parser case.

rust/src/cmd/src/cli/tests.rs

@@ -646,7 +646,7 @@ fn serve_args_reject_unknown_renderer_value() {
     .unwrap_err();
 
     expect![[r#"
-        error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)
+        error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)
 
         For more information, try '--help'.
     "#]]

rust/src/parser/src/tool/json/mod.rs

@@ -31,24 +31,24 @@ use super::utils::{
 };
 use super::{Result, ToolCallDelta, ToolParserOutput};
 
-type JsonToolInput<'i> = Partial<&'i str>;
+pub(crate) type JsonToolInput<'i> = Partial<&'i str>;
 
 #[derive(Debug, Clone, Copy)]
-struct JsonToolCallConfig {
-    parser_name: &'static str,
-    start_marker: &'static str,
-    end_marker: &'static str,
-    marker_whitespace: JsonToolCallWhitespace,
-    delimiter: Option<&'static str>,
-    name_key: &'static str,
+pub(crate) struct JsonToolCallConfig {
+    pub parser_name: &'static str,
+    pub start_marker: &'static str,
+    pub end_marker: &'static str,
+    pub marker_whitespace: JsonToolCallWhitespace,
+    pub delimiter: Option<&'static str>,
+    pub name_key: &'static str,
     /// Candidate JSON keys naming the arguments payload, tried in order.
     /// Most parsers use a single key like `["arguments"]`, but some accept
     /// multiple (e.g. InternLM2 accepts `parameters` or `arguments`).
-    arguments_key: &'static [&'static str],
+    pub arguments_key: &'static [&'static str],
 }
 
 #[derive(Debug, Clone, Copy)]
-enum JsonToolCallWhitespace {
+pub(crate) enum JsonToolCallWhitespace {
     Optional,
     Exact(&'static str),
 }
@@ -61,7 +61,7 @@ enum JsonToolCallMode {
 }
 
 #[derive(Debug, Clone, PartialEq, Eq)]
-enum JsonToolCallEvent {
+pub(crate) enum JsonToolCallEvent {
     Text { len: usize },
     ToolCallStart,
     ToolCallHeader { function_name: String },
@@ -220,7 +220,7 @@ fn tool_call_start_event(
 
 /// Parse a marker-wrapped JSON tool-call header before the raw arguments
 /// payload.
-fn tool_call_header_event(
+pub(crate) fn tool_call_header_event(
     input: &mut JsonToolInput<'_>,
     config: JsonToolCallConfig,
 ) -> ModalResult<JsonToolCallEvent> {

rust/src/parser/src/tool/mod.rs

@@ -9,7 +9,7 @@ mod deepseek_dsml;
 pub(crate) mod deepseek_json;
 mod glm_xml;
 mod hy_v3;
-mod json;
+pub(crate) mod json;
 mod kimi_k2;
 mod minimax_m2;
 mod minimax_m3;

rust/src/parser/src/unified/gemma4.rs

@@ -10,7 +10,7 @@ use winnow::prelude::*;
 use winnow::stream::{Partial, Stream};
 use winnow::token::{literal, take_till, take_until};
 
-use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
+use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
 use crate::reasoning::last_reasoning_boundary;
 use crate::tool::{Tool, ToolCallDelta};
 use crate::unified::parsing_failed;
@@ -79,17 +79,8 @@ pub struct Gemma4UnifiedParser {
 impl Gemma4UnifiedParser {
     /// Create a Gemma4 parser.
     pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
-        let channel_start_token_id = tokenizer.token_to_id(CHANNEL_START).ok_or_else(|| {
-            UnifiedParserError::MissingToken {
-                token: CHANNEL_START.to_string(),
-            }
-        })?;
-        let channel_end_token_id =
-            tokenizer
-                .token_to_id(CHANNEL_END)
-                .ok_or_else(|| UnifiedParserError::MissingToken {
-                    token: CHANNEL_END.to_string(),
-                })?;
+        let channel_start_token_id = token_id(tokenizer.as_ref(), CHANNEL_START)?;
+        let channel_end_token_id = token_id(tokenizer.as_ref(), CHANNEL_END)?;
 
         Ok(Self {
             buffer: String::new(),
@@ -524,10 +515,10 @@ mod tests {
 
     use super::{
         CHANNEL_END, CHANNEL_START, Gemma4UnifiedParser, ToolCallDelta, UnifiedParser,
-        UnifiedParserError, UnifiedParserOutput, gemma4_array_content, parse_gemma4_args,
+        UnifiedParserOutput, gemma4_array_content, parse_gemma4_args,
     };
     use crate::tool::Tool;
-    use crate::unified::{UnifiedParserEvent, parsing_failed};
+    use crate::unified::{UnifiedParserError, UnifiedParserEvent, parsing_failed};
 
     const CHANNEL_START_ID: u32 = 256;
     const CHANNEL_END_ID: u32 = 257;

rust/src/parser/src/unified/inkling.rs

@@ -0,0 +1,768 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+use winnow::ascii::multispace0 as ws0;
+use winnow::combinator::{alt, seq};
+use winnow::error::ModalResult;
+use winnow::prelude::*;
+use winnow::stream::Partial;
+use winnow::token::literal;
+
+use vllm_tokenizer::DynTokenizer;
+
+use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
+use crate::tool::json::{
+    JsonToolCallConfig, JsonToolCallEvent, JsonToolCallWhitespace, JsonToolInput,
+    tool_call_header_event,
+};
+use crate::tool::{Tool, ToolCallDelta};
+use crate::unified::parsing_failed;
+use crate::utils::{
+    JsonObjectScanState, parse_buffered_event, safe_text_len_mul, take_json_object,
+};
+
+const CONTENT_TEXT: &str = "<|content_text|>";
+const CONTENT_THINKING: &str = "<|content_thinking|>";
+const CONTENT_INVOKE_TOOL_JSON: &str = "<|content_invoke_tool_json|>";
+const CONTENT_INVOKE_TOOL_TEXT: &str = "<|content_invoke_tool_text|>";
+const CONTENT_MODEL_END_SAMPLING: &str = "<|content_model_end_sampling|>";
+const CONTENT_TOOL_ERROR: &str = "<|content_tool_error|>";
+const MESSAGE_MODEL: &str = "<|message_model|>";
+const END_MESSAGE: &str = "<|end_message|>";
+
+const IDLE_MARKERS: &[&str] = &[
+    MESSAGE_MODEL,
+    CONTENT_TEXT,
+    CONTENT_THINKING,
+    CONTENT_INVOKE_TOOL_JSON,
+    CONTENT_INVOKE_TOOL_TEXT,
+    CONTENT_MODEL_END_SAMPLING,
+    CONTENT_TOOL_ERROR,
+    END_MESSAGE,
+];
+const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING];
+
+const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
+    parser_name: "Inkling",
+    start_marker: CONTENT_INVOKE_TOOL_JSON,
+    end_marker: END_MESSAGE,
+    marker_whitespace: JsonToolCallWhitespace::Optional,
+    delimiter: None,
+    name_key: "name",
+    arguments_key: &["args"],
+};
+
+type InklingInput<'i> = Partial<&'i str>;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+enum InklingEvent {
+    Text { len: usize },
+    Reasoning { len: usize },
+    TextStart,
+    ReasoningStart,
+    MessageStart,
+    Header,
+    ToolJsonStart,
+    ToolJsonHeader { name: String },
+    ToolJsonArgs { len: usize, complete: bool },
+    BlockEnd,
+}
+
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+enum InklingMode {
+    #[default]
+    Idle,
+    MessageHeader,
+    Text,
+    Reasoning,
+    ToolJsonHeader,
+    ToolJsonArgs {
+        json_scan: JsonObjectScanState,
+    },
+    ToolJsonClose,
+}
+
+/// Unified parser for Inkling typed content blocks.
+pub struct InklingUnifiedParser {
+    buffer: String,
+    mode: InklingMode,
+    emitted_tool_count: usize,
+    active_tool_index: Option<usize>,
+    tokenizer: DynTokenizer,
+    message_model_token_id: u32,
+    content_text_token_id: u32,
+    content_thinking_token_id: u32,
+}
+
+impl InklingUnifiedParser {
+    /// Create a Inkling parser.
+    pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
+        let message_model_token_id = token_id(tokenizer.as_ref(), MESSAGE_MODEL)?;
+        let content_text_token_id = token_id(tokenizer.as_ref(), CONTENT_TEXT)?;
+        let content_thinking_token_id = token_id(tokenizer.as_ref(), CONTENT_THINKING)?;
+
+        Ok(Self {
+            buffer: String::new(),
+            mode: InklingMode::Idle,
+            emitted_tool_count: 0,
+            active_tool_index: None,
+            tokenizer,
+            message_model_token_id,
+            content_text_token_id,
+            content_thinking_token_id,
+        })
+    }
+
+    fn initialize_mode(&mut self, prompt_token_ids: &[u32]) {
+        self.mode = InklingMode::Idle;
+        for token_id in prompt_token_ids.iter().rev().copied() {
+            if token_id == self.message_model_token_id {
+                self.mode = InklingMode::MessageHeader;
+                return;
+            }
+            if token_id == self.content_thinking_token_id {
+                self.mode = InklingMode::Reasoning;
+                return;
+            }
+            if token_id == self.content_text_token_id {
+                self.mode = InklingMode::Text;
+                return;
+            }
+            if self.tokenizer.is_special_id(token_id) {
+                return;
+            }
+        }
+    }
+
+    fn apply_event(&mut self, event: InklingEvent, output: &mut UnifiedParserOutput) -> Result<()> {
+        match event {
+            InklingEvent::Text { len } => output.push_text(self.buffer[..len].to_string()),
+            InklingEvent::Reasoning { len } => {
+                output.push_reasoning(self.buffer[..len].to_string());
+            }
+            InklingEvent::MessageStart => self.mode = InklingMode::MessageHeader,
+            InklingEvent::Header => {}
+            InklingEvent::TextStart => self.mode = InklingMode::Text,
+            InklingEvent::ReasoningStart => self.mode = InklingMode::Reasoning,
+            InklingEvent::ToolJsonStart => self.mode = InklingMode::ToolJsonHeader,
+            InklingEvent::ToolJsonHeader { name } => {
+                let tool_index = self.emitted_tool_count;
+                self.emitted_tool_count += 1;
+                self.active_tool_index = Some(tool_index);
+                self.mode = InklingMode::ToolJsonArgs {
+                    json_scan: JsonObjectScanState::default(),
+                };
+                output.push_call(ToolCallDelta {
+                    tool_index,
+                    name: Some(name),
+                    arguments: String::new(),
+                });
+            }
+            InklingEvent::ToolJsonArgs { len, complete } => {
+                let Some(tool_index) = self.active_tool_index else {
+                    return Err(parsing_failed!(
+                        "Inkling arguments without an active tool call"
+                    ));
+                };
+                output.push_call(ToolCallDelta {
+                    tool_index,
+                    name: None,
+                    arguments: self.buffer[..len].to_string(),
+                });
+                if complete {
+                    self.mode = InklingMode::ToolJsonClose;
+                }
+            }
+            InklingEvent::BlockEnd => {
+                self.mode = InklingMode::Idle;
+                self.active_tool_index = None;
+            }
+        }
+        Ok(())
+    }
+
+    fn reset(&mut self) -> String {
+        self.mode = InklingMode::Idle;
+        self.active_tool_index = None;
+        self.emitted_tool_count = 0;
+        std::mem::take(&mut self.buffer)
+    }
+}
+
+impl UnifiedParser for InklingUnifiedParser {
+    fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
+    where
+        Self: Sized + 'static,
+    {
+        Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
+    }
+
+    fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
+        self.buffer.clear();
+        self.emitted_tool_count = 0;
+        self.active_tool_index = None;
+        self.initialize_mode(prompt_token_ids);
+        Ok(())
+    }
+
+    fn preserve_special_tokens(&self) -> bool {
+        true
+    }
+
+    fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> {
+        self.buffer.push_str(chunk);
+
+        while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
+            parse_next_inkling_event(input, &mut self.mode)
+        })? {
+            self.apply_event(event, output)?;
+            self.buffer.drain(..consumed_len);
+        }
+
+        Ok(())
+    }
+
+    fn finish(&mut self) -> Result<UnifiedParserOutput> {
+        let mut output = UnifiedParserOutput::default();
+
+        match &self.mode {
+            InklingMode::Idle | InklingMode::Text => {
+                output.push_text(std::mem::take(&mut self.buffer))
+            }
+            InklingMode::MessageHeader => self.buffer.clear(),
+            InklingMode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)),
+            InklingMode::ToolJsonHeader
+            | InklingMode::ToolJsonArgs { .. }
+            | InklingMode::ToolJsonClose => {
+                return Err(parsing_failed!("incomplete Inkling tool call"));
+            }
+        }
+
+        let _ = self.reset();
+        Ok(output)
+    }
+
+    fn reset(&mut self) -> String {
+        InklingUnifiedParser::reset(self)
+    }
+}
+
+/// Parse one Inkling event from buffered streaming input.
+fn parse_next_inkling_event(
+    input: &mut InklingInput<'_>,
+    mode: &mut InklingMode,
+) -> ModalResult<InklingEvent> {
+    match mode {
+        InklingMode::Idle => parse_idle_event(input),
+        InklingMode::MessageHeader => parse_message_header_event(input),
+        InklingMode::Text => parse_text_event(input),
+        InklingMode::Reasoning => parse_reasoning_event(input),
+        InklingMode::ToolJsonHeader => parse_tool_json_header_event(input),
+        InklingMode::ToolJsonArgs { json_scan } => parse_tool_json_args_event(input, json_scan),
+        InklingMode::ToolJsonClose => parse_tool_json_close_event(input),
+    }
+}
+
+/// Parse an event while waiting for a Inkling content kind.
+fn parse_idle_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    alt((
+        message_start_event,
+        reasoning_start_event,
+        text_start_event,
+        tool_json_start_event,
+        raw_text_start_event,
+        block_end_event,
+        safe_idle_text_event,
+    ))
+    .parse_next(input)
+}
+
+/// Parse a Inkling model-authored message start marker.
+fn message_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    literal(MESSAGE_MODEL).value(InklingEvent::MessageStart).parse_next(input)
+}
+
+/// Parse an event while waiting for an Inkling message content kind.
+fn parse_message_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    alt((
+        reasoning_start_event,
+        text_start_event,
+        tool_json_start_event,
+        raw_text_start_event,
+        block_end_event,
+        safe_header_event,
+    ))
+    .parse_next(input)
+}
+
+/// Parse an event inside a Inkling text block.
+fn parse_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    alt((block_end_event, safe_text_event)).parse_next(input)
+}
+
+/// Parse an event inside a Inkling reasoning block.
+fn parse_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    alt((block_end_event, safe_reasoning_event)).parse_next(input)
+}
+
+/// Parse a Inkling text start marker.
+fn text_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    literal(CONTENT_TEXT).value(InklingEvent::TextStart).parse_next(input)
+}
+
+/// Parse a Inkling reasoning start marker.
+fn reasoning_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    literal(CONTENT_THINKING).value(InklingEvent::ReasoningStart).parse_next(input)
+}
+
+/// Parse a Inkling JSON tool-call start marker.
+fn tool_json_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    literal(CONTENT_INVOKE_TOOL_JSON)
+        .value(InklingEvent::ToolJsonStart)
+        .parse_next(input)
+}
+
+/// Parse a Inkling content kind treated as visible text.
+fn raw_text_start_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    alt((
+        literal(CONTENT_INVOKE_TOOL_TEXT),
+        literal(CONTENT_TOOL_ERROR),
+    ))
+    .value(InklingEvent::TextStart)
+    .parse_next(input)
+}
+
+/// Parse a Inkling block end marker.
+fn block_end_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    alt((literal(END_MESSAGE), literal(CONTENT_MODEL_END_SAMPLING)))
+        .value(InklingEvent::BlockEnd)
+        .parse_next(input)
+}
+
+/// Parse safe text while waiting for the next Inkling marker.
+fn safe_idle_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    safe_text_len_mul(input, IDLE_MARKERS).map(|len| InklingEvent::Text { len })
+}
+
+/// Parse safe header text before the next Inkling marker.
+fn safe_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    safe_text_len_mul(input, IDLE_MARKERS).map(|_| InklingEvent::Header)
+}
+
+/// Parse safe text before the end of a Inkling text block.
+fn safe_text_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Text { len })
+}
+
+/// Parse safe reasoning before the end of a Inkling reasoning block.
+fn safe_reasoning_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    safe_text_len_mul(input, BLOCK_END_MARKERS).map(|len| InklingEvent::Reasoning { len })
+}
+
+/// Parse a Inkling JSON tool-call header.
+fn parse_tool_json_header_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    match tool_call_header_event(input, INKLING_TOOL_CONFIG)? {
+        JsonToolCallEvent::ToolCallHeader { function_name } => Ok(InklingEvent::ToolJsonHeader {
+            name: function_name,
+        }),
+        _ => unreachable!("tool_call_header_event only emits ToolCallHeader"),
+    }
+}
+
+/// Parse raw Inkling JSON tool-call argument bytes.
+fn parse_tool_json_args_event(
+    input: &mut JsonToolInput<'_>,
+    json_scan: &mut JsonObjectScanState,
+) -> ModalResult<InklingEvent> {
+    let len = take_json_object(input, json_scan)?;
+    Ok(InklingEvent::ToolJsonArgs {
+        len,
+        complete: json_scan.complete(),
+    })
+}
+
+/// Parse the close of a Inkling JSON tool-call block.
+fn parse_tool_json_close_event(input: &mut InklingInput<'_>) -> ModalResult<InklingEvent> {
+    seq!(
+        _: ws0,
+        _: literal("}"),
+        _: ws0,
+        _: literal(END_MESSAGE),
+    )
+    .value(InklingEvent::BlockEnd)
+    .parse_next(input)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use super::{CONTENT_TEXT, CONTENT_THINKING, InklingUnifiedParser, MESSAGE_MODEL};
+    use crate::tool::Tool;
+    use crate::unified::{UnifiedParser, UnifiedParserEvent, UnifiedParserOutput};
+    use thiserror_ext::AsReport;
+    use vllm_tokenizer::Tokenizer;
+
+    struct FakeTokenizer;
+
+    impl Tokenizer for FakeTokenizer {
+        fn encode(
+            &self,
+            text: &str,
+            _add_special_tokens: bool,
+        ) -> vllm_tokenizer::Result<Vec<u32>> {
+            Ok(text.chars().map(u32::from).collect())
+        }
+
+        fn decode(
+            &self,
+            token_ids: &[u32],
+            _skip_special_tokens: bool,
+        ) -> vllm_tokenizer::Result<String> {
+            Ok(token_ids
+                .iter()
+                .map(|token_id| char::from_u32(*token_id).unwrap_or('\u{FFFD}'))
+                .collect())
+        }
+
+        fn token_to_id(&self, token: &str) -> Option<u32> {
+            match token {
+                MESSAGE_MODEL => Some(200001),
+                "<|content_text|>" => Some(200004),
+                "<|content_model_end_sampling|>" => Some(200006),
+                "<|content_thinking|>" => Some(200008),
+                "<|end_message|>" => Some(200010),
+                "<|content_tool_error|>" => Some(200022),
+                "<|content_invoke_tool_json|>" => Some(200049),
+                "<|content_invoke_tool_text|>" => Some(200057),
+                _ => None,
+            }
+        }
+
+        fn id_to_token(&self, id: u32) -> Option<String> {
+            let token = match id {
+                200001 => MESSAGE_MODEL,
+                200004 => "<|content_text|>",
+                200006 => "<|content_model_end_sampling|>",
+                200008 => "<|content_thinking|>",
+                200010 => "<|end_message|>",
+                200022 => "<|content_tool_error|>",
+                200049 => "<|content_invoke_tool_json|>",
+                200057 => "<|content_invoke_tool_text|>",
+                _ => return None,
+            };
+            Some(token.to_string())
+        }
+
+        fn is_special_id(&self, token_id: u32) -> bool {
+            (199999..=200057).contains(&token_id)
+        }
+    }
+
+    trait UnifiedParserTestExt {
+        fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput>;
+        fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput>;
+    }
+
+    impl<T: UnifiedParser + ?Sized> UnifiedParserTestExt for T {
+        fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput> {
+            let mut output = UnifiedParserOutput::default();
+            self.parse_into(chunk, &mut output)?;
+            Ok(output)
+        }
+
+        fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput> {
+            let mut output = self.parse_chunk(text)?;
+            output.append(self.finish()?);
+            Ok(output)
+        }
+    }
+
+    trait UnifiedParserOutputTestExt {
+        fn normal_text(&self) -> String;
+        fn reasoning_text(&self) -> String;
+        fn calls(&self) -> Vec<crate::tool::ToolCallDelta>;
+    }
+
+    impl UnifiedParserOutputTestExt for UnifiedParserOutput {
+        fn normal_text(&self) -> String {
+            self.events
+                .iter()
+                .filter_map(|event| match event {
+                    UnifiedParserEvent::Text(text) => Some(text.as_str()),
+                    _ => None,
+                })
+                .collect()
+        }
+
+        fn reasoning_text(&self) -> String {
+            self.events
+                .iter()
+                .filter_map(|event| match event {
+                    UnifiedParserEvent::Reasoning(text) => Some(text.as_str()),
+                    _ => None,
+                })
+                .collect()
+        }
+
+        fn calls(&self) -> Vec<crate::tool::ToolCallDelta> {
+            self.events
+                .iter()
+                .filter_map(|event| match event {
+                    UnifiedParserEvent::ToolCall(call) => Some(call.clone()),
+                    _ => None,
+                })
+                .collect()
+        }
+    }
+
+    fn test_tools() -> Vec<Tool> {
+        vec![Tool {
+            name: "get_weather".to_string(),
+            description: Some("Get weather".to_string()),
+            parameters: serde_json::json!({
+                "type": "object",
+                "properties": {
+                    "city": {"type": "string"}
+                },
+            }),
+            strict: None,
+        }]
+    }
+
+    fn test_parser() -> InklingUnifiedParser {
+        InklingUnifiedParser::new(&test_tools(), Arc::new(FakeTokenizer)).unwrap()
+    }
+
+    fn collect_stream(chunks: &[&str]) -> UnifiedParserOutput {
+        let mut parser = test_parser();
+        let mut output = UnifiedParserOutput::default();
+        for chunk in chunks {
+            output.append(parser.parse_chunk(chunk).unwrap());
+        }
+        output.append(parser.finish().unwrap());
+        output
+    }
+
+    #[test]
+    fn inkling_streaming_emits_reasoning_then_text() {
+        let output = collect_stream(&[concat!(
+            "<|content_thinking|>reason<|end_message|>",
+            "<|message_model|><|content_text|>answer<|end_message|>",
+            "<|content_model_end_sampling|>"
+        )]);
+
+        assert_eq!(output.reasoning_text(), "reason");
+        assert_eq!(output.normal_text(), "answer");
+        assert!(output.calls().is_empty());
+    }
+
+    #[test]
+    fn inkling_streaming_holds_split_markers() {
+        let output = collect_stream(&[
+            "<|content_thin",
+            "king|>rea",
+            "son<|end_mes",
+            "sage|><|message_",
+            "model|><|content_text|>answer",
+            "<|end_message|>",
+        ]);
+
+        assert_eq!(output.reasoning_text(), "reason");
+        assert_eq!(output.normal_text(), "answer");
+    }
+
+    #[test]
+    fn inkling_streaming_accepts_content_blocks_without_message_start() {
+        let output = collect_stream(&[concat!(
+            "<|content_thinking|>reason<|end_message|>",
+            "<|content_text|>answer<|end_message|>",
+        )]);
+
+        assert_eq!(output.reasoning_text(), "reason");
+        assert_eq!(output.normal_text(), "answer");
+    }
+
+    #[test]
+    fn inkling_tool_json_streams_name_then_argument_deltas() {
+        let mut parser = test_parser();
+        let chunks = [
+            "<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":",
+            "{\"city\":",
+            "\"SF\"",
+            "}}<|end_message|>",
+        ];
+
+        let mut output = UnifiedParserOutput::default();
+        let mut observed_args = Vec::new();
+        for chunk in chunks {
+            let next = parser.parse_chunk(chunk).unwrap();
+            observed_args.extend(
+                next.calls()
+                    .into_iter()
+                    .filter(|call| call.name.is_none())
+                    .map(|call| call.arguments),
+            );
+            output.append(next);
+        }
+        output.append(parser.finish().unwrap());
+
+        let calls = output.calls();
+        assert_eq!(calls[0].name.as_deref(), Some("get_weather"));
+        assert_eq!(observed_args, ["{\"city\":", "\"SF\"", "}"]);
+        assert_eq!(
+            calls.iter().map(|call| call.arguments.as_str()).collect::<String>(),
+            "{\"city\":\"SF\"}"
+        );
+        assert!(output.normal_text().is_empty());
+    }
+
+    #[test]
+    fn inkling_discards_tool_name_from_message_header() {
+        let output = collect_stream(&[concat!(
+            "<|message_model|>get_weather<|content_invoke_tool_json|>",
+            "{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
+        )]);
+
+        assert!(output.normal_text().is_empty());
+        assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
+    }
+
+    #[test]
+    fn inkling_streaming_handles_multiple_tool_blocks() {
+        let output = collect_stream(&[concat!(
+            "<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"SF\"}}<|end_message|>",
+            "<|message_model|><|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\":\"NYC\"}}<|end_message|>"
+        )]);
+
+        let calls = output.calls();
+        assert_eq!(calls.iter().filter(|call| call.name.is_some()).count(), 2);
+        assert_eq!(calls[0].tool_index, 0);
+        assert_eq!(calls[2].tool_index, 1);
+    }
+
+    #[test]
+    fn inkling_initialize_open_text_prompt_starts_in_text() {
+        let mut parser = test_parser();
+        parser.initialize(&[200004]).unwrap();
+
+        let output = parser.parse_complete("answer<|end_message|>").unwrap();
+
+        assert_eq!(output.normal_text(), "answer");
+        assert!(output.reasoning_text().is_empty());
+    }
+
+    #[test]
+    fn inkling_initialize_open_reasoning_prompt_starts_in_reasoning() {
+        let mut parser = test_parser();
+        parser.initialize(&[200008]).unwrap();
+
+        let output = parser.parse_complete("reason<|end_message|>").unwrap();
+
+        assert_eq!(output.reasoning_text(), "reason");
+        assert!(output.normal_text().is_empty());
+    }
+
+    #[test]
+    fn inkling_initialize_model_opener_starts_in_message_header() {
+        let mut parser = test_parser();
+        parser.initialize(&[200001]).unwrap();
+
+        let output = parser
+            .parse_complete(concat!(
+                "get_weather<|content_invoke_tool_json|>",
+                "{\"name\":\"get_weather\",\"args\":{}}<|end_message|>"
+            ))
+            .unwrap();
+
+        assert!(output.normal_text().is_empty());
+        assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
+    }
+
+    #[test]
+    fn inkling_plain_text_falls_through_as_text() {
+        let output = collect_stream(&["plain ", "answer"]);
+
+        assert_eq!(output.normal_text(), "plain answer");
+        assert!(output.reasoning_text().is_empty());
+    }
+
+    #[test]
+    fn inkling_raw_tool_text_and_tool_error_are_visible_text() {
+        let output = collect_stream(&[concat!(
+            "<|content_invoke_tool_text|>search SF<|end_message|>",
+            "<|content_tool_error|>failed<|end_message|>"
+        )]);
+
+        assert_eq!(output.normal_text(), "search SFfailed");
+        assert!(output.calls().is_empty());
+    }
+
+    #[test]
+    fn inkling_finish_fails_incomplete_tool_call() {
+        let mut parser = test_parser();
+        parser
+            .parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":{\"city\"")
+            .unwrap();
+
+        let error = parser.finish().unwrap_err();
+
+        assert!(error.as_report().to_string().contains("incomplete Inkling tool call"));
+    }
+
+    #[test]
+    fn inkling_rejects_non_object_args() {
+        let mut parser = test_parser();
+        let error = parser
+            .parse_chunk("<|content_invoke_tool_json|>{\"name\":\"get_weather\",\"args\":42")
+            .unwrap_err();
+
+        assert!(error.as_report().to_string().contains("JSON object argument"));
+    }
+
+    #[test]
+    fn inkling_missing_token_fails_create() {
+        struct MissingTokenizer;
+
+        impl Tokenizer for MissingTokenizer {
+            fn encode(
+                &self,
+                _text: &str,
+                _add_special_tokens: bool,
+            ) -> vllm_tokenizer::Result<Vec<u32>> {
+                Ok(vec![])
+            }
+
+            fn decode(
+                &self,
+                _token_ids: &[u32],
+                _skip_special_tokens: bool,
+            ) -> vllm_tokenizer::Result<String> {
+                Ok(String::new())
+            }
+
+            fn token_to_id(&self, token: &str) -> Option<u32> {
+                match token {
+                    MESSAGE_MODEL => Some(200001),
+                    CONTENT_TEXT => Some(200004),
+                    _ => None,
+                }
+            }
+
+            fn id_to_token(&self, id: u32) -> Option<String> {
+                match id {
+                    200001 => Some(MESSAGE_MODEL.to_string()),
+                    200004 => Some(CONTENT_TEXT.to_string()),
+                    _ => None,
+                }
+            }
+        }
+
+        let error = match InklingUnifiedParser::new(&[], Arc::new(MissingTokenizer)) {
+            Ok(_) => panic!("expected parser creation to fail"),
+            Err(error) => error,
+        };
+
+        assert!(error.as_report().to_string().contains(CONTENT_THINKING));
+    }
+}

rust/src/parser/src/unified/mod.rs

@@ -5,9 +5,11 @@
 
 mod combined;
 mod gemma4;
+mod inkling;
 
 pub use combined::CombinedParser;
 pub use gemma4::Gemma4UnifiedParser;
+pub use inkling::InklingUnifiedParser;
 use thiserror::Error;
 use thiserror_ext::Macro;
 use vllm_tokenizer::DynTokenizer;
@@ -208,3 +210,10 @@ pub enum UnifiedParserError {
     #[error(transparent)]
     Tool(#[from] ToolParserError),
 }
+
+/// Returns the ID for the given token, or an error if it's not found.
+fn token_id(tokenizer: &dyn vllm_tokenizer::Tokenizer, token: &str) -> Result<u32> {
+    tokenizer.token_to_id(token).ok_or_else(|| UnifiedParserError::MissingToken {
+        token: token.to_string(),
+    })
+}

rust/src/text/src/backend/hf/config.rs

@@ -95,6 +95,7 @@ impl HfSpecialTokens {
 pub struct ModelConfig {
     model_type: Option<String>,
     vocab_size: Option<u32>,
+    eos_token_id: Option<OneOrManyTokenIds>,
     num_experts: Option<OneOrManyExpertCount>,
     moe_num_experts: Option<OneOrManyExpertCount>,
     n_routed_experts: Option<OneOrManyExpertCount>,
@@ -125,12 +126,16 @@ pub(super) enum OneOrManyTokenIds {
 }
 
 impl OneOrManyTokenIds {
-    pub(super) fn into_set(self) -> BTreeSet<u32> {
+    pub(super) fn as_slice(&self) -> &[u32] {
         match self {
-            Self::One(id) => BTreeSet::from([id]),
-            Self::Many(ids) => ids.into_iter().collect(),
+            Self::One(id) => std::slice::from_ref(id),
+            Self::Many(ids) => ids.as_slice(),
         }
     }
+
+    pub(super) fn into_set(self) -> BTreeSet<u32> {
+        self.as_slice().iter().copied().collect()
+    }
 }
 
 /// Hugging Face configs may expose the expert count either as one integer or
@@ -195,6 +200,18 @@ impl ModelConfig {
         }
     }
 
+    /// Return the effective model-side EOS token ids, following the same
+    /// simplified text-config selection as `vocab_size`.
+    pub(super) fn eos_token_ids(&self) -> &[u32] {
+        if let Some(eos_token_id) = self.eos_token_id.as_ref() {
+            eos_token_id.as_slice()
+        } else if let Some(text_config) = self.text_config.as_deref() {
+            text_config.eos_token_ids()
+        } else {
+            &[]
+        }
+    }
+
     /// Match Python's current expert-count priority on the selected text
     /// config.
     ///
@@ -354,6 +371,32 @@ mod tests {
         assert_eq!(config.vocab_size().unwrap(), 151936);
     }
 
+    #[test]
+    fn model_config_reads_top_level_eos_token_ids() {
+        let single: ModelConfig = serde_json::from_str(r#"{"eos_token_id":151645}"#).unwrap();
+        let many: ModelConfig =
+            serde_json::from_str(r#"{"eos_token_id":[128001,128008,128009]}"#).unwrap();
+        let null: ModelConfig = serde_json::from_str(r#"{"eos_token_id":null}"#).unwrap();
+
+        assert_eq!(single.eos_token_ids(), &[151645]);
+        assert_eq!(many.eos_token_ids(), &[128001, 128008, 128009]);
+        assert!(null.eos_token_ids().is_empty());
+    }
+
+    #[test]
+    fn model_config_uses_nested_eos_token_ids_when_top_level_is_absent() {
+        let config: ModelConfig = serde_json::from_str(
+            r#"{
+                "text_config": {
+                    "eos_token_id": [59246, 59253, 59255]
+                }
+            }"#,
+        )
+        .unwrap();
+
+        assert_eq!(config.eos_token_ids(), &[59246, 59253, 59255]);
+    }
+
     #[test]
     fn model_config_rejects_missing_vocab_size() {
         let config: ModelConfig = serde_json::from_str(r#"{}"#).unwrap();

rust/src/text/src/backend/hf/mod.rs

@@ -8,7 +8,9 @@ use std::collections::BTreeSet;
 use std::sync::Arc;
 
 use tracing::info;
-use vllm_tokenizer::{DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer};
+use vllm_tokenizer::{
+    DynTokenizer, HuggingFaceTokenizer, TekkenTokenizer, TiktokenTokenizer, Tokenizer,
+};
 
 use self::config::{GenerationConfig, load_generation_config};
 pub use self::config::{
@@ -56,23 +58,15 @@ impl HfTextBackend {
     pub fn from_resolved_model_files(files: ResolvedModelFiles, model_id: String) -> Result<Self> {
         let tokenizer_config = load_tokenizer_config(files.tokenizer_config_path.as_deref())?;
         let tokenizer = load_tokenizer(&files.tokenizer)?;
-        let primary_eos_token_id = tokenizer_config
-            .special_tokens
-            .eos_token
-            .as_ref()
-            .and_then(|token| tokenizer.token_to_id(token.as_str()));
-
         let model_config = load_model_config(files.config_path.as_deref())?;
         let model_vocab_size = model_config.vocab_size()? as usize;
         let generation_config = load_generation_config(files.generation_config_path.as_deref())?;
-        let mut extra_eos_token_ids = generation_config
-            .eos_token_id
-            .clone()
-            .map(|value| value.into_set())
-            .unwrap_or_default();
-        if let Some(primary_eos_token_id) = primary_eos_token_id {
-            extra_eos_token_ids.remove(&primary_eos_token_id);
-        }
+        let (primary_eos_token_id, extra_eos_token_ids) = resolve_eos_token_ids(
+            &tokenizer_config,
+            &model_config,
+            &generation_config,
+            tokenizer.as_ref(),
+        );
 
         info!(
             model_id,
@@ -98,6 +92,42 @@ impl HfTextBackend {
     }
 }
 
+/// Resolve EOS hints from tokenizer, model, and generation configs.
+///
+/// Resolution rules:
+/// 1. Use the tokenizer-side EOS token as the primary EOS when it resolves to a
+///    token id.
+/// 2. Fall back to the first model-config EOS id when the tokenizer config does
+///    not provide a primary EOS.
+/// 3. Keep any remaining model/generation EOS ids as extra stop-token ids so
+///    they still participate in stopping and min-token handling.
+fn resolve_eos_token_ids(
+    tokenizer_config: &HfTokenizerConfig,
+    model_config: &ModelConfig,
+    generation_config: &GenerationConfig,
+    tokenizer: &dyn Tokenizer,
+) -> (Option<u32>, BTreeSet<u32>) {
+    let model_config_eos_token_ids = model_config.eos_token_ids();
+    let primary_eos_token_id = tokenizer_config
+        .special_tokens
+        .eos_token
+        .as_ref()
+        .and_then(|token| tokenizer.token_to_id(token.as_str()))
+        .or_else(|| model_config_eos_token_ids.first().copied());
+
+    let mut extra_eos_token_ids = generation_config
+        .eos_token_id
+        .clone()
+        .map(|value| value.into_set())
+        .unwrap_or_default();
+    extra_eos_token_ids.extend(model_config_eos_token_ids.iter().copied());
+    if let Some(primary_eos_token_id) = primary_eos_token_id {
+        extra_eos_token_ids.remove(&primary_eos_token_id);
+    }
+
+    (primary_eos_token_id, extra_eos_token_ids)
+}
+
 impl TextBackend for HfTextBackend {
     fn tokenizer(&self) -> DynTokenizer {
         self.tokenizer.clone()
@@ -128,3 +158,77 @@ impl TextBackend for HfTextBackend {
         })
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::collections::BTreeSet;
+
+    use super::{GenerationConfig, HfTokenizerConfig, ModelConfig, resolve_eos_token_ids};
+    use vllm_tokenizer::Tokenizer;
+
+    struct FakeTokenizer;
+
+    impl Tokenizer for FakeTokenizer {
+        fn encode(
+            &self,
+            _text: &str,
+            _add_special_tokens: bool,
+        ) -> vllm_tokenizer::Result<Vec<u32>> {
+            Ok(vec![])
+        }
+
+        fn decode(
+            &self,
+            _token_ids: &[u32],
+            _skip_special_tokens: bool,
+        ) -> vllm_tokenizer::Result<String> {
+            Ok(String::new())
+        }
+
+        fn token_to_id(&self, token: &str) -> Option<u32> {
+            (token == "</s>").then_some(2)
+        }
+
+        fn id_to_token(&self, id: u32) -> Option<String> {
+            (id == 2).then(|| "</s>".to_string())
+        }
+    }
+
+    #[test]
+    fn eos_resolution_uses_model_config_when_tokenizer_has_no_eos() {
+        let tokenizer_config: HfTokenizerConfig = serde_json::from_str("{}").unwrap();
+        let model_config: ModelConfig =
+            serde_json::from_str(r#"{"eos_token_id":[200006,200010]}"#).unwrap();
+        let generation_config: GenerationConfig = serde_json::from_str("{}").unwrap();
+
+        let (primary, extra) = resolve_eos_token_ids(
+            &tokenizer_config,
+            &model_config,
+            &generation_config,
+            &FakeTokenizer,
+        );
+
+        assert_eq!(primary, Some(200006));
+        assert_eq!(extra, BTreeSet::from([200010]));
+    }
+
+    #[test]
+    fn eos_resolution_keeps_tokenizer_eos_primary() {
+        let tokenizer_config: HfTokenizerConfig =
+            serde_json::from_str(r#"{"eos_token":"</s>"}"#).unwrap();
+        let model_config: ModelConfig =
+            serde_json::from_str(r#"{"eos_token_id":[2,200006]}"#).unwrap();
+        let generation_config: GenerationConfig =
+            serde_json::from_str(r#"{"eos_token_id":[2,200010]}"#).unwrap();
+
+        let (primary, extra) = resolve_eos_token_ids(
+            &tokenizer_config,
+            &model_config,
+            &generation_config,
+            &FakeTokenizer,
+        );
+
+        assert_eq!(primary, Some(2));
+        assert_eq!(extra, BTreeSet::from([200006, 200010]));
+    }
+}

setup.py

@@ -445,6 +445,17 @@ class cmake_build_ext(build_ext):
                     dirs_exist_ok=True,
                 )
 
+            tml_fa4_build = os.path.join(
+                self.build_lib, "vllm", "third_party", "tml_fa4"
+            )
+            if os.path.exists(tml_fa4_build):
+                print(f"Copying {tml_fa4_build} to vllm/third_party/tml_fa4")
+                shutil.copytree(
+                    tml_fa4_build,
+                    "vllm/third_party/tml_fa4",
+                    dirs_exist_ok=True,
+                )
+
 
 class precompiled_build_ext(build_ext):
     """Disables extension building when using precompiled binaries."""
@@ -803,6 +814,7 @@ class precompiled_wheel_utils:
                 # DeepGEMM: extract all files (.py, .so, .cuh, .h, .hpp, etc.)
                 deep_gemm_regex = re.compile(r"vllm/third_party/deep_gemm/.*")
                 fmha_sm100_regex = re.compile(r"vllm/third_party/fmha_sm100/.*")
+                tml_fa4_regex = re.compile(r"vllm/third_party/tml_fa4/.*")
                 file_members = []
                 for member in wheel.filelist:
                     if member.filename in exact_members:
@@ -828,6 +840,7 @@ class precompiled_wheel_utils:
                         or triton_kernels_regex.match(member.filename)
                         or flashmla_regex.match(member.filename)
                         or deep_gemm_regex.match(member.filename)
+                        or tml_fa4_regex.match(member.filename)
                         or fmha_sm100_regex.match(member.filename)
                     ):
                         file_members.append(member)
@@ -1146,6 +1159,8 @@ if _is_cuda():
         ext_modules.append(CMakeExtension(name="vllm._qutlass_C", optional=True))
     # fmha_sm100 is a Python/CuTe-DSL package installed into vllm.third_party.
     ext_modules.append(CMakeExtension(name="vllm.fmha_sm100", optional=True))
+    # tml-fa4 is copied into an isolated vllm.third_party package.
+    ext_modules.append(CMakeExtension(name="vllm.tml_fa4", optional=True))
 
 if _is_cpu():
     import platform

tests/kernels/test_ll_bf16_gemm.py

@@ -190,8 +190,8 @@ def test_arbitrary_N_splitk(N):
 
 @pytest.mark.parametrize(
     "N,K",
-    [(256, 7168), (256, 14400), (8, 4096), (384, 7168)],
-    ids=["DSV3", "DSV4-Flash", "Mixtral", "DSV4-Pro"],
+    [(256, 7168), (256, 14400), (8, 4096), (384, 7168), (264, 6144)],
+    ids=["DSV3", "DSV4-Flash", "Mixtral", "DSV4-Pro", "Inkling"],
 )
 def test_single_token(N, K):
     torch.manual_seed(42)
@@ -202,6 +202,15 @@ def test_single_token(N, K):
     _assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
 
 
+def test_inkling_max_tokens():
+    torch.manual_seed(42)
+    a = torch.randn(64, 6144, dtype=torch.bfloat16, device="cuda")
+    b = torch.randn(264, 6144, dtype=torch.bfloat16, device="cuda")
+    out = _gemm(a, b)
+    assert out.shape == (64, 264)
+    _assert_close(out, _ref(a, b), context="Inkling M=64")
+
+
 # =================================================================
 # Numerical robustness
 # =================================================================

tests/models/inkling/test_contract_validation.py

@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import numpy as np
+import pytest
+
+from vllm.config.compilation import CompilationConfig, CUDAGraphMode
+from vllm.models.inkling.common.mm_preprocess import InklingMultiModalDataParser
+from vllm.models.inkling.configs import (
+    InklingAudioConfig,
+    InklingVisionConfig,
+)
+from vllm.models.inkling.nvidia.sconv_swa_attn import (
+    InklingSconvMetadataBuilder,
+)
+from vllm.v1.attention.backend import AttentionCGSupport
+
+
+@pytest.mark.parametrize(
+    ("config_cls", "kwargs", "missing"),
+    [
+        (InklingAudioConfig, {"decoder_dmodel": 16}, "n_mel_bins"),
+        (InklingVisionConfig, {"decoder_dmodel": 16}, "vision_encoder_type"),
+    ],
+)
+def test_enabled_tower_requires_architecture_fields(config_cls, kwargs, missing):
+    with pytest.raises(ValueError, match=missing):
+        config_cls(**kwargs)
+
+
+def test_inkling_raw_2d_audio_is_rejected_as_ambiguous():
+    parser = InklingMultiModalDataParser(target_sr=16_000, target_channels=1)
+    with pytest.raises(ValueError, match="ambiguous channel layout"):
+        parser._parse_audio_data(np.zeros((2, 100), dtype=np.float32))
+
+
+def test_inkling_supports_full_decode_only_cudagraphs():
+    support = InklingSconvMetadataBuilder.get_cudagraph_support
+    assert support(None, None) == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
+
+    compilation_config = CompilationConfig(
+        cudagraph_mode=CUDAGraphMode.FULL,
+        splitting_ops=[],
+    )
+    resolved_mode = compilation_config.resolve_cudagraph_mode_and_sizes(
+        AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE,
+        "InklingSconvBackend",
+    )
+
+    assert resolved_mode == CUDAGraphMode.FULL_DECODE_ONLY

tests/models/inkling/test_fa4_rel_attention.py

@@ -0,0 +1,381 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Correctness test for the Inkling FA4 relative-attention score-mod kernel.
+
+Checks ``inkling_fa4_rel_attention`` against a pure-PyTorch reference that
+implements the relative bias exactly as documented in the Inkling architecture
+guide::
+
+    logit(i, j, h) = (1 / head_dim) * dot(q[i, h], k[j, h]) + rel_bias(i, j, h)
+    rel_bias(i, j, h) = rel_logits[i, h, i - j]   if 0 <= i - j < rel_extent
+                      = 0                          otherwise
+
+with causal (and optionally sliding-window) masking handled by the backend.
+"""
+
+import pytest
+import torch
+
+from vllm.models.inkling.nvidia.attention import (
+    InklingAttention,
+    compute_log_scaling_tau,
+)
+from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
+    inkling_fa4_num_splits,
+    inkling_fa4_rel_attention,
+)
+from vllm.platforms import current_platform
+from vllm.platforms.interface import DeviceCapability
+
+_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None
+
+NUM_HEADS = [(4, 4), (8, 2)]  # (num_heads, num_kv_heads)
+GLOBAL_REL_EXTENTS = [128, 1024]
+LOCAL_REL_EXTENTS = [128, 256]
+HEAD_DIM = 128
+BLOCK_SIZE = 16
+DTYPE = torch.bfloat16
+
+
+def test_log_scaling_tau_matches_reference():
+    positions = torch.tensor([0, 127999, 128000, 999999], dtype=torch.int64)
+    actual = compute_log_scaling_tau(positions, 128000, 0.1)
+    expected = 1.0 + 0.1 * torch.log(
+        torch.clamp((positions + 1).float() / 128000.0, min=1.0)
+    )
+    torch.testing.assert_close(actual, expected, rtol=0, atol=0)
+
+
+def test_split_packed_kv_cache():
+    attention = InklingAttention.__new__(InklingAttention)
+    torch.nn.Module.__init__(attention)
+    attention.head_dim = 8
+    attention.kv_cache = torch.arange(2 * 3 * 4 * 16).reshape(2, 3, 4, 16)
+
+    key_cache, value_cache = attention._split_kv_cache()
+
+    assert key_cache.shape == value_cache.shape == (2, 4, 3, 8)
+    torch.testing.assert_close(key_cache, attention.kv_cache[..., :8].transpose(1, 2))
+    torch.testing.assert_close(value_cache, attention.kv_cache[..., 8:].transpose(1, 2))
+
+
+def test_num_splits_hopper_is_unsplit(monkeypatch):
+    monkeypatch.setattr(
+        current_platform,
+        "get_device_capability",
+        lambda: DeviceCapability(major=9, minor=0),
+    )
+    assert (
+        inkling_fa4_num_splits(
+            is_local=False,
+            batch_size=1,
+            max_query_len=1,
+            num_heads=16,
+            num_kv_heads=2,
+            max_kv_len=1_048_576,
+        )
+        == 1
+    )
+
+
+@pytest.fixture
+def blackwell_platform(monkeypatch):
+    monkeypatch.setattr(
+        current_platform,
+        "get_device_capability",
+        lambda: DeviceCapability(major=10, minor=0),
+    )
+
+
+@pytest.mark.parametrize(
+    ("batch_size", "max_query_len", "expected"),
+    [
+        (1, 1, (16, 32, 128, 128)),
+        (8, 1, (2, 4, 8, 16)),
+        (32, 1, (1, 1, 2, 4)),
+        (1, 128, (2, 4, 8, 16)),
+        (1, 2048, (1, 1, 1, 1)),
+    ],
+)
+def test_num_splits_all_tp(blackwell_platform, batch_size, max_query_len, expected):
+    actual = tuple(
+        inkling_fa4_num_splits(
+            is_local=False,
+            batch_size=batch_size,
+            max_query_len=max_query_len,
+            num_heads=64 // tp,
+            num_kv_heads=8 // tp,
+            max_kv_len=131072,
+        )
+        for tp in (1, 2, 4, 8)
+    )
+    assert actual == expected
+
+
+@pytest.mark.parametrize("tp", [1, 2, 4, 8])
+def test_num_splits_local_is_unsplit(tp):
+    assert (
+        inkling_fa4_num_splits(
+            is_local=True,
+            batch_size=1,
+            max_query_len=1,
+            num_heads=64 // tp,
+            num_kv_heads=16 // tp,
+            max_kv_len=512,
+        )
+        == 1
+    )
+
+
+@pytest.mark.parametrize(
+    ("max_kv_len", "expected"),
+    [(8192, 32), (65536, 64), (1048576, 128)],
+)
+@pytest.mark.parametrize("tp", [4, 8])
+def test_num_splits_long_context_bound(blackwell_platform, tp, max_kv_len, expected):
+    assert (
+        inkling_fa4_num_splits(
+            is_local=False,
+            batch_size=1,
+            max_query_len=1,
+            num_heads=64 // tp,
+            num_kv_heads=8 // tp,
+            max_kv_len=max_kv_len,
+        )
+        == expected
+    )
+
+
+def _ref_rel_attn(
+    q: torch.Tensor,  # [total_q, H, D]
+    key_cache: torch.Tensor,  # [num_blocks, block, Hkv, D]
+    value_cache: torch.Tensor,
+    rel_logits: torch.Tensor,  # [total_q, H, rel_extent]
+    *,
+    q_lens: list[int],
+    kv_lens: list[int],
+    block_table: torch.Tensor,
+    scale: float,
+    rel_extent: int,
+    window_left: int | None,
+) -> torch.Tensor:
+    num_kv_heads = key_cache.shape[2]
+    num_heads = q.shape[1]
+    g = num_heads // num_kv_heads
+    bt = block_table.cpu().numpy()
+    out = torch.empty_like(q)
+
+    start = 0
+    for i, (ql, kl) in enumerate(zip(q_lens, kv_lens)):
+        qi = q[start : start + ql].float()  # [ql, H, D]
+        rl = rel_logits[start : start + ql].float()  # [ql, H, rel_extent]
+
+        nblk = (kl + BLOCK_SIZE - 1) // BLOCK_SIZE
+        blk = bt[i, :nblk]
+        k = key_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float()
+        v = value_cache[blk].reshape(-1, num_kv_heads, HEAD_DIM)[:kl].float()
+        k = k.repeat_interleave(g, dim=1)  # [kl, H, D]
+        v = v.repeat_interleave(g, dim=1)
+
+        # [H, ql, kl]
+        scores = torch.einsum("qhd,khd->hqk", qi, k) * scale
+
+        dev = q.device
+        qpos = torch.arange(ql, device=dev).view(ql, 1) + (kl - ql)  # query pos
+        kpos = torch.arange(kl, device=dev).view(1, kl)
+        dist = qpos - kpos  # [ql, kl] = i - j
+
+        # Relative bias: rel_logits[i, h, dist] when 0 <= dist < rel_extent.
+        in_rng = (dist >= 0) & (dist < rel_extent)  # [ql, kl]
+        idx = dist.clamp(0, rel_extent - 1)
+        # gather per head: bias[h, i, j] = rl[i, h, idx[i, j]]
+        bias = rl.permute(1, 0, 2).gather(  # [H, ql, rel_extent]
+            2, idx.unsqueeze(0).expand(num_heads, -1, -1)
+        )  # [H, ql, kl]
+        bias = torch.where(in_rng.unsqueeze(0), bias, torch.zeros_like(bias))
+        scores = scores + bias
+
+        mask = dist < 0  # causal
+        if window_left is not None:
+            mask = mask | (dist > window_left)
+        scores.masked_fill_(mask.unsqueeze(0), float("-inf"))
+
+        probs = torch.softmax(scores, dim=-1)
+        out[start : start + ql] = torch.einsum("hqk,khd->qhd", probs, v).to(q.dtype)
+        start += ql
+    return out
+
+
+def _run_case(seq_lens, num_heads, num_kv_heads, rel_extent, window_left, seed=0):
+    torch.manual_seed(seed)
+    device = "cuda"
+    q_lens = [s[0] for s in seq_lens]
+    kv_lens = [s[1] for s in seq_lens]
+    total_q = sum(q_lens)
+    num_seqs = len(seq_lens)
+
+    scale = 1.0 / HEAD_DIM
+
+    # q/k are RMS-normed in the model (unit-ish norm); normalize here so the
+    # logit magnitudes are realistic and the bias is not numerically dwarfed.
+    q = torch.randn(total_q, num_heads, HEAD_DIM, device=device, dtype=DTYPE)
+    q = torch.nn.functional.normalize(q.float(), dim=-1).to(DTYPE)
+
+    # Paged KV cache.
+    max_blocks = (max(kv_lens) + BLOCK_SIZE - 1) // BLOCK_SIZE
+    num_blocks = num_seqs * max_blocks + 1
+    key_cache = torch.randn(
+        num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE
+    )
+    key_cache = torch.nn.functional.normalize(key_cache.float(), dim=-1).to(DTYPE)
+    value_cache = torch.randn(
+        num_blocks, BLOCK_SIZE, num_kv_heads, HEAD_DIM, device=device, dtype=DTYPE
+    )
+
+    # Distinct blocks per sequence (block 0 left as a never-referenced pad).
+    block_table = torch.zeros(num_seqs, max_blocks, dtype=torch.int32, device=device)
+    for i in range(num_seqs):
+        block_table[i] = torch.arange(
+            1 + i * max_blocks, 1 + (i + 1) * max_blocks, dtype=torch.int32
+        )
+
+    cu_seqlens_q = torch.tensor(
+        [0, *torch.cumsum(torch.tensor(q_lens), 0).tolist()],
+        dtype=torch.int32,
+        device=device,
+    )
+    cache_seqlens = torch.tensor(kv_lens, dtype=torch.int32, device=device)
+
+    rel_logits = torch.randn(total_q, num_heads, rel_extent, device=device, dtype=DTYPE)
+
+    window_size = (-1, -1) if window_left is None else (window_left, 0)
+
+    preallocated_out = torch.empty_like(q)
+    out = inkling_fa4_rel_attention(
+        q,
+        key_cache,
+        value_cache,
+        block_table=block_table,
+        cache_seqlens=cache_seqlens,
+        cu_seqlens_q=cu_seqlens_q,
+        max_seqlen_q=max(q_lens),
+        softmax_scale=scale,
+        causal=True,
+        window_size=window_size,
+        rel_extent=rel_extent,
+        rel_logits=rel_logits,
+        out=preallocated_out,
+    )
+    assert out.data_ptr() == preallocated_out.data_ptr()
+    out = out.view(total_q, num_heads, HEAD_DIM)
+
+    ref = _ref_rel_attn(
+        q,
+        key_cache,
+        value_cache,
+        rel_logits,
+        q_lens=q_lens,
+        kv_lens=kv_lens,
+        block_table=block_table,
+        scale=scale,
+        rel_extent=rel_extent,
+        window_left=window_left,
+    )
+
+    torch.testing.assert_close(out.float(), ref.float(), atol=2e-2, rtol=2e-2)
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
+@pytest.mark.skipif(
+    _cap is None or _cap.major < 9,
+    reason="FA4 score-mod requires Hopper+ (SM90+)",
+)
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
+@pytest.mark.parametrize(
+    "seq_lens",
+    [
+        [(64, 64)],  # single full prefill
+        [(64, 64), (33, 33), (17, 17)],  # ragged prefill batch
+        [(512, 512)],  # seq_len >> rel_extent (most keys get zero bias)
+        [(300, 300), (512, 512), (129, 129)],  # large ragged batch
+    ],
+)
+@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
+@torch.inference_mode()
+def test_full_attention(seq_lens, num_heads, rel_extent):
+    # rel_extent=128 exercises the out-of-range (zero bias) path; 1024 covers all.
+    # With the 512-token cases and rel_extent=128, query/seq lengths are far
+    # larger than rel_extent so the vast majority of (i, j) pairs are out of
+    # range and must contribute zero bias.
+    _run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
+@pytest.mark.skipif(
+    _cap is None or _cap.major < 9,
+    reason="FA4 score-mod requires Hopper+ (SM90+)",
+)
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
+@pytest.mark.parametrize(
+    "seq_lens",
+    [
+        [(200, 512)],  # chunked prefill: q_len=200 (> rel_extent), 312 cached
+        [(200, 512), (50, 300), (1, 400)],  # mixed chunked + decode
+    ],
+)
+@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
+@torch.inference_mode()
+def test_chunked_prefill(seq_lens, num_heads, rel_extent):
+    # q_len < kv_len with q_len itself larger than rel_extent (for the 128 case):
+    # exercises the seqlen_k - seqlen_q offset together with the out-of-range path.
+    _run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
+@pytest.mark.skipif(
+    _cap is None or _cap.major < 9,
+    reason="FA4 score-mod requires Hopper+ (SM90+)",
+)
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
+@pytest.mark.parametrize(
+    "seq_lens",
+    [
+        [(64, 64), (40, 40)],  # seq_len > window
+        [(512, 512), (300, 300)],  # seq_len/query_len >> window
+        [(1, 512)],  # decode with kv_len >> window
+    ],
+)
+@pytest.mark.parametrize("local_extent", LOCAL_REL_EXTENTS)
+@torch.inference_mode()
+def test_sliding_window(seq_lens, num_heads, local_extent):
+    # Local layers use window_size=(local_extent-1, 0) and rel_extent==local_extent.
+    # With the 512-token cases, query/seq lengths far exceed the window so most
+    # keys are masked out by the sliding window.
+    _run_case(
+        seq_lens,
+        num_heads[0],
+        num_heads[1],
+        rel_extent=local_extent,
+        window_left=local_extent - 1,
+    )
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
+@pytest.mark.skipif(
+    _cap is None or _cap.major < 9,
+    reason="FA4 score-mod requires Hopper+ (SM90+)",
+)
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
+@pytest.mark.parametrize(
+    "seq_lens",
+    [
+        [(1, 50)],
+        [(1, 50), (1, 7), (1, 200)],
+        [(1, 512), (1, 333)],  # kv_len >> rel_extent
+    ],
+)
+@pytest.mark.parametrize("rel_extent", GLOBAL_REL_EXTENTS)
+@torch.inference_mode()
+def test_decode(seq_lens, num_heads, rel_extent):
+    # q_len=1 with kv_len>q_len: the score-mod's seqlen_k - seqlen_q offset path.
+    _run_case(seq_lens, num_heads[0], num_heads[1], rel_extent, window_left=None)

tests/models/inkling/test_fa4_warmup.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.models.inkling.nvidia.ops.fa4_rel_attention import (
+    bucket_max_seqlen_q,
+    inkling_fa4_num_splits,
+)
+from vllm.models.inkling.nvidia.ops.fa4_warmup import (
+    InklingFA4WarmupConfig,
+    _iter_compile_units,
+    _num_warps_bucket,
+)
+
+
+def test_bucket_max_seqlen_q():
+    assert [bucket_max_seqlen_q(n) for n in range(1, 10)] == [
+        1,
+        2,
+        4,
+        4,
+        8,
+        8,
+        8,
+        8,
+        16,
+    ]
+
+
+def test_warmup_enumerates_every_runtime_compile_class():
+    config = InklingFA4WarmupConfig(
+        num_heads=16,
+        num_kv_heads=2,
+        head_dim=128,
+        rel_extent=1024,
+        window_size=(-1, -1),
+        is_local=False,
+        max_kv_len=65536,
+        dtype=torch.bfloat16,
+        kv_dtype=torch.bfloat16,
+        block_size=16,
+        max_num_reqs=64,
+        max_num_batched_tokens=192,
+    )
+    warmed_keys = {unit.key[-1] for unit in _iter_compile_units(config)}
+
+    for query_len in range(1, config.max_num_batched_tokens + 1):
+        max_seqlen_q = bucket_max_seqlen_q(query_len)
+        max_num_reqs = min(
+            config.max_num_reqs,
+            config.max_num_batched_tokens - query_len + 1,
+        )
+        for num_reqs in range(1, max_num_reqs + 1):
+            num_splits = inkling_fa4_num_splits(
+                is_local=config.is_local,
+                batch_size=num_reqs,
+                max_query_len=max_seqlen_q,
+                num_heads=config.num_heads,
+                num_kv_heads=config.num_kv_heads,
+                max_kv_len=config.max_kv_len,
+            )
+            runtime_key = (
+                max_seqlen_q,
+                num_splits,
+                _num_warps_bucket(num_reqs) if num_splits > 1 else None,
+                num_reqs > 1024,
+            )
+            assert runtime_key in warmed_keys

tests/models/inkling/test_mm_towers.py

@@ -0,0 +1,138 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Correctness tests for the fused Inkling vision/audio tower kernels.
+
+Each kernel is checked against a pure-PyTorch reference implementing the
+towers' original op sequence (native ``rms_norm`` semantics: fp32 variance,
+bf16-rounded weight multiply; exact-erf GELU; fp32-accumulated embedding sum).
+The fused kernels keep the same accumulation dtype and rounding points, so
+outputs differ from the reference only by reduction order — a few bf16 ulps.
+"""
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from vllm.platforms import current_platform
+
+if not current_platform.is_cuda():
+    pytest.skip("requires CUDA", allow_module_level=True)
+
+from vllm.models.inkling.common.towers import fold_timespace_to_depth
+from vllm.models.inkling.nvidia.ops.mm_towers import dmel_embed_sum_norm, rmsnorm_gelu
+
+DTYPE = torch.bfloat16
+
+
+def _bf16_spacing(x: torch.Tensor) -> torch.Tensor:
+    return torch.exp2(torch.floor(torch.log2(x.float().abs().clamp(min=1e-30)))) * (
+        2**-7
+    )
+
+
+def _assert_close_ulps(
+    got: torch.Tensor,
+    ref: torch.Tensor,
+    max_ulps: float = 4.0,
+    atol: float = 1e-3,
+    pre_act: torch.Tensor | None = None,
+) -> None:
+    """Fused vs reference differ only by fp32 reduction order, which shows up
+    as a few bf16 ulps at any magnitude — a fixed rtol misrepresents that, so
+    compare against the reference's local ulp spacing. ``pre_act`` (the
+    reference pre-activation) adds derivative-propagated slack: near
+    ``gelu(x) ~ 0`` an ulp-level input flip legitimately moves the output by
+    many of ITS (tiny) ulps, bounded by |gelu'| <= 1.13 times the input ulps."""
+    g, r = got.float(), ref.float()
+    tol = torch.clamp(max_ulps * _bf16_spacing(ref), min=atol)
+    if pre_act is not None:
+        tol = tol + 2.5 * _bf16_spacing(pre_act)
+    bad = (g - r).abs() > tol
+    assert not bad.any(), (
+        f"{int(bad.sum())}/{ref.numel()} elements beyond tolerance; "
+        f"max abs diff {(g - r).abs().max().item():.3e}"
+    )
+
+
+def _ref_rms_norm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
+    # Matches ir.ops.rms_norm: fp32 normalize, bf16 weight multiply.
+    x32 = x.float()
+    var = x32.pow(2).mean(dim=-1, keepdim=True)
+    xn = x32 * torch.rsqrt(var + eps)
+    return (xn.to(weight.dtype) * weight).to(x.dtype)
+
+
+@pytest.mark.parametrize("rows", [1, 7, 1200, 38400])
+@pytest.mark.parametrize("dim", [128, 320, 4800, 6144])
+@pytest.mark.parametrize("gelu", [True, False])
+def test_rmsnorm_gelu(rows: int, dim: int, gelu: bool) -> None:
+    torch.manual_seed(rows * dim)
+    x = torch.randn(rows, dim, device="cuda", dtype=DTYPE)
+    w = torch.randn(dim, device="cuda", dtype=DTYPE)
+
+    h = _ref_rms_norm(x, w, 1e-5)
+    ref = F.gelu(h) if gelu else h
+    got = rmsnorm_gelu(x, w, 1e-5, gelu=gelu)
+
+    _assert_close_ulps(got, ref, pre_act=h if gelu else None)
+
+
+@pytest.mark.parametrize("n", [1, 5, 64])
+@pytest.mark.parametrize(
+    "shape,fold",
+    [
+        ((2, 8, 8, 128), (1, 2)),  # the real L0 -> L1 vision transition
+        ((2, 4, 4, 320), (1, 2)),
+        ((2, 8, 8, 128), (2, 2)),  # temporal + spatial fold
+    ],
+)
+def test_rmsnorm_gelu_folded_store(
+    n: int, shape: tuple[int, ...], fold: tuple[int, int]
+) -> None:
+    torch.manual_seed(n)
+    x = torch.randn(n, *shape, device="cuda", dtype=DTYPE)
+    w = torch.randn(shape[-1], device="cuda", dtype=DTYPE)
+
+    plain = rmsnorm_gelu(x, w, 1e-5, gelu=True)
+    ref = fold_timespace_to_depth(plain, *fold)
+    got = rmsnorm_gelu(x, w, 1e-5, gelu=True, fold=fold)
+
+    # The folded store is a pure permutation of the unfolded output.
+    assert got.shape == ref.shape
+    torch.testing.assert_close(got, ref, rtol=0, atol=0)
+
+
+@pytest.mark.parametrize("num_frames", [1, 3, 100, 4097])
+@pytest.mark.parametrize("with_norm", [True, False])
+def test_dmel_embed_sum_norm(num_frames: int, with_norm: bool) -> None:
+    torch.manual_seed(num_frames)
+    n_bins, vocab, dim = 80, 16, 6144
+    idx = torch.randint(
+        0, vocab, (num_frames, n_bins), device="cuda", dtype=torch.int32
+    )
+    table = torch.randn(n_bins * vocab, dim, device="cuda", dtype=DTYPE)
+    norm_w = torch.randn(dim, device="cuda", dtype=DTYPE)
+
+    flat = (torch.arange(n_bins, device="cuda", dtype=torch.int32) * vocab).unsqueeze(
+        0
+    ) + idx
+    ref = (
+        F.embedding(flat.reshape(-1).long(), table)
+        .reshape(num_frames, n_bins, dim)
+        .sum(dim=1)
+    )
+    if with_norm:
+        ref = _ref_rms_norm(ref, norm_w, 1e-6)
+
+    got = dmel_embed_sum_norm(idx, table, norm_w if with_norm else None, 1e-6)
+    _assert_close_ulps(got, ref)
+
+
+def test_empty_inputs() -> None:
+    x = torch.empty(0, 320, device="cuda", dtype=DTYPE)
+    w = torch.randn(320, device="cuda", dtype=DTYPE)
+    assert rmsnorm_gelu(x, w, 1e-5).shape == (0, 320)
+
+    idx = torch.empty(0, 80, device="cuda", dtype=torch.int32)
+    table = torch.randn(1280, 6144, device="cuda", dtype=DTYPE)
+    assert dmel_embed_sum_norm(idx, table, None, 0.0).shape == (0, 6144)

tests/models/inkling/test_moe_weight_layout.py

@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from vllm.models.inkling.nvidia import moe
+from vllm.platforms import current_platform
+
+
+def test_gate_loads_directly_into_padded_runtime_weight() -> None:
+    gate = moe.InklingGate(
+        d_model=4,
+        n_routed_experts=5,
+        n_shared_experts=2,
+        experts_per_token=2,
+        route_scale=1.0,
+    )
+    loaded = torch.arange(28, dtype=gate.weight.dtype).reshape(7, 4)
+
+    gate.weight.weight_loader(gate.weight, loaded)
+
+    assert gate.weight.shape == (8, 4)
+    torch.testing.assert_close(gate.weight[:7], loaded)
+    torch.testing.assert_close(gate.weight[7], torch.zeros(4))
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
+@pytest.mark.parametrize(("num_tokens", "expected_calls"), [(1, 1), (64, 1), (65, 0)])
+def test_gate_uses_ll_bf16_gemm_through_token_limit(
+    monkeypatch, num_tokens, expected_calls
+) -> None:
+    gate = moe.InklingGate(
+        d_model=8,
+        n_routed_experts=5,
+        n_shared_experts=2,
+        experts_per_token=2,
+        route_scale=1.0,
+    ).to(device="cuda", dtype=torch.bfloat16)
+    hidden_states = torch.randn(num_tokens, 8, device="cuda", dtype=torch.bfloat16)
+    calls = []
+
+    def fake_ll_bf16_gemm(x, weight):
+        calls.append((x, weight))
+        return torch.ones(num_tokens, 8, device="cuda", dtype=torch.float32)
+
+    monkeypatch.setattr(
+        moe.current_platform, "has_device_capability", lambda capability: True
+    )
+    monkeypatch.setattr(moe.ll_bf16, "is_available", lambda: True)
+    monkeypatch.setattr(moe.ll_bf16, "ll_bf16_gemm", fake_ll_bf16_gemm)
+
+    logits = gate.compute_logits(hidden_states)
+
+    assert len(calls) == expected_calls
+    if calls:
+        assert calls[0][0] is hidden_states
+        assert calls[0][1] is gate.weight
+    assert logits.shape == (num_tokens, 8)
+    assert logits.dtype == torch.float32
+
+
+@pytest.mark.parametrize(("projection", "amax"), [("w13", 4.375), ("w2", 2960.0)])
+def test_moe_loads_calibrated_input_scale(projection: str, amax: float) -> None:
+    experts = SimpleNamespace(
+        w13_input_scale=torch.nn.Parameter(torch.empty(3, 2)),
+        w2_input_scale=torch.nn.Parameter(torch.empty(3)),
+    )
+    layer = SimpleNamespace(experts=SimpleNamespace(routed_experts=experts))
+
+    loaded = moe.InklingMoE.load_expert_weight(
+        layer,
+        f"experts.{projection}_weight.input_amax",
+        torch.tensor([amax]),
+    )
+
+    scale = getattr(experts, f"{projection}_input_scale")
+    expected = torch.full_like(scale, amax / (448.0 * 6.0))
+    torch.testing.assert_close(scale, expected)
+    assert loaded == [f"experts.routed_experts.{projection}_input_scale"]
+
+
+def test_sink_down_projection_is_packed_during_load(monkeypatch) -> None:
+    monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
+    monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
+    sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8)
+    loaded = torch.arange(48, dtype=sink.w2_weight.dtype).reshape(2, 3, 8)
+
+    sink.load_weight("w2_weight", loaded)
+
+    expected = loaded[:, :, 4:].permute(1, 0, 2).reshape(3, 8)
+    torch.testing.assert_close(sink.w2_weight, expected)
+
+
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="requires CUDA")
+def test_sink_packed_weight_forward_matches_expert_sum(monkeypatch) -> None:
+    monkeypatch.setattr(moe, "get_tensor_model_parallel_world_size", lambda: 2)
+    monkeypatch.setattr(moe, "get_tensor_model_parallel_rank", lambda: 1)
+    sink = moe.InklingSinkExperts(n_experts=2, d_model=3, d_mlp=8).to(
+        device="cuda", dtype=torch.bfloat16
+    )
+    torch.manual_seed(1)
+    w13 = torch.randn(2, 16, 3, dtype=torch.bfloat16)
+    w2 = torch.randn(2, 3, 8, dtype=torch.bfloat16)
+    sink.load_weight("w13_weight", w13)
+    sink.load_weight("w2_weight", w2)
+    x = torch.randn(5, 3, device="cuda", dtype=torch.bfloat16)
+    gammas = torch.randn(5, 2, device="cuda")
+
+    output = sink(x, gammas)
+
+    raw = torch.einsum("td,efd->tef", x, w13[:, 8:].to("cuda"))
+    hidden = torch.nn.functional.silu(raw[:, :, 0::2].float())
+    hidden = (hidden * raw[:, :, 1::2] * gammas[:, :, None]).to(torch.bfloat16)
+    expected = torch.einsum("tef,edf->td", hidden, w2[:, :, 4:].to("cuda"))
+    torch.testing.assert_close(output, expected, rtol=0, atol=0)

tests/models/inkling/test_qkvr_prep.py

@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+import vllm.models.inkling.nvidia.ops.sconv as sconv
+from vllm.models.inkling.nvidia.ops import qkvr_prep
+from vllm.platforms import current_platform
+
+_cap = current_platform.get_device_capability() if current_platform.is_cuda() else None
+
+
+def _make_inputs(*, is_local: bool, tokens: int = 33, tp_size: int = 4):
+    torch.manual_seed(0)
+    heads = 64 // tp_size
+    kv_heads = (16 if is_local else 8) // tp_size
+    head_dim = 128
+    d_rel = 16
+    rel_extent = 512 if is_local else 1024
+    page_size = 16
+    num_blocks = (tokens + page_size - 1) // page_size
+    q_width = heads * head_dim
+    kv_width = kv_heads * head_dim
+    r_width = heads * d_rel
+    device = "cuda"
+
+    qkvr = torch.randn(
+        tokens,
+        q_width + 2 * kv_width + r_width,
+        device=device,
+        dtype=torch.bfloat16,
+    )
+    k_weight = torch.randn(kv_width, 4, device=device, dtype=torch.bfloat16)
+    v_weight = torch.randn_like(k_weight)
+    q_norm_weight = torch.randn(head_dim, device=device, dtype=torch.bfloat16)
+    k_norm_weight = torch.randn_like(q_norm_weight)
+    rel_proj = torch.randn(d_rel, rel_extent, device=device, dtype=torch.bfloat16)
+    conv_cache = torch.zeros(
+        num_blocks,
+        kv_heads,
+        page_size,
+        2 * head_dim,
+        device=device,
+        dtype=torch.bfloat16,
+    )
+    key_cache = torch.empty(
+        num_blocks,
+        page_size,
+        kv_heads,
+        head_dim,
+        device=device,
+        dtype=torch.bfloat16,
+    )
+    value_cache = torch.empty_like(key_cache)
+    positions = torch.arange(tokens, device=device, dtype=torch.int64)
+    block_table = torch.arange(num_blocks, device=device, dtype=torch.int32)[None]
+    seq_idx = torch.zeros(tokens, device=device, dtype=torch.int32)
+    slots = torch.arange(tokens, device=device, dtype=torch.int64)
+    query_start = torch.zeros(tokens, device=device, dtype=torch.int32)
+    log_scaling_n_floor = None if is_local else 128000
+    log_scaling = None
+    if log_scaling_n_floor is not None:
+        log_scaling = torch.linspace(
+            1.0,
+            1.1,
+            tokens,
+            device=device,
+            dtype=torch.float32,
+        )
+    return (
+        qkvr,
+        k_weight,
+        v_weight,
+        q_norm_weight,
+        k_norm_weight,
+        rel_proj,
+        1e-6,
+        heads,
+        kv_heads,
+        head_dim,
+        d_rel,
+        conv_cache,
+        key_cache,
+        value_cache,
+        positions,
+        block_table,
+        seq_idx,
+        slots,
+        query_start,
+        slots,
+        0,
+        head_dim,
+        page_size,
+        log_scaling,
+    )
+
+
+def _reference(args):
+    qkvr = args[0]
+    heads, kv_heads, head_dim, d_rel = args[7:11]
+    q_width = heads * head_dim
+    kv_width = kv_heads * head_dim
+    q, k, v, r = qkvr.split((q_width, kv_width, kv_width, heads * d_rel), dim=1)
+    conv_cache = args[11].clone()
+    k = sconv.fused_sconv(
+        k.contiguous(),
+        args[1],
+        conv_cache,
+        args[14],
+        args[15],
+        args[16],
+        args[17],
+        args[18],
+        args[20],
+        head_dim,
+        args[22],
+    )
+    v = sconv.fused_sconv(
+        v.contiguous(),
+        args[2],
+        conv_cache,
+        args[14],
+        args[15],
+        args[16],
+        args[17],
+        args[18],
+        args[21],
+        head_dim,
+        args[22],
+    )
+
+    def rms_norm(x, weight):
+        x = x.reshape(-1, head_dim).float()
+        rstd = torch.rsqrt(x.square().mean(1, keepdim=True) + args[6])
+        return (x * rstd * weight.float()).to(qkvr.dtype)
+
+    q = rms_norm(q, args[3]).view(qkvr.shape[0], heads, head_dim)
+    k = rms_norm(k, args[4]).view(qkvr.shape[0], kv_heads, head_dim)
+    v = v.view(qkvr.shape[0], kv_heads, head_dim)
+    rel = torch.mm(r.reshape(-1, d_rel), args[5]).view(qkvr.shape[0], heads, -1)
+    if args[23] is not None:
+        q = (q.float() * args[23][:, None, None]).to(q.dtype)
+        rel = (rel.float() * args[23][:, None, None]).to(rel.dtype)
+
+    key_cache = args[12].clone()
+    value_cache = args[13].clone()
+    slots = args[19]
+    valid = slots >= 0
+    key_cache.view(-1, kv_heads, head_dim)[slots[valid]] = k[valid]
+    value_cache.view(-1, kv_heads, head_dim)[slots[valid]] = v[valid]
+    return q.flatten(1), rel, key_cache, value_cache
+
+
+@pytest.mark.skipif(
+    _cap is None,
+    reason="Inkling QKVR prep kernels require CUDA",
+)
+@pytest.mark.parametrize(
+    ("is_local", "tokens", "tp_size"),
+    [
+        (True, 9, 4),
+        (False, 33, 4),
+        (True, 33, 8),
+        (True, 128, 4),
+        (False, 128, 8),
+        (True, 512, 4),
+        (False, 640, 8),
+    ],
+)
+@torch.inference_mode()
+def test_qkvr_prep_matches_reference(is_local, tokens, tp_size):
+    args = _make_inputs(is_local=is_local, tokens=tokens, tp_size=tp_size)
+    ref_q, ref_rel, ref_key, ref_value = _reference(args)
+
+    q, rel = qkvr_prep.fused_qkvr_prep(*args)
+
+    torch.testing.assert_close(q, ref_q, rtol=0.01, atol=0.02)
+    torch.testing.assert_close(rel, ref_rel, rtol=0.02, atol=0.125)
+    torch.testing.assert_close(args[12], ref_key, rtol=0.01, atol=0.01)
+    torch.testing.assert_close(args[13], ref_value, rtol=0, atol=0)
+
+
+@pytest.mark.skipif(
+    _cap is None,
+    reason="Inkling QKVR prep kernels require CUDA",
+)
+@pytest.mark.parametrize("tokens", [9, 128])
+@torch.inference_mode()
+def test_qkvr_log_scaling_preserves_bf16_norm_output(tokens):
+    args = list(_make_inputs(is_local=False, tokens=tokens, tp_size=8))
+    tau = args[23]
+    assert tau is not None
+
+    scaled_q, scaled_rel = qkvr_prep.fused_qkvr_prep(*args)
+    args[23] = None
+    unscaled_q, unscaled_rel = qkvr_prep.fused_qkvr_prep(*args)
+
+    expected_q = (unscaled_q.float() * tau[:, None]).to(unscaled_q.dtype)
+    expected_rel = (unscaled_rel.float() * tau[:, None, None]).to(unscaled_rel.dtype)
+    torch.testing.assert_close(scaled_q, expected_q, rtol=0, atol=0)
+    torch.testing.assert_close(scaled_rel, expected_rel, rtol=0, atol=0)
+
+
+@pytest.mark.skipif(
+    _cap is None,
+    reason="Inkling QKVR prep kernels require CUDA",
+)
+@pytest.mark.parametrize("tokens", [9, 128])
+@torch.inference_mode()
+def test_qkvr_prep_negative_slots_skip_cache_writes(tokens):
+    args = list(_make_inputs(is_local=True, tokens=tokens))
+    args[17].fill_(-1)
+    args[19].fill_(-1)
+    conv_cache = args[11].clone()
+    key_cache = args[12].clone()
+    value_cache = args[13].clone()
+
+    qkvr_prep.fused_qkvr_prep(*args)
+
+    torch.testing.assert_close(args[11], conv_cache, rtol=0, atol=0)
+    torch.testing.assert_close(args[12], key_cache, rtol=0, atol=0)
+    torch.testing.assert_close(args[13], value_cache, rtol=0, atol=0)
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+@torch.inference_mode()
+def test_fused_sconv_negative_slots_skip_cache_writes():
+    args = list(_make_inputs(is_local=True, tokens=9))
+    args[17].fill_(-1)
+    cache = args[11].clone()
+
+    sconv.fused_sconv(
+        args[0][:, 16 * 128 : 16 * 128 + 4 * 128].contiguous(),
+        args[1],
+        args[11],
+        args[14],
+        args[15],
+        args[16],
+        args[17],
+        args[18],
+        0,
+        128,
+        args[22],
+    )
+
+    torch.testing.assert_close(args[11], cache, rtol=0, atol=0)
+
+
+@pytest.mark.parametrize(
+    ("rel_extent", "last_latency_rows", "first_throughput_rows"),
+    [(512, 8191, 8192), (1024, 2047, 2048)],
+)
+def test_rel_projection_schedule_crossover(
+    rel_extent, last_latency_rows, first_throughput_rows
+):
+    assert not qkvr_prep.use_rel_proj_throughput(last_latency_rows, rel_extent)
+    assert qkvr_prep.use_rel_proj_throughput(first_throughput_rows, rel_extent)

tests/models/inkling/test_sconv_metadata.py

@@ -0,0 +1,119 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Parity test for the fused sconv seq-metadata kernel.
+
+``sconv_seq_metadata`` fills the per-token ``seq_idx`` (owning request) and
+``query_start`` (first x-row of that request) buffers in a single launch. Actual
+tokens must match the searchsorted-based reference, while CUDA graph padding
+must be initialized to safe zero values.
+"""
+
+import pytest
+import torch
+
+from vllm.models.inkling.nvidia.ops.sconv import sconv_seq_metadata
+from vllm.models.inkling.nvidia.sconv_swa_attn import InklingSconvMetadataBuilder
+
+CASES = [
+    # (query_lens, extra_pad_tokens)
+    ([1], 0),  # bsz1 decode
+    ([1] * 8, 0),  # uniform decode
+    ([1] * 8, 3),  # uniform decode, padded tokens past the last request
+    ([2] * 4, 0),  # uniform spec-decode
+    ([2048], 0),  # single prefill
+    ([517, 1, 1, 33, 1, 128], 0),  # mixed prefill/decode
+    ([517, 1, 1, 33, 1, 128], 5),  # mixed, padded
+    ([1] * 500, 0),  # many requests (deep binary search)
+]
+
+
+def _ref(query_start_loc: torch.Tensor, num_reqs: int, num_tokens: int):
+    cu_seqlens = query_start_loc[: num_reqs + 1].to(torch.int64)
+    token_idx = torch.arange(num_tokens, device=cu_seqlens.device, dtype=torch.int64)
+    seq_idx = (torch.searchsorted(cu_seqlens, token_idx, right=True) - 1).clamp(
+        max=num_reqs - 1
+    )
+    return seq_idx.to(torch.int32), cu_seqlens[seq_idx].to(torch.int32)
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+@pytest.mark.parametrize("query_lens,extra_pad", CASES)
+def test_sconv_seq_metadata_matches_searchsorted(query_lens, extra_pad):
+    device = "cuda"
+    num_reqs = len(query_lens)
+    query_start_loc = torch.tensor(
+        [0] + list(torch.tensor(query_lens).cumsum(0)), dtype=torch.int32
+    ).to(device)
+    num_actual_tokens = int(query_start_loc[-1])
+    num_padded_tokens = num_actual_tokens + extra_pad
+
+    ref_seq, ref_qs = _ref(query_start_loc, num_reqs, num_actual_tokens)
+
+    seq_idx = torch.full((num_padded_tokens,), -1, dtype=torch.int32, device=device)
+    query_start = torch.full_like(seq_idx, -1)
+    sconv_seq_metadata(
+        query_start_loc,
+        num_reqs,
+        num_actual_tokens,
+        seq_idx,
+        query_start,
+        num_padded_tokens,
+    )
+
+    torch.testing.assert_close(seq_idx[:num_actual_tokens], ref_seq, rtol=0, atol=0)
+    torch.testing.assert_close(query_start[:num_actual_tokens], ref_qs, rtol=0, atol=0)
+    assert torch.count_nonzero(seq_idx[num_actual_tokens:]) == 0
+    assert torch.count_nonzero(query_start[num_actual_tokens:]) == 0
+
+
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
+def test_sconv_metadata_reuses_padded_static_buffers():
+    device = torch.device("cuda")
+    builder = object.__new__(InklingSconvMetadataBuilder)
+    builder.seq_idx_buffer = torch.empty(8, dtype=torch.int32, device=device)
+    builder.query_start_buffer = torch.empty(8, dtype=torch.int32, device=device)
+
+    class CommonMetadata:
+        num_reqs = 1
+        num_actual_tokens = 8
+        query_start_loc = torch.tensor([0, 3], dtype=torch.int32, device=device)
+        query_start_loc_cpu = torch.tensor([0, 3], dtype=torch.int32)
+        block_table_tensor = torch.zeros((1, 1), dtype=torch.int32, device=device)
+        slot_mapping = torch.tensor(
+            [0, 1, 2, -1, -1, -1, -1, -1],
+            dtype=torch.int64,
+            device=device,
+        )
+
+    common = CommonMetadata()
+    first = builder.build(0, common)
+    pointers = tuple(
+        tensor.data_ptr()
+        for tensor in (
+            first.block_table,
+            first.slot_mapping,
+            first.seq_idx,
+            first.query_start,
+        )
+    )
+
+    common.query_start_loc[1] = 5
+    common.query_start_loc_cpu[1] = 5
+    common.slot_mapping[:5] = torch.arange(5, dtype=torch.int64, device=device)
+    second = builder.build(0, common)
+
+    assert second.slot_mapping.shape == (8,)
+    assert second.seq_idx.shape == (8,)
+    assert second.query_start.shape == (8,)
+    assert pointers == tuple(
+        tensor.data_ptr()
+        for tensor in (
+            second.block_table,
+            second.slot_mapping,
+            second.seq_idx,
+            second.query_start,
+        )
+    )
+    assert torch.count_nonzero(second.seq_idx[5:]) == 0
+    assert torch.count_nonzero(second.query_start[5:]) == 0
+    assert torch.all(second.slot_mapping[5:] == -1)

tests/models/registry.py

@@ -330,6 +330,12 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
         "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B",
         min_transformers_version="5.9.0",
     ),
+    "InklingForCausalLM": _HfExamplesInfo(
+        "thinkingmachines/Inkling-NVFP4",
+        tokenizer_mode="inkling",
+        trust_remote_code=True,
+        max_model_len=4096,
+    ),
     "InternLM2ForCausalLM": _HfExamplesInfo(
         "internlm/internlm2-chat-7b", trust_remote_code=True
     ),
@@ -956,6 +962,12 @@ _MULTIMODAL_EXAMPLE_MODELS = {
         "HuggingFaceM4/Idefics3-8B-Llama3",
         extras={"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"},
     ),
+    "InklingForConditionalGeneration": _HfExamplesInfo(
+        "thinkingmachines/Inkling-NVFP4",
+        tokenizer_mode="inkling",
+        trust_remote_code=True,
+        max_model_len=4096,
+    ),
     "IsaacForConditionalGeneration": _HfExamplesInfo(
         "PerceptronAI/Isaac-0.1",
         trust_remote_code=True,

tests/parser/engine/test_delegating_replay.py

@@ -88,6 +88,11 @@ def _discover_pairings() -> list[_PairingInfo]:
         if cfg.name not in _BUILDERS:
             missing_builders.append(f"{engine_cls.__name__} (config.name={cfg.name!r})")
             continue
+        if cfg.name == "inkling":
+            # Inkling uses typed structural blocks and opts out of token-id
+            # terminal matching; combined-parser replay coverage lives in
+            # test_inkling.py.
+            continue
 
         parser_cls = type(
             f"_Delegating{engine_cls.__name__}",

tests/parser/engine/test_inkling.py

@@ -0,0 +1,506 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for the engine-based Inkling parser.
+
+Inkling output is a sequence of typed content blocks delimited by dedicated
+special tokens; the tool-call payload is ``{"name":...,"args":{...}}``
+between ``<|content_invoke_tool_json|>`` and ``<|end_message|>``. The
+cases mirror the Rust unified parser's tests
+(``rust/src/parser/src/unified/inkling.rs``) where applicable.
+"""
+
+import json
+
+import pytest
+
+from tests.parser.engine.conftest import make_mock_tokenizer
+from tests.parser.engine.streaming_helpers import (
+    collect_content,
+    collect_function_name,
+    collect_tool_arguments,
+)
+from vllm.parser.engine.parser_engine_config import ParserState
+from vllm.parser.inkling import InklingParser, _inkling_arg_converter
+from vllm.parser.parser_manager import ParserManager
+
+MSG_MODEL = "<|message_model|>"
+TEXT_START = "<|content_text|>"
+THINK_START = "<|content_thinking|>"
+TOOL_JSON = "<|content_invoke_tool_json|>"
+TOOL_TEXT = "<|content_invoke_tool_text|>"
+TOOL_ERROR = "<|content_tool_error|>"
+END_MESSAGE = "<|end_message|>"
+END_SAMPLING = "<|content_model_end_sampling|>"
+
+_TML_VOCAB = {
+    MSG_MODEL: 200001,
+    TEXT_START: 200004,
+    END_SAMPLING: 200006,
+    THINK_START: 200008,
+    END_MESSAGE: 200010,
+    TOOL_ERROR: 200022,
+    TOOL_JSON: 200049,
+    TOOL_TEXT: 200057,
+}
+
+
+@pytest.fixture
+def mock_tokenizer():
+    return make_mock_tokenizer(_TML_VOCAB)
+
+
+@pytest.fixture
+def parser(mock_tokenizer):
+    return InklingParser(mock_tokenizer)
+
+
+def _tool_block(name: str, args: str) -> str:
+    return f'{TOOL_JSON}{{"name":"{name}","args":{args}}}{END_MESSAGE}'
+
+
+_MARKERS = sorted(_TML_VOCAB, key=len, reverse=True)
+
+
+def _tokenize(text: str) -> list[tuple[int, str]]:
+    """Tokenize like the real stream: markers are atomic special tokens,
+    plain text becomes one token per character (matching the mock
+    tokenizer's ``chr``-based decode)."""
+    tokens: list[tuple[int, str]] = []
+    i = 0
+    while i < len(text):
+        for marker in _MARKERS:
+            if text.startswith(marker, i):
+                tokens.append((_TML_VOCAB[marker], marker))
+                i += len(marker)
+                break
+        else:
+            tokens.append((ord(text[i]), text[i]))
+            i += 1
+    return tokens
+
+
+def _stream(parser, request, text: str, chunk_size: int):
+    """Stream production-shaped deltas: ``chunk_size`` tokens per delta,
+    with delta_token_ids covering every token (specials and text)."""
+    tokens = _tokenize(text)
+    results = []
+    previous_text = ""
+    previous_token_ids: list[int] = []
+    for start in range(0, len(tokens), chunk_size):
+        batch = tokens[start : start + chunk_size]
+        delta_text = "".join(t for _, t in batch)
+        delta_token_ids = [tid for tid, _ in batch]
+        current_text = previous_text + delta_text
+        current_token_ids = previous_token_ids + delta_token_ids
+        delta = parser.extract_tool_calls_streaming(
+            previous_text=previous_text,
+            current_text=current_text,
+            delta_text=delta_text,
+            previous_token_ids=tuple(previous_token_ids),
+            current_token_ids=tuple(current_token_ids),
+            delta_token_ids=tuple(delta_token_ids),
+            request=request,
+        )
+        results.append((delta, current_text))
+        previous_text = current_text
+        previous_token_ids = current_token_ids
+    finish = parser.finish_streaming()
+    if finish is not None:
+        results.append((finish, text))
+    return results
+
+
+def _stream_text_only(parser, request, text: str, chunk_size: int):
+    """Stream text-only deltas (no token ids), chunked at arbitrary
+    character boundaries — exercises the text-lexing fallback path,
+    including markers split across chunks."""
+    results = []
+    previous_text = ""
+    for start in range(0, len(text), chunk_size):
+        delta_text = text[start : start + chunk_size]
+        current_text = previous_text + delta_text
+        delta = parser.extract_tool_calls_streaming(
+            previous_text=previous_text,
+            current_text=current_text,
+            delta_text=delta_text,
+            previous_token_ids=(),
+            current_token_ids=(),
+            delta_token_ids=(),
+            request=request,
+        )
+        results.append((delta, current_text))
+        previous_text = current_text
+    finish = parser.finish_streaming()
+    if finish is not None:
+        results.append((finish, text))
+    return results
+
+
+def _collect_reasoning(results) -> str:
+    return "".join(d.reasoning for d, _ in results if d and d.reasoning)
+
+
+class TestArgConverter:
+    def test_complete_wrapper(self):
+        raw = '{"name":"get_weather","args":{"city":"SF"}}'
+        assert _inkling_arg_converter(raw, False) == '{"city":"SF"}'
+
+    def test_partial_before_args(self):
+        assert _inkling_arg_converter('{"name":"get_w', True) == ""
+
+    def test_partial_inside_args(self):
+        raw = '{"name":"x","args":{"a":1'
+        assert _inkling_arg_converter(raw, True) == '{"a":1'
+
+    def test_prefix_stability(self):
+        full = '{"name":"x","args":{"a":{"b":[1,2]},"c":"d"}}'
+        prev = ""
+        for end in range(len(full)):
+            out = _inkling_arg_converter(full[:end], True)
+            assert out.startswith(prev) or prev.startswith(out) or not prev
+            if out.startswith(prev):
+                prev = out
+
+    def test_args_value_appearing_in_name(self):
+        raw = '{"name":"args","args":{"k":1}}'
+        assert _inkling_arg_converter(raw, False) == '{"k":1}'
+
+    def test_whitespace_tolerated(self):
+        raw = '{ "name" : "x" , "args" : {"a": 1} }'
+        assert _inkling_arg_converter(raw, False) == '{"a": 1}'
+
+    def test_missing_args_defaults_empty(self):
+        assert _inkling_arg_converter('{"name":"x"}', False) == "{}"
+
+    def test_non_object_args_rejected(self):
+        with pytest.raises(ValueError, match="JSON object"):
+            _inkling_arg_converter('{"name":"x","args":[1]}', False)
+
+
+class TestNonStreaming:
+    def test_plain_text(self, parser, mock_request):
+        reasoning, content, tools = parser.parse(
+            f"{TEXT_START}hello world{END_MESSAGE}", mock_request
+        )
+        assert reasoning is None
+        assert content == "hello world"
+        assert tools is None
+
+    def test_reasoning_text_tool(self, parser, mock_request):
+        text = (
+            f"{THINK_START}I should check the weather.{END_MESSAGE}"
+            f"{MSG_MODEL}{TEXT_START}Let me check.{END_MESSAGE}"
+            f"{MSG_MODEL}" + _tool_block("get_weather", '{"city":"SF"}')
+        )
+        reasoning, content, tools = parser.parse(text, mock_request)
+        assert reasoning == "I should check the weather."
+        assert content == "Let me check."
+        assert [t.name for t in tools] == ["get_weather"]
+        assert json.loads(tools[0].arguments) == {"city": "SF"}
+
+    def test_tool_header_name_is_not_visible_content(self, parser, mock_request):
+        text = "get_weather" + _tool_block("get_weather", '{"city":"SF"}')
+        _, content, tools = parser.parse(text, mock_request)
+        assert content is None
+        assert [tool.name for tool in tools] == ["get_weather"]
+
+    def test_parallel_tool_calls(self, parser, mock_request):
+        text = _tool_block("a", "{}") + MSG_MODEL + _tool_block("b", '{"x":[1,2]}')
+        _, _, tools = parser.parse(text, mock_request)
+        assert [t.name for t in tools] == ["a", "b"]
+        assert json.loads(tools[0].arguments) == {}
+        assert json.loads(tools[1].arguments) == {"x": [1, 2]}
+
+    def test_nested_args(self, parser, mock_request):
+        args = '{"q":{"deep":{"list":[{"k":"v"}]}},"s":"a}b"}'
+        _, _, tools = parser.parse(_tool_block("f", args), mock_request)
+        assert json.loads(tools[0].arguments) == json.loads(args)
+
+    def test_invoke_tool_text_is_visible_text(self, parser, mock_request):
+        reasoning, content, tools = parser.parse(
+            f"{TOOL_TEXT}do something{END_MESSAGE}", mock_request
+        )
+        assert content == "do something"
+        assert tools is None
+
+    def test_tool_error_is_visible_text(self, parser, mock_request):
+        _, content, tools = parser.parse(f"{TOOL_ERROR}boom{END_MESSAGE}", mock_request)
+        assert content == "boom"
+        assert tools is None
+
+    def test_end_sampling_closes_blocks(self, parser, mock_request):
+        reasoning, content, _ = parser.parse(
+            f"{THINK_START}hm{END_MESSAGE}{MSG_MODEL}{TEXT_START}hi{END_SAMPLING}",
+            mock_request,
+        )
+        assert reasoning == "hm"
+        assert content == "hi"
+
+    def test_multiple_reasoning_blocks_concatenate(self, parser, mock_request):
+        text = (
+            f"{THINK_START}one{END_MESSAGE}"
+            f"{MSG_MODEL}{TEXT_START}mid{END_MESSAGE}"
+            f"{MSG_MODEL}{THINK_START}two{END_MESSAGE}"
+        )
+        reasoning, content, _ = parser.parse(text, mock_request)
+        assert reasoning == "onetwo"
+        assert content == "mid"
+
+    def test_text_after_tool_call(self, parser, mock_request):
+        text = _tool_block("f", "{}") + f"{MSG_MODEL}{TEXT_START}done{END_MESSAGE}"
+        _, content, tools = parser.parse(text, mock_request)
+        assert [t.name for t in tools] == ["f"]
+        assert content == "done"
+
+    def test_incomplete_tool_call_at_eos(self, parser, mock_request):
+        # Engine convention: best-effort with what arrived. (The Rust
+        # parser instead errors with "incomplete Inkling tool call".)
+        _, _, tools = parser.parse(
+            f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"', mock_request
+        )
+        assert [t.name for t in tools] == ["d"]
+
+    def test_prose_marker_without_token_ids_is_structural(self, parser, mock_request):
+        # Inkling opts into text-lexer terminal recognition so held-back
+        # structural marker text from the detokenizer is still parsed.
+        _, content, _ = parser.parse(
+            f"{TEXT_START}see {TEXT_START} token{END_MESSAGE}", mock_request
+        )
+        assert content == "see  token"
+
+
+class TestStreaming:
+    @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64, 4096])
+    def test_chunk_invariance_tool_call(self, mock_tokenizer, mock_request, chunk_size):
+        parser = InklingParser(mock_tokenizer)
+        text = f"{TEXT_START}Check this.{END_MESSAGE}{MSG_MODEL}" + _tool_block(
+            "get_weather", '{"city":"San Francisco"}'
+        )
+        results = _stream(parser, mock_request, text, chunk_size)
+        assert collect_content(results) == "Check this."
+        assert collect_function_name(results) == "get_weather"
+        assert json.loads(collect_tool_arguments(results)) == {"city": "San Francisco"}
+
+    @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
+    def test_chunk_invariance_tool_call_text_only(
+        self, mock_tokenizer, mock_request, chunk_size
+    ):
+        # Same case through the text-lexing fallback (no token ids),
+        # with markers split at arbitrary character boundaries.
+        parser = InklingParser(mock_tokenizer)
+        text = f"{TEXT_START}Check this.{END_MESSAGE}{MSG_MODEL}" + _tool_block(
+            "get_weather", '{"city":"San Francisco"}'
+        )
+        results = _stream_text_only(parser, mock_request, text, chunk_size)
+        assert collect_content(results) == "Check this."
+        assert collect_function_name(results) == "get_weather"
+        assert json.loads(collect_tool_arguments(results)) == {"city": "San Francisco"}
+
+    @pytest.mark.parametrize("chunk_size", [1, 5, 11])
+    def test_chunk_invariance_reasoning(self, mock_tokenizer, mock_request, chunk_size):
+        parser = InklingParser(mock_tokenizer)
+        text = (
+            f"{THINK_START}thinking...{END_MESSAGE}"
+            f"{MSG_MODEL}{TEXT_START}answer{END_MESSAGE}"
+        )
+        results = _stream(parser, mock_request, text, chunk_size)
+        assert _collect_reasoning(results) == "thinking..."
+        assert collect_content(results) == "answer"
+
+    def test_split_marker_held_across_chunks(self, parser, mock_request):
+        # Mirrors Rust `inkling_streaming_holds_split_markers`.
+        text = f"{TEXT_START}hello{END_MESSAGE}"
+        results = _stream_text_only(parser, mock_request, text, 9)
+        assert collect_content(results) == "hello"
+
+    def test_name_streams_before_args_complete(self, parser, mock_request):
+        # Feed only up to the name's closing quote — the name delta must
+        # already be emitted before any args arrive.
+        prefix = f'{TOOL_JSON}{{"name":"get_weather",'
+        results = _stream(parser, mock_request, prefix, 4096)
+        assert collect_function_name(results) == "get_weather"
+
+    def test_combined_parser_reasoning_to_tool_handoff_uses_text_markers(
+        self, mock_tokenizer, mock_request
+    ):
+        parser_cls = ParserManager.get_parser(
+            tool_parser_name="inkling",
+            reasoning_parser_name="inkling",
+            enable_auto_tools=True,
+        )
+        parser = parser_cls(mock_tokenizer, [])
+
+        first = parser.parse_delta(
+            THINK_START,
+            [_TML_VOCAB[THINK_START]],
+            mock_request,
+            prompt_token_ids=[_TML_VOCAB[MSG_MODEL]],
+            finished=False,
+        )
+        assert first is None
+
+        second = parser.parse_delta(
+            "thinking",
+            [ord(c) for c in "thinking"],
+            mock_request,
+            finished=False,
+        )
+        assert second is not None
+        assert second.reasoning == "thinking"
+
+        # Mirrors the DelegatingParser handoff after reasoning closes: the
+        # tool pass receives reconstructed text that starts at the Inkling
+        # tool marker, while the token-id slice has already moved past it.
+        body = (
+            "get_weather"
+            f'{TOOL_JSON}{{"name":"get_weather","args":{{"city":"Seattle"}}}}'
+            f"{END_MESSAGE}"
+        )
+        third = parser.parse_delta(
+            body,
+            [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[END_SAMPLING]],
+            mock_request,
+            finished=True,
+        )
+        assert third is not None
+        assert third.tool_calls
+        assert third.tool_calls[0].function.name == "get_weather"
+        assert third.tool_calls[0].function.arguments == '{"city":"Seattle"}'
+        assert TOOL_JSON not in ((third.content or "") + (third.reasoning or ""))
+
+    def test_streamed_args_are_object_only(self, parser, mock_request):
+        # The streamed `arguments` must be the bare args object, never
+        # the `{"name":...}` wrapper.
+        text = _tool_block("f", '{"a":1}')
+        results = _stream(parser, mock_request, text, 3)
+        args = collect_tool_arguments(results)
+        assert json.loads(args) == {"a": 1}
+        assert "name" not in args
+
+    @pytest.mark.parametrize("chunk_size", [1, 9])
+    def test_parallel_calls_streaming(self, mock_tokenizer, mock_request, chunk_size):
+        parser = InklingParser(mock_tokenizer)
+        text = _tool_block("a", '{"i":1}') + MSG_MODEL + _tool_block("b", '{"i":2}')
+        results = _stream(parser, mock_request, text, chunk_size)
+        indexed: dict[int, dict[str, str]] = {}
+        for delta, _ in results:
+            if not (delta and delta.tool_calls):
+                continue
+            for tc in delta.tool_calls:
+                slot = indexed.setdefault(tc.index, {"name": "", "args": ""})
+                if tc.function and tc.function.name:
+                    slot["name"] = tc.function.name
+                if tc.function and tc.function.arguments:
+                    slot["args"] += tc.function.arguments
+        assert indexed[0]["name"] == "a"
+        assert indexed[1]["name"] == "b"
+        assert json.loads(indexed[0]["args"]) == {"i": 1}
+        assert json.loads(indexed[1]["args"]) == {"i": 2}
+
+
+class TestPromptSeededState:
+    def test_prompt_ending_in_thinking_starts_reasoning(self, parser, mock_request):
+        parser.adjust_initial_state_from_prompt([200001, _TML_VOCAB[THINK_START]])
+        assert parser._engine.state == ParserState.REASONING
+
+    def test_prompt_ending_in_text_starts_content(self, parser):
+        parser.adjust_initial_state_from_prompt([200001, _TML_VOCAB[TEXT_START]])
+        assert parser._engine.state == ParserState.CONTENT
+
+    def test_generation_prompt_tail_starts_message_header(self, parser):
+        parser.adjust_initial_state_from_prompt(
+            [_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]]
+        )
+        assert parser._engine.state == ParserState.MESSAGE_HEADER
+
+    def test_generation_prompt_header_hides_tool_name(self, parser, mock_request):
+        text = "get_weather" + _tool_block("get_weather", '{"city":"SF"}')
+        delta = parser.parse_delta(
+            text,
+            [token_id for token_id, _ in _tokenize(text)],
+            mock_request,
+            prompt_token_ids=[_TML_VOCAB[END_MESSAGE], _TML_VOCAB[MSG_MODEL]],
+            finished=True,
+        )
+        assert delta is not None
+        assert delta.content is None
+        assert delta.tool_calls[0].function.name == "get_weather"
+
+
+class TestToolCallFiltering:
+    """Inkling equivalents of the generic tool-call-filtering replay tests
+    (Inkling is excluded from those in test_replay.py: its structural
+    role/kind tokens and shared block-end token don't fit the generic
+    reasoning/tool split model)."""
+
+    def test_skip_tool_parsing_round_trip(self, mock_tokenizer, mock_request):
+        # First pass (reasoning adapter, skip_tool_parsing): reasoning is
+        # classified as reasoning while tool markup survives in content;
+        # second pass (tool adapter) re-extracts the calls from it.
+        text = (
+            f"{THINK_START}plan{END_MESSAGE}{MSG_MODEL}"
+            + _tool_block("f", '{"a":1}')
+            + MSG_MODEL
+            + _tool_block("g", '{"b":[2]}')
+        )
+        first = InklingParser(mock_tokenizer)
+        first.skip_tool_parsing = True
+        reasoning, content = first.extract_reasoning(text, mock_request)
+        assert reasoning == "plan"
+        assert content.count(TOOL_JSON) == 2
+
+        second = InklingParser(mock_tokenizer)
+        result = second.extract_tool_calls_from_content(content, mock_request)
+        assert result.tools_called
+        assert [tc.function.name for tc in result.tool_calls] == ["f", "g"]
+        assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1}
+        assert json.loads(result.tool_calls[1].function.arguments) == {"b": [2]}
+
+    @pytest.fixture
+    def none_request(self, mock_request):
+        mock_request.tools = [{"type": "function", "function": {"name": "f"}}]
+        mock_request.tool_choice = "none"
+        return mock_request
+
+    def test_tool_choice_none_non_streaming(self, mock_tokenizer, none_request):
+        parser = InklingParser(mock_tokenizer)
+        text = (
+            f"{THINK_START}plan{END_MESSAGE}"
+            f"{MSG_MODEL}{TEXT_START}visible{END_MESSAGE}"
+            f"{MSG_MODEL}" + _tool_block("f", '{"a":1}')
+        )
+        reasoning, content, tools = parser.parse(text, none_request)
+        assert reasoning == "plan"
+        assert content == "visible"
+        assert not tools
+
+    def test_tool_choice_none_streaming(self, mock_tokenizer, none_request):
+        parser = InklingParser(mock_tokenizer)
+        text = f"{TEXT_START}visible{END_MESSAGE}{MSG_MODEL}" + _tool_block(
+            "f", '{"a":1}'
+        )
+        results = _stream(parser, none_request, text, 3)
+        assert collect_content(results) == "visible"
+        assert all(not (d and d.tool_calls) for d, _ in results)
+
+
+class TestRegisteredAdapters:
+    def test_adapters_resolve(self):
+        from vllm.reasoning import ReasoningParserManager
+        from vllm.tool_parsers import ToolParserManager
+
+        reasoning_cls = ReasoningParserManager.get_reasoning_parser("inkling")
+        tool_cls = ToolParserManager.get_tool_parser("inkling")
+        assert reasoning_cls._parser_engine_cls is InklingParser
+        assert tool_cls._parser_engine_cls is InklingParser
+        assert tool_cls.supports_required_and_named is False
+
+    def test_adapter_round_trip(self, mock_tokenizer, mock_request):
+        from vllm.tool_parsers import ToolParserManager
+
+        tool_cls = ToolParserManager.get_tool_parser("inkling")
+        adapter = tool_cls(mock_tokenizer)
+        result = adapter.extract_tool_calls(_tool_block("f", '{"a":1}'), mock_request)
+        assert result.tools_called
+        assert result.tool_calls[0].function.name == "f"
+        assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1}

tests/parser/engine/test_replay.py

@@ -68,6 +68,10 @@ def _discover_parsers() -> list[_ParserInfo]:
         if cfg.name not in _BUILDERS:
             missing_builders.append(f"{obj.__name__} (config.name={cfg.name!r})")
             continue
+        if cfg.name == "inkling":
+            # Inkling opts out of token-id terminal matching and has typed
+            # structural blocks; its replay coverage lives in test_inkling.py.
+            continue
         tool_end = cfg.token_id_terminals.get("TOOL_END")
         if not tool_end:
             raise RuntimeError(
@@ -299,7 +303,12 @@ _TOOL_CALL_SAMPLES = [
     (p.parser_cls, s, p.think_end, p.tool_start)
     for p in _PARSERS
     for s in p.samples
-    if s.expected_tool_calls and s.expected_reasoning
+    if s.expected_tool_calls
+    and s.expected_reasoning
+    # Inkling's typed-block format (structural role/kind tokens, shared
+    # block-end token) doesn't fit the generic reasoning/tool split
+    # below; its filtering modes are covered in test_inkling.py instead.
+    and p.name != "inkling"
 ]
 
 

tests/parser/engine/trace_builder.py

@@ -33,6 +33,7 @@ from vllm.parser.engine.registered_adapters import (
     DeepSeekV32Parser,
     Gemma4Parser,
     Glm47MoeParser,
+    InklingParser,
     KimiK2Parser,
     MinimaxM2Parser,
     NemotronV3Parser,
@@ -948,6 +949,77 @@ _KIMI_K2_SCENARIOS = [
 ]
 
 
+# ── Inkling (typed content blocks, JSON tool payloads) ───────────────────
+
+_TML_VOCAB: dict[str, int] = {
+    "<|message_model|>": 200001,
+    "<|content_text|>": 200004,
+    "<|content_model_end_sampling|>": 200006,
+    "<|content_thinking|>": 200008,
+    "<|end_message|>": 200010,
+    "<|content_tool_error|>": 200022,
+    "<|content_invoke_tool_json|>": 200049,
+    "<|content_invoke_tool_text|>": 200057,
+}
+
+
+def _inkling_block(
+    segs: list[tuple[str, bool]],
+    kind_token: str,
+    body: str,
+) -> None:
+    """Append one Inkling content block; blocks after the first start with
+    the ``<|message_model|>`` role token (the first block continues the
+    generation prompt directly)."""
+    if segs:
+        segs.append(("<|message_model|>", True))
+    segs.append((kind_token, True))
+    if body:
+        segs.append((body, False))
+    segs.append(("<|end_message|>", True))
+
+
+def _inkling_segments(scenario: Scenario) -> list[tuple[str, bool]]:
+    segs: list[tuple[str, bool]] = []
+    if scenario.reasoning is not None:
+        _inkling_block(segs, "<|content_thinking|>", scenario.reasoning)
+    if scenario.tool_calls is not None and not scenario.tool_calls:
+        _inkling_block(segs, "<|content_invoke_tool_json|>", "")
+    if scenario.content is not None:
+        _inkling_block(segs, "<|content_text|>", scenario.content)
+    if scenario.tool_calls:
+        for tc in scenario.tool_calls:
+            args = json.dumps(tc.arguments, ensure_ascii=False, separators=(",", ":"))
+            payload = f'{{"name":"{tc.name}","args":{args}}}'
+            _inkling_block(segs, "<|content_invoke_tool_json|>", payload)
+    return segs
+
+
+def _build_inkling(scenario: Scenario, validate: bool = True) -> Sample:
+    prompt_token_ids = None
+    if scenario.after_tool_response:
+        # Prompt ends with a closed tool-response block and the
+        # generation-prompt role token.
+        prompt_token_ids = [
+            _TML_VOCAB["<|end_message|>"],
+            _TML_VOCAB["<|message_model|>"],
+        ]
+    sample = _make_sample(
+        sample_id=f"inkling-{scenario.id}",
+        description=scenario.description,
+        vocab=_TML_VOCAB,
+        segments=_inkling_segments(scenario),
+        expected_reasoning=scenario.reasoning,
+        expected_content=_qwen3_expected_content(scenario),
+        expected_tool_calls=_expected_tc(scenario),
+        tools=_expected_tools(scenario),
+        prompt_token_ids=prompt_token_ids,
+    )
+    if validate:
+        _validate_sample(sample, InklingParser)
+    return sample
+
+
 # ── Registry and public API ──────────────────────────────────────────
 
 _BUILDERS: dict[str, Any] = {
@@ -960,6 +1032,7 @@ _BUILDERS: dict[str, Any] = {
     "glm47_moe": _build_glm47_moe,
     "kimi_k2": _build_kimi_k2,
     "qwen3": _build_qwen3,
+    "inkling": _build_inkling,
 }
 
 

tests/renderers/test_inkling.py

@@ -0,0 +1,484 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Golden tests for the native Inkling renderer encoding.
+
+The message fixtures mirror the Rust renderer's fixture tests
+(``rust/src/chat/src/renderer/inkling/tests.rs``) so the two frontends stay
+in token-level parity. Image blocks emit the bare ``<|content_image|>`` marker;
+``InklingMultiModalProcessor`` inserts the per-patch placeholder run.
+"""
+
+import pytest
+
+from vllm.renderers.inkling import (
+    InklingRenderer,
+    _HfBackedTmlTokenizer,
+    _resolve_reasoning_effort,
+)
+from vllm.renderers.inkling_encoding import (
+    SPECIAL_TOKEN_SPELLINGS,
+    render_inkling_messages,
+)
+from vllm.renderers.params import ChatParams
+
+
+@pytest.fixture()
+def should_do_global_cleanup_after_test() -> bool:
+    # These tests touch no distributed or device state; the global
+    # cleanup fixture is unnecessary (and trips a torch MPS allocator
+    # assert on macOS dev machines).
+    return False
+
+
+# One id per special token, mirroring the real Inkling vocab layout; plain
+# text encodes one token per character so decoded output is exact.
+_SPECIAL_VOCAB = {
+    "<|message_user|>": 200000,
+    "<|message_model|>": 200001,
+    "<|message_system|>": 200002,
+    "<|message_tool|>": 200003,
+    "<|content_text|>": 200004,
+    "<|content_image|>": 200005,
+    "<|content_model_end_sampling|>": 200006,
+    "<|content_thinking|>": 200008,
+    "<|end_message|>": 200010,
+    "<|content_audio_input|>": 200020,
+    # The HF vocab spells CONTENT_XML as an unused slot.
+    "<|unused_200024|>": 200024,
+    "<|audio_end|>": 200043,
+    "<|content_invoke_tool_json|>": 200049,
+}
+
+_ID_TO_SPECIAL = {v: k for k, v in _SPECIAL_VOCAB.items()}
+
+
+class FakeHfTokenizer:
+    def get_vocab(self):
+        return dict(_SPECIAL_VOCAB)
+
+    def encode(self, text, add_special_tokens=False):
+        assert not add_special_tokens
+        return [ord(ch) for ch in text]
+
+
+def decode(token_ids):
+    return "".join(
+        _ID_TO_SPECIAL.get(tid, chr(tid) if tid < 200000 else f"<{tid}>")
+        for tid in token_ids
+    )
+
+
+@pytest.fixture
+def inkling_tokenizer():
+    return _HfBackedTmlTokenizer(FakeHfTokenizer())
+
+
+def render_text(inkling_tokenizer, messages, **kwargs):
+    return decode(render_inkling_messages(messages, inkling_tokenizer, **kwargs))
+
+
+class TestRustFixtureParity:
+    def test_tool_round_trip(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "assistant",
+                "reasoning_content": "think",
+                "content": "answer",
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "function": {
+                            "name": "get_weather",
+                            "arguments": '{"city":"SF"}',
+                        },
+                    }
+                ],
+            },
+            {"role": "tool", "tool_call_id": "call_1", "content": "sunny"},
+        ]
+        assert render_text(
+            inkling_tokenizer, messages, add_generation_prompt=False
+        ) == (
+            "<|message_model|><|content_thinking|>think<|end_message|>"
+            "<|message_model|><|content_text|>answer<|end_message|>"
+            "<|message_model|>get_weather<|content_invoke_tool_json|>"
+            '{"name":"get_weather","args":{"city":"SF"}}<|end_message|>'
+            "<|content_model_end_sampling|>"
+            "<|message_tool|>get_weather<|content_text|>sunny<|end_message|>"
+        )
+
+    def test_tool_declare(self, inkling_tokenizer):
+        tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": "get_weather",
+                    "description": "Get weather information",
+                    "parameters": {
+                        "type": "object",
+                        "required": ["city"],
+                        "properties": {"city": {"type": "string"}},
+                    },
+                },
+            }
+        ]
+        messages = [
+            {
+                "role": "developer",
+                "content": "rules",
+                "tools": [
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": "local_tool",
+                            "parameters": {"z": 1, "a": {"b": 2}},
+                        },
+                    }
+                ],
+            },
+            {"role": "user", "content": "hi"},
+        ]
+        assert render_text(inkling_tokenizer, messages, tools=tools) == (
+            "<|message_system|>tool_declare<|unused_200024|>"
+            '[{"description":"Get weather information","name":"get_weather",'
+            '"parameters":{"properties":{"city":{"type":"string"}},'
+            '"required":["city"],"type":"object"},"type":"function"},'
+            '{"description":"","name":"local_tool",'
+            '"parameters":{"a":{"b":2},"z":1},"type":"function"}]'
+            "<|end_message|>"
+            "<|message_system|><|content_text|>rules<|end_message|>"
+            "<|message_user|><|content_text|>hi<|end_message|>"
+            "<|message_model|>"
+        )
+
+    def test_text_image(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "look"},
+                    {"type": "image_url", "image_url": "data:image/png;base64,"},
+                ],
+            }
+        ]
+        # Multimodal preprocessing expands this bare marker after rendering.
+        assert render_text(inkling_tokenizer, messages) == (
+            "<|message_user|><|content_text|>look<|end_message|>"
+            "<|message_user|><|content_image|><|end_message|>"
+            "<|message_model|>"
+        )
+
+
+class TestRenderingSemantics:
+    @pytest.mark.parametrize(
+        ("value", "expected"),
+        [
+            ("none", 0.0),
+            ("minimal", 0.1),
+            ("low", 0.2),
+            ("medium", 0.7),
+            ("high", 0.9),
+            ("xhigh", 0.99),
+            ("max", 0.99),
+            (None, 0.9),
+            (0.8, 0.8),
+            (True, None),
+            ("invalid", None),
+        ],
+    )
+    def test_resolve_reasoning_effort(self, value, expected):
+        assert _resolve_reasoning_effort(value) == expected
+
+    def test_generation_prompt_default(self, inkling_tokenizer):
+        text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}])
+        assert text.endswith("<|end_message|><|message_model|>")
+
+    def test_developer_folds_into_system(self, inkling_tokenizer):
+        assert (
+            render_text(
+                inkling_tokenizer,
+                [{"role": "developer", "content": "be nice"}],
+                add_generation_prompt=False,
+            )
+            == "<|message_system|><|content_text|>be nice<|end_message|>"
+        )
+
+    def test_empty_string_content_skipped(self, inkling_tokenizer):
+        assert (
+            render_text(
+                inkling_tokenizer,
+                [{"role": "user", "content": ""}],
+                add_generation_prompt=False,
+            )
+            == ""
+        )
+
+    def test_empty_reasoning_skipped(self, inkling_tokenizer):
+        assert (
+            render_text(
+                inkling_tokenizer,
+                [{"role": "assistant", "reasoning_content": "", "content": "hi"}],
+                add_generation_prompt=False,
+            )
+            == "<|message_model|><|content_text|>hi<|end_message|>"
+            "<|content_model_end_sampling|>"
+        )
+
+    def test_reasoning_field(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "assistant",
+                "reasoning": "think",
+                "content": "answer",
+            }
+        ]
+        assert render_text(
+            inkling_tokenizer, messages, add_generation_prompt=False
+        ) == (
+            "<|message_model|><|content_thinking|>think<|end_message|>"
+            "<|message_model|><|content_text|>answer<|end_message|>"
+            "<|content_model_end_sampling|>"
+        )
+
+    def test_audio_part(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "user",
+                "content": [{"type": "input_audio", "input_audio": {}}],
+            }
+        ]
+        assert render_text(
+            inkling_tokenizer, messages, add_generation_prompt=False
+        ) == ("<|message_user|><|content_audio_input|><|audio_end|><|end_message|>")
+
+    def test_tool_response_name_from_message(self, inkling_tokenizer):
+        assert (
+            render_text(
+                inkling_tokenizer,
+                [{"role": "tool", "name": "my_tool", "content": "ok"}],
+                add_generation_prompt=False,
+            )
+            == "<|message_tool|>my_tool<|content_text|>ok<|end_message|>"
+        )
+
+    def test_tool_call_args_object_form(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "assistant",
+                "tool_calls": [
+                    {
+                        "id": "x",
+                        "function": {
+                            "name": "f",
+                            # dict-form arguments, unsorted keys
+                            "arguments": {"z": 1, "a": 2},
+                        },
+                    }
+                ],
+            }
+        ]
+        assert render_text(
+            inkling_tokenizer, messages, add_generation_prompt=False
+        ) == (
+            "<|message_model|>f<|content_invoke_tool_json|>"
+            '{"name":"f","args":{"a":2,"z":1}}<|end_message|>'
+            "<|content_model_end_sampling|>"
+        )
+
+    def test_tool_call_empty_args(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "assistant",
+                "tool_calls": [{"id": "x", "function": {"name": "f", "arguments": ""}}],
+            }
+        ]
+        assert render_text(
+            inkling_tokenizer, messages, add_generation_prompt=False
+        ) == (
+            "<|message_model|>f<|content_invoke_tool_json|>"
+            '{"name":"f","args":{}}<|end_message|>'
+            "<|content_model_end_sampling|>"
+        )
+
+    def test_tool_call_non_object_args_rejected(self, inkling_tokenizer):
+        messages = [
+            {
+                "role": "assistant",
+                "tool_calls": [
+                    {"id": "x", "function": {"name": "f", "arguments": "[1]"}}
+                ],
+            }
+        ]
+        with pytest.raises(TypeError, match="decode to an object"):
+            render_inkling_messages(
+                messages, inkling_tokenizer, add_generation_prompt=False
+            )
+
+    def test_unsupported_role_rejected(self, inkling_tokenizer):
+        with pytest.raises(ValueError, match="unsupported Inkling message role"):
+            render_inkling_messages(
+                [{"role": "narrator", "content": "hi"}], inkling_tokenizer
+            )
+
+
+class TestReasoningEffort:
+    def test_frontend_defaults_to_high(self, inkling_tokenizer):
+        renderer = InklingRenderer.__new__(InklingRenderer)
+        renderer._inkling_tokenizer = inkling_tokenizer
+
+        text = decode(
+            renderer._render(
+                [
+                    {"role": "system", "content": "rules"},
+                    {"role": "user", "content": "hi"},
+                ],
+                ChatParams(),
+            )
+        )
+
+        assert text.startswith(
+            "<|message_system|><|content_text|>rules<|end_message|>"
+            "<|message_system|><|content_text|>Thinking effort level: 0.9"
+            "<|end_message|>"
+        )
+
+    @pytest.mark.parametrize("value", ["none", 0, 0.0, -0.0])
+    def test_zero_effort_has_one_canonical_spelling(self, inkling_tokenizer, value):
+        renderer = InklingRenderer.__new__(InklingRenderer)
+        renderer._inkling_tokenizer = inkling_tokenizer
+
+        text = decode(
+            renderer._render(
+                [{"role": "user", "content": "hi"}],
+                ChatParams(chat_template_kwargs={"reasoning_effort": value}),
+            )
+        )
+
+        assert text.startswith(
+            "<|message_system|><|content_text|>Thinking effort level: 0.0"
+            "<|end_message|>"
+        )
+
+    def test_emits_one_effort_after_initial_prefix(self, inkling_tokenizer):
+        effort_block = (
+            "<|message_system|><|content_text|>Thinking effort level: 0.7"
+            "<|end_message|>"
+        )
+        text = render_text(
+            inkling_tokenizer,
+            [
+                {"role": "system", "content": "rules"},
+                {"role": "developer", "content": "policy"},
+                {"role": "user", "content": "user1"},
+                {"role": "assistant", "content": "assistant1"},
+                {"role": "user", "content": "user2"},
+            ],
+            tools=[{"type": "function", "function": {"name": "f"}}],
+            reasoning_effort=0.7,
+        )
+
+        assert text.count(effort_block) == 1
+        assert (
+            text.index("tool_declare")
+            < text.index("rules")
+            < text.index("policy")
+            < text.index(effort_block)
+            < text.index("user1")
+            < text.index("assistant1")
+            < text.index("user2")
+        )
+
+    def test_renders_after_tool_declare(self, inkling_tokenizer):
+        tools = [{"type": "function", "function": {"name": "f"}}]
+        text = render_text(
+            inkling_tokenizer,
+            [{"role": "user", "content": "hi"}],
+            tools=tools,
+            reasoning_effort=0.8,
+        )
+        declare_end = text.index("<|end_message|>") + len("<|end_message|>")
+        assert text[declare_end:].startswith(
+            "<|message_system|><|content_text|>Thinking effort level: 0.8"
+            "<|end_message|>"
+        )
+
+    @pytest.mark.parametrize(
+        ("value", "expected"),
+        [(0.2, "0.2"), (0.7, "0.7"), (0.9, "0.9"), (0.99, "0.99")],
+    )
+    def test_formats_at_most_two_decimals(self, inkling_tokenizer, value, expected):
+        text = render_text(
+            inkling_tokenizer,
+            [{"role": "user", "content": "hi"}],
+            reasoning_effort=value,
+        )
+        assert f"Thinking effort level: {expected}<|end_message|>" in text
+
+    def test_absent_emits_no_block(self, inkling_tokenizer):
+        text = render_text(inkling_tokenizer, [{"role": "user", "content": "hi"}])
+        assert "Thinking effort" not in text
+
+    @pytest.mark.parametrize("value", [0.9900001, 1, 1.5, -0.1])
+    def test_out_of_range_rejected(self, inkling_tokenizer, value):
+        with pytest.raises(ValueError, match="must be in"):
+            render_inkling_messages(
+                [{"role": "user", "content": "hi"}],
+                inkling_tokenizer,
+                reasoning_effort=value,
+            )
+
+
+class TestSpecialTokenResolution:
+    def test_missing_special_token_raises(self):
+        class IncompleteTokenizer(FakeHfTokenizer):
+            def get_vocab(self):
+                vocab = dict(_SPECIAL_VOCAB)
+                del vocab["<|content_invoke_tool_json|>"]
+                return vocab
+
+        with pytest.raises(ValueError, match="missing special tokens"):
+            _HfBackedTmlTokenizer(IncompleteTokenizer())
+
+    def test_semantic_spelling_preferred(self):
+        class SemanticSpellingTokenizer(FakeHfTokenizer):
+            def get_vocab(self):
+                vocab = dict(_SPECIAL_VOCAB)
+                del vocab["<|unused_200024|>"]
+                vocab["<|content_xml|>"] = 200024
+                return vocab
+
+        inkling_tokenizer = _HfBackedTmlTokenizer(SemanticSpellingTokenizer())
+        tools = [{"type": "function", "function": {"name": "f"}}]
+        ids = render_inkling_messages(
+            [{"role": "user", "content": "hi"}], inkling_tokenizer, tools=tools
+        )
+        assert 200024 in ids
+
+    def test_all_spellings_covered(self):
+        # Every semantic token must resolve from the reference vocab.
+        for token, spellings in SPECIAL_TOKEN_SPELLINGS.items():
+            assert any(s in _SPECIAL_VOCAB for s in spellings), token
+
+
+def test_render_inkling_messages_direct_protocol():
+    """The encoding core only needs the structural tokenizer protocol."""
+
+    class ProtocolTokenizer:
+        def encode_text(self, text):
+            return [ord(c) for c in text]
+
+        def encode_special(self, token):
+            return {
+                "<|message_user|>": 200000,
+                "<|message_model|>": 200001,
+                "<|content_text|>": 200004,
+                "<|content_model_end_sampling|>": 200006,
+                "<|end_message|>": 200010,
+            }[token]
+
+    ids = render_inkling_messages(
+        [{"role": "user", "content": "hi"}],
+        ProtocolTokenizer(),
+        add_generation_prompt=True,
+    )
+    assert ids == [200000, 200004, ord("h"), ord("i"), 200010, 200001]

tests/test_config.py

@@ -211,6 +211,26 @@ def test_resolve_cudagraph_mode_adjusts_spec_decode_sizes_only_for_v1(
             ),
             True,
         ),
+        (
+            SimpleNamespace(
+                model="thinkingmachines/Inkling",
+                architectures=["InklingForCausalLM"],
+                runner_type="generate",
+                is_moe=True,
+                is_quantized=False,
+            ),
+            True,
+        ),
+        (
+            SimpleNamespace(
+                model="thinkingmachines/Inkling",
+                architectures=["InklingForConditionalGeneration"],
+                runner_type="generate",
+                is_moe=True,
+                is_quantized=False,
+            ),
+            True,
+        ),
         (
             SimpleNamespace(
                 model="mistralai/Mixtral-8x7B-Instruct-v0.1",

vllm/config/model.py

@@ -83,7 +83,9 @@ _REGISTERED_MODEL_CLASS_OVERRIDES: set[tuple[str, str]] = set()
 RunnerOption = Literal["auto", RunnerType]
 ConvertType = Literal["none", "embed", "classify"]
 ConvertOption = Literal["auto", ConvertType]
-TokenizerMode = Literal["auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4"]
+TokenizerMode = Literal[
+    "auto", "hf", "slow", "mistral", "deepseek_v32", "deepseek_v4", "inkling"
+]
 ModelDType = Literal["auto", "half", "float16", "bfloat16", "float", "float32"]
 LogprobsMode = Literal[
     "raw_logits", "raw_logprobs", "processed_logits", "processed_logprobs"
@@ -628,6 +630,8 @@ class ModelConfig:
                 self.tokenizer_mode = "deepseek_v32"
             elif arch == "DeepseekV4ForCausalLM":
                 self.tokenizer_mode = "deepseek_v4"
+            elif arch in ("InklingForCausalLM", "InklingForConditionalGeneration"):
+                self.tokenizer_mode = "inkling"
 
             if self.tokenizer_mode != "auto":
                 logger.info(

vllm/config/vllm.py

@@ -68,9 +68,11 @@ logger = init_logger(__name__)
 DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset(
     {
         "DeepseekV2ForCausalLM",
-        "Qwen2MoeForCausalLM",
         "GraniteMoeForCausalLM",
+        "InklingForCausalLM",
+        "InklingForConditionalGeneration",
         "LongcatFlashNgramForCausalLM",
+        "Qwen2MoeForCausalLM",
     }
 )
 

vllm/model_executor/models/registry.py

@@ -159,6 +159,11 @@ _TEXT_GENERATION_MODELS = {
         "vllm.models.minimax_m3",
         "MiniMaxM3SparseForCausalLM",
     ),
+    "InklingForCausalLM": ("vllm.models.inkling", "InklingForCausalLM"),
+    "InklingForConditionalGeneration": (
+        "vllm.models.inkling",
+        "InklingForConditionalGeneration",
+    ),
     "Ministral3ForCausalLM": ("mistral", "MistralForCausalLM"),
     "MistralForCausalLM": ("mistral", "MistralForCausalLM"),
     "MistralLarge3ForCausalLM": ("mistral_large_3", "MistralLarge3ForCausalLM"),

vllm/model_executor/warmup/kernel_warmup.py

@@ -43,6 +43,7 @@ if TYPE_CHECKING:
 logger = init_logger(__name__)
 
 _LL_BF16_WARMUP_MODEL_SHAPES: tuple[tuple[int, int], ...] = (
+    (6144, 264),  # Inkling
     (7168, 256),  # DSV3
     (7168, 384),  # DSV4-Pro
     (14400, 256),  # DSV4-Flash

vllm/models/inkling/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .nvidia.model import (
+        InklingForCausalLM,
+        InklingForConditionalGeneration,
+    )
+
+__all__ = [
+    "InklingForConditionalGeneration",
+    "InklingForCausalLM",
+]
+
+
+def __getattr__(name: str):
+    if name in __all__:
+        from .nvidia import model
+
+        return getattr(model, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

vllm/models/inkling/common/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/models/inkling/common/mm_preprocess.py

@@ -0,0 +1,361 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling multimodal preprocessing."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any, cast
+
+import numpy as np
+import regex as re
+import torch
+from transformers.feature_extraction_utils import BatchFeature
+
+from vllm.config.multimodal import (
+    AudioDummyOptions,
+    BaseDummyOptions,
+    ImageDummyOptions,
+)
+from vllm.inputs import MultiModalDataDict
+from vllm.multimodal.inputs import (
+    MultiModalFieldConfig,
+    MultiModalKwargsItems,
+)
+from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
+from vllm.multimodal.processing import (
+    BaseDummyInputsBuilder,
+    BaseMultiModalProcessor,
+    BaseProcessingInfo,
+    PromptReplacement,
+    PromptUpdate,
+    PromptUpdateDetails,
+)
+from vllm.transformers_utils.processors.inkling import (
+    AUDIO_MARKER_ID,
+    AUDIO_TOKEN_ID,
+    IMAGE_MARKER_ID,
+    IMAGE_TOKEN_ID,
+    InklingAudioFeatureExtractor,
+    InklingImageProcessor,
+    InklingProcessor,
+)
+
+from ..configs import InklingMMConfig
+
+# Maximum audio tokens accepted per clip. At the dMel rate of 20 tokens/s
+# (50 ms hop) this is ~10 minutes of audio. It bounds the persistent per-request
+# buffers and the encoder/memory budget; longer clips are rejected up front.
+MAX_AUDIO_TOKENS = 12_000
+
+
+class InklingMultiModalDataParser(MultiModalDataParser):
+    def _parse_audio_data(self, data: Any) -> Any:
+        if isinstance(data, (np.ndarray, torch.Tensor)) and data.ndim == 2:
+            raise ValueError(
+                "Inkling raw 2-D audio has an ambiguous channel layout. "
+                "Provide encoded audio or a list of mono waveforms."
+            )
+        return super()._parse_audio_data(data)
+
+
+def inkling_vision_enabled(config: InklingMMConfig) -> bool:
+    return getattr(config.vision_config, "decoder_dmodel", None) is not None
+
+
+def inkling_audio_enabled(config: InklingMMConfig) -> bool:
+    return getattr(config.audio_config, "decoder_dmodel", None) is not None
+
+
+class InklingProcessingInfo(BaseProcessingInfo):
+    def get_hf_config(self) -> InklingMMConfig:
+        return self.ctx.get_hf_config(InklingMMConfig)
+
+    def get_hf_processor(self, **kwargs: object) -> InklingProcessor:
+        config = self.get_hf_config()
+        vision_config = config.vision_config
+        audio_config = config.audio_config
+
+        image_processor = InklingImageProcessor(
+            patch_size=getattr(vision_config, "patch_size", None) or 40,
+        )
+
+        if inkling_audio_enabled(config):
+            audio_params = {
+                "n_mels": audio_config.n_mel_bins,
+                "num_dmel_bins": audio_config.mel_vocab_size,
+                "dmel_min_value": audio_config.dmel_min_value,
+                "dmel_max_value": audio_config.dmel_max_value,
+            }
+        else:
+            audio_params = {}
+        audio_extractor = InklingAudioFeatureExtractor(params=audio_params)
+
+        return InklingProcessor(
+            image_processor=image_processor,
+            audio_feature_extractor=audio_extractor,
+            tokenizer=self.get_tokenizer(),
+        )
+
+    def get_data_parser(self) -> MultiModalDataParser:
+        # Audio inputs must be resampled to the dMel feature extractor's rate
+        # before `process_audios` (see InklingAudioFeatureExtractor._decode_one).
+        # Without a target_sr the default parser raises on any audio input.
+        extractor = self.get_hf_processor().audio_feature_extractor
+        return InklingMultiModalDataParser(
+            target_sr=extractor.params.sample_rate,
+            target_channels=1,
+            expected_hidden_size=self._get_expected_hidden_size(),
+        )
+
+    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
+        config = self.get_hf_config()
+        limits: dict[str, int | None] = {}
+        if inkling_vision_enabled(config):
+            limits["image"] = None
+        if inkling_audio_enabled(config):
+            limits["audio"] = None
+        return limits
+
+    def get_mm_max_tokens_per_item(
+        self,
+        seq_len: int,
+        mm_counts: Mapping[str, int],
+    ) -> Mapping[str, int] | None:
+        # Let vLLM profile dummy inputs to determine the max token counts; the
+        # image patch count is data-dependent, and the dummy audio is sized to
+        # MAX_AUDIO_TOKENS so audio is profiled/budgeted at its allowed maximum.
+        return None
+
+
+class InklingDummyInputsBuilder(BaseDummyInputsBuilder[InklingProcessingInfo]):
+    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
+        # One placeholder per media item; the processor expands each into N
+        # copies once feature row counts are known.
+        num_images = mm_counts.get("image", 0)
+        num_audios = mm_counts.get("audio", 0)
+        # Use spellings the renderer would emit; tokenization is bypassed in
+        # _call_hf_processor (we build input_ids directly), so the exact text
+        # only needs to be a stable per-item marker.
+        return ("<|content_image|>" * num_images) + (
+            "<|content_audio_input|>" * num_audios
+        )
+
+    def get_dummy_mm_data(
+        self,
+        seq_len: int,
+        mm_counts: Mapping[str, int],
+        mm_options: Mapping[str, BaseDummyOptions],
+    ) -> MultiModalDataDict:
+        config = self.info.get_hf_config()
+        num_images = mm_counts.get("image", 0)
+        num_audios = mm_counts.get("audio", 0)
+
+        mm_data: dict[str, Any] = {}
+        if num_images:
+            patch_size = getattr(config.vision_config, "patch_size", 40)
+            # A square image ~4 patches wide so the dummy emits several patches.
+            side = patch_size * 4
+            image_overrides = mm_options.get("image")
+            mm_data["image"] = self._get_dummy_images(
+                width=side,
+                height=side,
+                num_images=num_images,
+                overrides=cast(ImageDummyOptions | None, image_overrides),
+            )
+        if num_audios:
+            # Size the dummy at the maximum allowed audio so memory/encoder
+            # budgeting reflects the largest clip we accept (MAX_AUDIO_TOKENS).
+            params = self.info.get_hf_processor().audio_feature_extractor.params
+            hop = int(round(params.audio_token_duration_s * params.sample_rate))
+            audio_len = MAX_AUDIO_TOKENS * hop
+            audio_overrides = mm_options.get("audio")
+            mm_data["audio"] = self._get_dummy_audios(
+                length=audio_len,
+                num_audios=num_audios,
+                overrides=cast(AudioDummyOptions | None, audio_overrides),
+            )
+        return mm_data
+
+
+class InklingMultiModalProcessor(BaseMultiModalProcessor[InklingProcessingInfo]):
+    def _hf_processor_applies_updates(
+        self,
+        prompt_text: str,
+        mm_items: MultiModalDataItems,
+        hf_processor_mm_kwargs: Mapping[str, object],
+        tokenization_kwargs: Mapping[str, object],
+    ) -> bool:
+        return False
+
+    def _call_hf_processor(
+        self,
+        prompt: str,
+        mm_data: Mapping[str, object],
+        mm_kwargs: Mapping[str, object],
+        tok_kwargs: Mapping[str, object],
+    ) -> BatchFeature:
+        # Inkling is not a standard HF processor (no fused text+mm call), so we run
+        # the vendored extractors ourselves and tokenize the text separately.
+        # The MM placeholders in `prompt` are expanded later by the prompt
+        # updates, so here we emit ONE placeholder id per media item.
+        processor = self.info.get_hf_processor(**mm_kwargs)
+        tokenizer = self.info.get_tokenizer()
+
+        images = mm_data.get("images") or []
+        audios = mm_data.get("audios") or []
+        if not isinstance(images, list):
+            images = list(cast(Iterable[Any], images))
+        if not isinstance(audios, list):
+            audios = list(cast(Iterable[Any], audios))
+
+        prompt_ids = self._tokenize_with_placeholders(
+            prompt, tokenizer, len(images), len(audios)
+        )
+
+        data: dict[str, Any] = {"input_ids": [prompt_ids]}
+
+        if images:
+            img_feat = processor.process_images(images)
+            data["pixel_values"] = img_feat["vision_patches_bthwc"]
+            data["num_patches"] = torch.tensor(
+                img_feat["num_patches"], dtype=torch.int64
+            )
+
+        if audios:
+            aud_feat = processor.process_audios(audios)
+            per_clip = aud_feat["dmel_bins"]
+            num_audio_tokens = aud_feat["num_audio_tokens"]
+            for i, n in enumerate(num_audio_tokens):
+                if int(n) > MAX_AUDIO_TOKENS:
+                    raise ValueError(
+                        f"Audio clip {i} produces {int(n)} tokens, exceeding the "
+                        f"maximum of {MAX_AUDIO_TOKENS} (~10 min at 20 tokens/s). "
+                        "Provide a shorter clip."
+                    )
+            if per_clip:
+                input_audio_features = torch.cat(
+                    [torch.as_tensor(c) for c in per_clip], dim=0
+                )
+            else:
+                input_audio_features = torch.empty(0)
+            data["input_audio_features"] = input_audio_features
+            data["num_audio_tokens"] = torch.tensor(num_audio_tokens, dtype=torch.int64)
+
+        return BatchFeature(data=data, tensor_type=None)
+
+    def _tokenize_with_placeholders(
+        self,
+        prompt: str,
+        tokenizer: Any,
+        num_images: int,
+        num_audios: int,
+    ) -> list[int]:
+        """Tokenize `prompt`, emitting the block-start marker id per media item.
+
+        Each marker (kept verbatim) is later expanded by ``_get_prompt_updates``
+        into ``<marker> + <placeholder> * N``.
+        """
+        image_marker = "<|content_image|>"
+        audio_marker = "<|content_audio_input|>"
+
+        pattern = f"({re.escape(image_marker)}|{re.escape(audio_marker)})"
+        chunks = re.split(pattern, prompt)
+
+        ids: list[int] = []
+        seen_img = seen_aud = 0
+        for chunk in chunks:
+            if chunk == image_marker:
+                ids.append(IMAGE_MARKER_ID)
+                seen_img += 1
+            elif chunk == audio_marker:
+                ids.append(AUDIO_MARKER_ID)
+                seen_aud += 1
+            elif chunk:
+                ids.extend(tokenizer.encode(chunk, add_special_tokens=False))
+
+        # Reconcile against the declared media counts only when media is
+        # present. With no media items -- e.g. the base text-only tokenization
+        # probe (``_apply_hf_processor_text_only``), which calls this via
+        # ``_call_hf_processor`` with empty ``mm_data`` -- emit the markers
+        # verbatim; the marker<->item correspondence is enforced later by
+        # ``_get_prompt_updates`` once the media features are available.
+        if num_images or num_audios:
+            # Fail clearly on a placeholder/media-count mismatch instead of
+            # crashing with an IndexError deep in the per-item replacement logic.
+            if num_images and seen_img != num_images:
+                raise ValueError(
+                    f"Prompt contains {seen_img} image placeholder(s), but only "
+                    f"{num_images} image(s) were provided."
+                )
+            if num_audios and seen_aud != num_audios:
+                raise ValueError(
+                    f"Prompt contains {seen_aud} audio placeholder(s), but only "
+                    f"{num_audios} audio input(s) were provided."
+                )
+        return ids
+
+    def _get_mm_fields_config(
+        self,
+        hf_inputs: BatchFeature,
+        hf_processor_mm_kwargs: Mapping[str, object],
+    ) -> Mapping[str, MultiModalFieldConfig]:
+        num_patches = hf_inputs.get("num_patches", torch.empty(0, dtype=torch.int64))
+        num_audio_tokens = hf_inputs.get(
+            "num_audio_tokens", torch.empty(0, dtype=torch.int64)
+        )
+        return dict(
+            # Ragged per-image patches, grouped by num_patches.
+            pixel_values=MultiModalFieldConfig.flat_from_sizes("image", num_patches),
+            num_patches=MultiModalFieldConfig.batched("image"),
+            # Ragged per-audio frames, grouped by num_audio_tokens.
+            input_audio_features=MultiModalFieldConfig.flat_from_sizes(
+                "audio", num_audio_tokens
+            ),
+            num_audio_tokens=MultiModalFieldConfig.batched("audio"),
+        )
+
+    def _get_prompt_updates(
+        self,
+        mm_items: Any,
+        hf_processor_mm_kwargs: Mapping[str, Any],
+        out_mm_kwargs: MultiModalKwargsItems,
+    ) -> Sequence[PromptUpdate]:
+        out_mm_data = out_mm_kwargs.get_data()
+        num_patches: Any = out_mm_data.get("num_patches")
+        num_audio_tokens: Any = out_mm_data.get("num_audio_tokens")
+
+        # Keep the block-start marker and append N placeholder tokens after it;
+        # only the placeholder positions are flagged as embeddings (is_embed), so
+        # the marker stays a normal text token while the tower features scatter
+        # into the placeholders.
+        def image_replacement(item_idx: int) -> PromptUpdateDetails:
+            n = int(num_patches[item_idx])
+            return PromptUpdateDetails.select_token_id(
+                [IMAGE_MARKER_ID] + [IMAGE_TOKEN_ID] * n, IMAGE_TOKEN_ID
+            )
+
+        def audio_replacement(item_idx: int) -> PromptUpdateDetails:
+            n = int(num_audio_tokens[item_idx])
+            return PromptUpdateDetails.select_token_id(
+                [AUDIO_MARKER_ID] + [AUDIO_TOKEN_ID] * n, AUDIO_TOKEN_ID
+            )
+
+        updates: list[PromptUpdate] = []
+        if num_patches is not None and len(num_patches) > 0:
+            updates.append(
+                PromptReplacement(
+                    modality="image",
+                    target=[IMAGE_MARKER_ID],
+                    replacement=image_replacement,
+                )
+            )
+        if num_audio_tokens is not None and len(num_audio_tokens) > 0:
+            updates.append(
+                PromptReplacement(
+                    modality="audio",
+                    target=[AUDIO_MARKER_ID],
+                    replacement=audio_replacement,
+                )
+            )
+        return updates

vllm/models/inkling/common/towers.py

@@ -0,0 +1,321 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling Titan vision + audio towers.
+
+The vision tower (``InklingVision`` / ``HMLPPatchEncoder``) emits one token per
+image patch; the audio tower (``InklingAudio``) emits one token per audio frame.
+Both use vLLM's standard ``RMSNorm`` (CPU-friendly, with a native fallback).
+"""
+
+from __future__ import annotations
+
+from typing import cast
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+
+from vllm.model_executor.layers.layernorm import RMSNorm
+
+from ..configs import InklingAudioConfig, InklingVisionConfig
+
+# ===========================================================================
+# Vision tower (HMLPPatchEncoder / InklingVision)
+# ===========================================================================
+
+
+def _prime_factors(n: int) -> list[int]:
+    """Return the prime factors of *n* in ascending order."""
+    if n < 1:
+        raise ValueError("n must be a positive integer")
+
+    factors: list[int] = []
+    while n % 2 == 0:
+        factors.append(2)
+        n //= 2
+    p = 3
+    while p * p <= n:
+        while n % p == 0:
+            factors.append(p)
+            n //= p
+        p += 2
+    if n > 1:
+        factors.append(n)
+    return factors
+
+
+def plan_out_scales(
+    temporal_patch_size: int, patch_size: int, n_layers: int, n_channels: int = 3
+) -> list[tuple[int, int, int, int]]:
+    """Plan the (t, h, w, c) dimensions for each HMLP layer.
+
+    Spatial dims expand first, then temporal; channel counts round up to
+    multiples of 64.
+    """
+    if patch_size <= 1:
+        raise ValueError(
+            "patch_size must be greater than 1, otherwise this doesn't make sense"
+        )
+
+    def _round_up(x: int) -> int:
+        return int(np.ceil(x / 64)) * 64
+
+    last_h_scale = 1
+    scales: list[tuple[int, int, int, int]] = [(1, 1, 1, n_channels)]
+    for pscale in _prime_factors(patch_size)[::-1]:
+        last_h_scale *= pscale
+        scales.append(
+            (
+                1,
+                last_h_scale,
+                last_h_scale,
+                _round_up((last_h_scale**2) * n_channels),
+            )
+        )
+    last_t_scale = 1
+    for tscale in _prime_factors(temporal_patch_size)[::-1]:
+        last_t_scale *= tscale
+        scales.append(
+            (
+                last_t_scale,
+                last_h_scale,
+                last_h_scale,
+                _round_up((last_h_scale**2) * n_channels * last_t_scale),
+            )
+        )
+
+    size_reduction = np.prod(np.array(scales)[:, :-1], 1)
+
+    log_ideal_scales = np.linspace(
+        0,
+        np.log(patch_size * patch_size * temporal_patch_size * n_channels),
+        n_layers + 1,
+    )
+    cost_matrix = np.abs(log_ideal_scales[:, None] - np.log(size_reduction)[None])
+
+    if n_layers >= len(scales):
+        idxs = np.argmin(cost_matrix, axis=1)
+    else:
+        from scipy.optimize import linear_sum_assignment
+
+        idxs = linear_sum_assignment(cost_matrix)[1]
+
+    assert len(idxs) >= 2
+    idxs[0] = 0
+    idxs[-1] = len(scales) - 1
+
+    return [scales[i] for i in idxs]
+
+
+def fold_timespace_to_depth(
+    vision_patches_bthwc: torch.Tensor, t_fold: int, hw_fold: int
+) -> torch.Tensor:
+    """(B, T, H, W, C) -> (B, T//t, H//hw, W//hw, C*(t*hw**2))."""
+    B, T, H, W, C = vision_patches_bthwc.shape
+
+    assert T % t_fold == 0, f"Temporal dimension {T} must be divisible by {t_fold}"
+    assert H % hw_fold == 0, f"Height dimension {H} must be divisible by {hw_fold}"
+    assert W % hw_fold == 0, f"Width dimension {W} must be divisible by {hw_fold}"
+
+    t_new = T // t_fold
+    h_new = H // hw_fold
+    w_new = W // hw_fold
+
+    x = vision_patches_bthwc.reshape(
+        B, t_new, t_fold, h_new, hw_fold, w_new, hw_fold, C
+    )
+    x = x.permute(0, 1, 3, 5, 2, 4, 6, 7)
+    x = x.reshape(B, t_new, h_new, w_new, t_fold * hw_fold * hw_fold * C)
+    return x
+
+
+class HMLPPatchEncoder(nn.Module):
+    def __init__(self, config: InklingVisionConfig):
+        super().__init__()
+        self.decoder_dmodel = config.decoder_dmodel
+        self.patch_size = config.patch_size
+        self.temporal_patch_size = config.temporal_patch_size
+        self.n_channels = config.n_channels
+        self.n_layers = config.n_layers
+        self.use_vision_norm = config.use_vision_norm
+
+        self.scales: list[tuple[int, int, int, int]] = plan_out_scales(
+            self.temporal_patch_size, self.patch_size, self.n_layers, self.n_channels
+        )
+        self.layers: nn.ModuleDict = nn.ModuleDict()
+        for i, (start_scale, end_scale) in enumerate(
+            zip(self.scales[:-1], self.scales[1:])
+        ):
+            shuffle_mult = (
+                (end_scale[0] // start_scale[0])
+                * (end_scale[1] // start_scale[1])
+                * (end_scale[2] // start_scale[2])
+            )
+            if i == self.n_layers - 1:
+                self.layers[f"linear_{i}"] = nn.Linear(
+                    start_scale[3] * shuffle_mult, self.decoder_dmodel, bias=False
+                )
+            else:
+                self.layers[f"linear_{i}"] = nn.Linear(
+                    start_scale[3] * shuffle_mult, end_scale[3], bias=False
+                )
+                self.layers[f"norm_{i}"] = RMSNorm(end_scale[3])
+
+        self.final_norm: RMSNorm | None = None
+        if self.use_vision_norm:
+            assert self.decoder_dmodel is not None
+            self.final_norm = RMSNorm(self.decoder_dmodel)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        # Fused norm+gelu on CUDA (same fp32-accum/bf16-rounding structure as
+        # the generic path below; differs only by reduction order).
+        fused = None
+        if x.is_cuda and x.dtype == torch.bfloat16:
+            from vllm.models.inkling.nvidia.ops.mm_towers import rmsnorm_gelu
+
+            fused = rmsnorm_gelu
+
+        num_patches, T, H, W, C = x.shape
+        prefolded = False
+        for i, (start_scale, end_scale) in enumerate(
+            zip(self.scales[:-1], self.scales[1:])
+        ):
+            t_fold = end_scale[0] // start_scale[0]
+            hw_fold = end_scale[1] // start_scale[1]
+            if (hw_fold > 1 or t_fold > 1) and not prefolded:
+                x = fold_timespace_to_depth(x, t_fold, hw_fold)
+            prefolded = False
+            assert x.shape[1:-1] == (
+                T // end_scale[0],
+                H // end_scale[1],
+                W // end_scale[2],
+            )
+            x = self.layers[f"linear_{i}"](x)
+            if i < self.n_layers - 1:
+                norm = cast(RMSNorm, self.layers[f"norm_{i}"])
+                if fused is not None:
+                    # If the NEXT layer starts with a copying fold (spatial
+                    # dims still > 1 after folding), store this layer's
+                    # output directly in the folded layout instead.
+                    nxt = self.scales[i + 2]
+                    ntf = nxt[0] // end_scale[0]
+                    nhf = nxt[1] // end_scale[1]
+                    copy_fold = (ntf > 1 or nhf > 1) and (
+                        x.shape[2] // nhf > 1 or x.shape[3] // nhf > 1
+                    )
+                    x = fused(
+                        x,
+                        norm.weight,
+                        norm.variance_epsilon,
+                        gelu=True,
+                        fold=(ntf, nhf) if copy_fold else None,
+                    )
+                    prefolded = copy_fold
+                else:
+                    # rms_norm kernel only supports rank 2-4; x is 5-D here.
+                    orig = x.shape
+                    x = norm(x.reshape(-1, x.shape[-1])).reshape(orig)
+                    x = F.gelu(x)
+
+        if self.final_norm is not None:
+            if fused is not None:
+                x = fused(
+                    x,
+                    self.final_norm.weight,
+                    self.final_norm.variance_epsilon,
+                    gelu=False,
+                )
+            else:
+                orig = x.shape
+                x = self.final_norm(x.reshape(-1, x.shape[-1])).reshape(orig)
+
+        x = x.reshape(num_patches, -1)
+        return x
+
+
+class InklingVision(nn.Module):
+    def __init__(self, config: InklingVisionConfig, prefix: str = ""):
+        del prefix
+        super().__init__()
+        assert config.vision_encoder_type == "hmlp"
+        self.vision_encoder = HMLPPatchEncoder(config)
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return next(self.parameters()).dtype
+
+    @property
+    def device(self) -> torch.device:
+        return next(self.parameters()).device
+
+    def forward(self, vision_features: torch.Tensor) -> torch.Tensor:
+        return self.vision_encoder(vision_features)
+
+
+# ===========================================================================
+# Audio tower (InklingAudio)
+# ===========================================================================
+
+
+class InklingAudio(nn.Module):
+    def __init__(self, config: InklingAudioConfig, prefix: str = ""):
+        del prefix
+        super().__init__()
+        assert config.audio_mode == "dmel"
+        self.n_mel_bins = config.n_mel_bins
+        self.mel_vocab_size = config.mel_vocab_size
+        self.use_audio_norm = config.use_audio_norm
+        self.encoder = nn.Embedding(
+            config.n_mel_bins * config.mel_vocab_size, config.decoder_dmodel
+        )
+        self.final_norm: RMSNorm | None = None
+        if self.use_audio_norm:
+            assert config.decoder_dmodel is not None
+            self.final_norm = RMSNorm(config.decoder_dmodel, eps=1e-6)
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return self.encoder.weight.dtype
+
+    @property
+    def device(self) -> torch.device:
+        return self.encoder.weight.device
+
+    def forward(self, audio_features: torch.Tensor) -> torch.Tensor:
+        assert audio_features.shape[1] == self.n_mel_bins
+
+        # dMel bins are integer indices; cast once to int32 on the right device
+        # (no float round-trip).
+        audio_features = audio_features.to(
+            device=self.encoder.weight.device, dtype=torch.int32
+        )
+
+        weight = self.encoder.weight
+        if audio_features.is_cuda and weight.dtype == torch.bfloat16:
+            # One kernel: per-bin offset + embedding gather + fp32 sum + norm.
+            # Skips the [T, n_mel_bins, D] intermediate entirely (bit-exact).
+            from vllm.models.inkling.nvidia.ops.mm_towers import dmel_embed_sum_norm
+
+            return dmel_embed_sum_norm(
+                audio_features.contiguous(),
+                weight,
+                self.final_norm.weight if self.final_norm is not None else None,
+                self.final_norm.variance_epsilon if self.final_norm else 0.0,
+            )
+
+        embedding_indices = (
+            torch.arange(self.n_mel_bins, device=audio_features.device)
+            * self.mel_vocab_size
+        ).unsqueeze(0) + audio_features
+
+        hidden_states = (
+            self.encoder(embedding_indices.reshape(-1))
+            .reshape(audio_features.shape[0], audio_features.shape[1], -1)
+            .sum(axis=1)
+        )
+
+        if self.final_norm is not None:
+            hidden_states = self.final_norm(hidden_states)
+
+        return hidden_states

vllm/models/inkling/configs.py

@@ -0,0 +1,373 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling model configs for the text backbone and audio/vision towers."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, ClassVar, Literal, cast
+
+import torch
+from transformers.configuration_utils import PretrainedConfig
+
+
+class InklingModelConfig(PretrainedConfig):
+    model_type = "inkling_model"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        *,
+        vocab_size: int = 201024,
+        hidden_size: int = 1536,
+        intermediate_size: int = 768,
+        dense_intermediate_size: int | None = None,
+        num_hidden_layers: int = 16,
+        num_attention_heads: int = 12,
+        num_key_value_heads: int = 4,
+        head_dim: int | None = None,
+        v_head_dim: int | None = None,
+        d_rel: int = 16,
+        rel_extent: int = 1024,
+        local_layer_ids: list[int] | None = None,
+        sliding_window_size: int = 512,
+        swa_num_attention_heads: int | None = None,
+        swa_num_key_value_heads: int | None = None,
+        swa_head_dim: int | None = None,
+        swa_v_head_dim: int | None = None,
+        rms_norm_eps: float = 1e-6,
+        hidden_act: str = "silu",
+        q_bias: bool = False,
+        o_bias: bool = False,
+        use_embed_norm: bool = False,
+        use_sconv: bool = False,
+        sconv_kernel_size: int = 4,
+        dense_mlp_idx: int = 0,
+        n_routed_experts: int = 0,
+        n_shared_experts: int = 0,
+        num_experts_per_tok: int = 1,
+        route_scale: float = 1.0,
+        use_gate_bias: bool = False,
+        use_global_scale: bool = False,
+        norm_after_topk: bool = True,
+        gate_activation: Literal["sigmoid", "softmax"] = "sigmoid",
+        shared_expert_sink: bool = False,
+        shared_experts_size: int = 1,
+        inference_moe_w13_interleaved: bool = True,
+        log_scaling_n_floor: int | None = None,
+        log_scaling_alpha: float = 0.1,
+        unpadded_vocab_size: int | None = None,
+        padded_vocab_size: int | None = None,
+        logits_mup_width_multiplier: float | None = None,
+        final_logit_softcapping: float | None = None,
+        tie_word_embeddings: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        if head_dim is None:
+            head_dim = hidden_size // num_attention_heads
+        if v_head_dim is None:
+            v_head_dim = head_dim
+        if swa_num_attention_heads is None:
+            swa_num_attention_heads = num_attention_heads
+        if swa_num_key_value_heads is None:
+            swa_num_key_value_heads = num_key_value_heads
+        if swa_head_dim is None:
+            swa_head_dim = head_dim
+        if swa_v_head_dim is None:
+            swa_v_head_dim = swa_head_dim
+        if dense_intermediate_size is None:
+            dense_intermediate_size = intermediate_size
+        if local_layer_ids is None:
+            local_layer_ids = []
+
+        if padded_vocab_size is None:
+            padded_vocab_size = vocab_size
+            vocab_size = (
+                unpadded_vocab_size
+                if (
+                    unpadded_vocab_size is not None
+                    and unpadded_vocab_size < padded_vocab_size
+                )
+                else vocab_size
+            )
+
+        self.vocab_size = vocab_size
+        self.padded_vocab_size = padded_vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.dense_intermediate_size = dense_intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.head_dim = head_dim
+        self.v_head_dim = v_head_dim
+        self.d_rel = d_rel
+        self.rel_extent = rel_extent
+        self.local_layer_ids = local_layer_ids
+        self.sliding_window_size = sliding_window_size
+        self.swa_num_attention_heads = swa_num_attention_heads
+        self.swa_num_key_value_heads = swa_num_key_value_heads
+        self.swa_head_dim = swa_head_dim
+        self.swa_v_head_dim = swa_v_head_dim
+        self.rms_norm_eps = rms_norm_eps
+        self.hidden_act = hidden_act
+        self.q_bias = q_bias
+        self.o_bias = o_bias
+        self.use_embed_norm = use_embed_norm
+        self.use_sconv = use_sconv
+        self.sconv_kernel_size = sconv_kernel_size
+        self.dense_mlp_idx = dense_mlp_idx
+        self.n_routed_experts = n_routed_experts
+        self.num_experts = n_routed_experts
+        self.n_shared_experts = n_shared_experts
+        self.num_shared_experts = n_shared_experts
+        self.num_experts_per_tok = num_experts_per_tok
+        self.route_scale = route_scale
+        self.use_gate_bias = use_gate_bias
+        self.use_global_scale = use_global_scale
+        self.norm_after_topk = norm_after_topk
+        self.gate_activation = gate_activation
+        self.shared_expert_sink = shared_expert_sink
+        self.shared_experts_size = shared_experts_size
+        self.inference_moe_w13_interleaved = inference_moe_w13_interleaved
+        self.log_scaling_n_floor = log_scaling_n_floor
+        self.log_scaling_alpha = log_scaling_alpha
+        self.unpadded_vocab_size = self.vocab_size
+        self.logits_mup_width_multiplier = logits_mup_width_multiplier
+        self.final_logit_softcapping = final_logit_softcapping
+
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+    @property
+    def conv_layer_ids(self) -> list[int]:
+        return list(range(self.num_hidden_layers))
+
+    @property
+    def linear_layer_ids(self) -> list[int]:
+        return self.conv_layer_ids
+
+    @property
+    def full_attention_layer_ids(self) -> list[int]:
+        return list(range(self.num_hidden_layers))
+
+    @property
+    def mamba_chunk_size(self) -> int:
+        # Floor at 64: mamba_cache_chunk_size = max(mamba_chunk_size, page_size),
+        # and a floor of 1 lets the radix tree adopt another request's KV at
+        # tiny shared prefixes, whose different kernel-rounding perturbs decode
+        # logits.
+        return 64
+
+    @property
+    def mamba2_cache_params(self) -> TMLConvCacheParams | None:
+        try:
+            from vllm.distributed import get_tensor_model_parallel_world_size
+
+            tp_size = get_tensor_model_parallel_world_size()
+        except (AssertionError, RuntimeError):
+            tp_size = 1
+
+        def tp_local_kv_conv_dim(num_kv_heads: int, head_dim: int) -> int:
+            return max(1, num_kv_heads // tp_size) * head_dim
+
+        full_kv_conv_dim = tp_local_kv_conv_dim(self.num_key_value_heads, self.head_dim)
+        local_kv_conv_dim = tp_local_kv_conv_dim(
+            self.swa_num_key_value_heads, self.swa_head_dim
+        )
+        stream_dim = self.hidden_size
+        conv_len = self.sconv_kernel_size - 1
+        shape = TMLConvStateShape(
+            conv=[
+                (conv_len, full_kv_conv_dim),
+                (conv_len, full_kv_conv_dim),
+                (conv_len, local_kv_conv_dim),
+                (conv_len, local_kv_conv_dim),
+                (conv_len, stream_dim),
+                (conv_len, stream_dim),
+            ],
+            temporal=(0, 0, 0),
+        )
+        dtype = TMLStateDType(conv=torch.bfloat16, temporal=torch.bfloat16)
+        return TMLConvCacheParams(shape=shape, layers=self.conv_layer_ids, dtype=dtype)
+
+
+class InklingAudioConfig(PretrainedConfig):
+    model_type = "inkling_audio_model"
+
+    def __init__(
+        self,
+        *,
+        decoder_dmodel: int | None = None,
+        n_mel_bins: int | None = None,
+        mel_vocab_size: int | None = None,
+        dmel_min_value: float | None = None,
+        dmel_max_value: float | None = None,
+        use_audio_norm: bool | None = None,
+        audio_mode: Literal["dmel", "flow"] | None = None,
+        **kwargs: Any,
+    ) -> None:
+        values = {
+            "n_mel_bins": n_mel_bins,
+            "mel_vocab_size": mel_vocab_size,
+            "dmel_min_value": dmel_min_value,
+            "dmel_max_value": dmel_max_value,
+            "use_audio_norm": use_audio_norm,
+            "audio_mode": audio_mode,
+        }
+        if decoder_dmodel is not None and (
+            missing := [name for name, value in values.items() if value is None]
+        ):
+            raise ValueError(
+                "Enabled Inkling audio tower is missing config fields: "
+                + ", ".join(missing)
+            )
+        self.decoder_dmodel = decoder_dmodel
+        self.n_mel_bins = cast(int, n_mel_bins)
+        self.mel_vocab_size = cast(int, mel_vocab_size)
+        self.dmel_min_value = cast(float, dmel_min_value)
+        self.dmel_max_value = cast(float, dmel_max_value)
+        self.use_audio_norm = cast(bool, use_audio_norm)
+        self.audio_mode = cast(Literal["dmel", "flow"], audio_mode)
+        super().__init__(**kwargs)
+
+
+class InklingVisionConfig(PretrainedConfig):
+    model_type = "inkling_vision_model"
+
+    def __init__(
+        self,
+        *,
+        vision_encoder_type: Literal["linear", "hmlp"] | None = None,
+        decoder_dmodel: int | None = None,
+        patch_size: int | None = None,
+        temporal_patch_size: int | None = None,
+        n_channels: int | None = None,
+        n_layers: int | None = None,
+        use_vision_norm: bool | None = None,
+        **kwargs: Any,
+    ) -> None:
+        values = {
+            "vision_encoder_type": vision_encoder_type,
+            "patch_size": patch_size,
+            "temporal_patch_size": temporal_patch_size,
+            "n_channels": n_channels,
+            "n_layers": n_layers,
+            "use_vision_norm": use_vision_norm,
+        }
+        if decoder_dmodel is not None and (
+            missing := [name for name, value in values.items() if value is None]
+        ):
+            raise ValueError(
+                "Enabled Inkling vision tower is missing config fields: "
+                + ", ".join(missing)
+            )
+        self.vision_encoder_type = cast(Literal["linear", "hmlp"], vision_encoder_type)
+        self.decoder_dmodel = decoder_dmodel
+        self.patch_size = cast(int, patch_size)
+        self.temporal_patch_size = cast(int, temporal_patch_size)
+        self.n_channels = cast(int, n_channels)
+        self.n_layers = cast(int, n_layers)
+        self.use_vision_norm = cast(bool, use_vision_norm)
+        super().__init__(**kwargs)
+
+
+class InklingMMConfig(PretrainedConfig):
+    model_type = "inkling_mm_model"
+    keys_to_ignore_at_inference = ["past_key_values"]
+    sub_configs: ClassVar[dict[str, type[PretrainedConfig]]] = {
+        "text_config": InklingModelConfig,
+        "audio_config": InklingAudioConfig,
+        "vision_config": InklingVisionConfig,
+    }
+
+    def __init__(
+        self,
+        *,
+        text_config: dict[str, Any] | InklingModelConfig | None = None,
+        audio_config: dict[str, Any] | InklingAudioConfig | None = None,
+        vision_config: dict[str, Any] | InklingVisionConfig | None = None,
+        tie_word_embeddings: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        self.text_config = (
+            text_config
+            if isinstance(text_config, InklingModelConfig)
+            else InklingModelConfig(**(text_config or {}))
+        )
+        self.audio_config = (
+            audio_config
+            if isinstance(audio_config, InklingAudioConfig)
+            else InklingAudioConfig(**(audio_config or {}))
+        )
+        self.vision_config = (
+            vision_config
+            if isinstance(vision_config, InklingVisionConfig)
+            else InklingVisionConfig(**(vision_config or {}))
+        )
+        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+
+    def get_text_config(self, *args: Any, **kwargs: Any) -> InklingModelConfig:
+        return self.text_config
+
+    @property
+    def vocab_size(self) -> int:
+        return self.text_config.vocab_size
+
+    @property
+    def hidden_size(self) -> int:
+        return self.text_config.hidden_size
+
+    @property
+    def num_hidden_layers(self) -> int:
+        return self.text_config.num_hidden_layers
+
+    @property
+    def num_attention_heads(self) -> int:
+        return self.text_config.num_attention_heads
+
+    @property
+    def num_key_value_heads(self) -> int:
+        return self.text_config.num_key_value_heads
+
+    @property
+    def head_dim(self) -> int:
+        return self.text_config.head_dim
+
+    @property
+    def full_attention_layer_ids(self) -> list[int]:
+        return self.text_config.full_attention_layer_ids
+
+    @property
+    def linear_layer_ids(self) -> list[int]:
+        return self.text_config.linear_layer_ids
+
+    @property
+    def conv_layer_ids(self) -> list[int]:
+        return self.text_config.conv_layer_ids
+
+    @property
+    def mamba_chunk_size(self) -> int:
+        return self.text_config.mamba_chunk_size
+
+    @property
+    def mamba2_cache_params(self) -> TMLConvCacheParams | None:
+        return self.text_config.mamba2_cache_params
+
+
+@dataclass(kw_only=True, frozen=True)
+class TMLConvStateShape:
+    conv: list[tuple[int, int]]
+    temporal: tuple[int, int, int]
+
+
+@dataclass(kw_only=True, frozen=True)
+class TMLStateDType:
+    conv: torch.dtype = torch.bfloat16
+    temporal: torch.dtype = torch.bfloat16
+
+
+@dataclass(kw_only=True, frozen=True)
+class TMLConvCacheParams:
+    shape: TMLConvStateShape
+    layers: list[int]
+    dtype: TMLStateDType = field(default_factory=TMLStateDType)

vllm/models/inkling/nvfp4.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""NVFP4 (ModelOpt) support for the Inkling mixture-of-experts.
+
+Only the routed MoE experts are quantized in the Inkling checkpoint;
+attention, the dense MLP, and the shared "sink" experts stay bf16 (they are
+in the checkpoint ``exclude_modules``). The routed experts are served by
+vLLM's standard ModelOpt NVFP4 fused-MoE stack (see ``moe.py``); this module
+keeps the checkpoint detection.
+"""
+
+from __future__ import annotations
+
+FLOAT8_E4M3_MAX = 448.0
+FLOAT4_E2M1_MAX = 6.0
+
+
+class InklingNvfp4Config:
+    """Lightweight NVFP4 descriptor parsed from the checkpoint quant config.
+
+    Holds the (mapped) ``exclude_modules`` so the model can decide, per MoE
+    layer and per expert group, whether the weights are NVFP4 or plain bf16.
+    """
+
+    def __init__(self, group_size: int, exclude_modules: list[str]) -> None:
+        self.group_size = group_size
+        self.exclude_modules = set(exclude_modules)
+
+    @staticmethod
+    def _is_nvfp4(quant_cfg: dict) -> bool:
+        wq = quant_cfg["modelopt_quant_config"]["quant_cfg"]["*weight_quantizer"]
+        return tuple(wq["num_bits"]) == (2, 1) and tuple(
+            wq["block_sizes"].get("scale_bits", [])
+        ) == (4, 3)
+
+    @classmethod
+    def from_hf_config(cls, hf_config) -> InklingNvfp4Config | None:
+        quant_cfg = getattr(hf_config, "quantization_config", None)
+        text_config = getattr(hf_config, "text_config", None)
+        if quant_cfg is None and text_config is not None:
+            quant_cfg = getattr(text_config, "quantization_config", None)
+        if quant_cfg is None:
+            return None
+        # ModelOpt <=0.29 nests everything under "quantization".
+        if "quantization" in quant_cfg:
+            quant_cfg = quant_cfg["quantization"]
+        if not cls._is_nvfp4(quant_cfg):
+            return None
+        group_size = quant_cfg.get("group_size", 16)
+        if group_size != 16:
+            raise ValueError("Inkling NVFP4 only supports group size 16")
+        exclude = list(quant_cfg.get("exclude_modules", []) or [])
+        return cls(group_size=group_size, exclude_modules=exclude)
+
+    def experts_quantized(self, layer_id: int) -> bool:
+        """Whether the routed experts of ``layer_id`` are NVFP4 (vs excluded)."""
+        return f"model.llm.layers.{layer_id}.mlp.experts" not in self.exclude_modules
+
+    def shared_experts_quantized(self, layer_id: int) -> bool:
+        """Whether the shared sink experts of ``layer_id`` are NVFP4."""
+        return (
+            f"model.llm.layers.{layer_id}.mlp.shared_experts"
+            not in self.exclude_modules
+        )

vllm/models/inkling/nvidia/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

vllm/models/inkling/nvidia/attention.py

@@ -0,0 +1,325 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from __future__ import annotations
+
+from typing import cast
+
+import torch
+from torch import nn
+
+from vllm.config import VllmConfig, get_current_vllm_config
+from vllm.distributed import get_tensor_model_parallel_world_size
+from vllm.forward_context import get_forward_context
+from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
+from vllm.model_executor.layers.linear import (
+    MergedColumnParallelLinear,
+    RowParallelLinear,
+)
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.utils.torch_utils import (
+    canonicalize_singleton_dim_strides,
+    kv_cache_dtype_str_to_dtype,
+)
+from vllm.v1.attention.backend import AttentionBackend
+from vllm.v1.attention.backends.flash_attn import (
+    FlashAttentionBackend,
+    FlashAttentionMetadata,
+)
+from vllm.v1.kv_cache_interface import (
+    FullAttentionSpec,
+    KVCacheSpec,
+    SlidingWindowSpec,
+)
+
+from ..configs import InklingModelConfig
+from .layernorm import InklingRMSNorm
+from .ops.fa4_rel_attention import (
+    bucket_max_seqlen_q,
+    inkling_fa4_num_splits,
+    inkling_fa4_rel_attention,
+)
+from .ops.fa4_warmup import InklingFA4WarmupConfig, register_fa4_warmup
+from .ops.qkvr_prep import fused_qkvr_prep
+from .sconv_swa_attn import _K, _V, InklingConvState, InklingSconvMetadata
+from .short_conv import InklingShortConv
+
+
+def compute_log_scaling_tau(
+    positions: torch.Tensor, n_floor: int, alpha: float
+) -> torch.Tensor:
+    effective_n = (positions + 1).to(torch.float32)
+    return 1.0 + alpha * torch.log(torch.clamp(effective_n / float(n_floor), min=1.0))
+
+
+class RelLogitsProj(nn.Module):
+    """Project the per-head relative branch ``r`` to per-distance logits."""
+
+    def __init__(self, d_rel: int, rel_extent: int) -> None:
+        super().__init__()
+        self.d_rel = d_rel
+        self.rel_extent = rel_extent
+        self.proj = nn.Parameter(torch.empty(d_rel, rel_extent), requires_grad=False)
+
+    def forward(self, r_out: torch.Tensor) -> torch.Tensor:
+        # r_out: (T, num_heads, d_rel) -> (T, num_heads, rel_extent)
+        return torch.einsum("thd,de->the", r_out, self.proj)
+
+
+class InklingAttention(nn.Module, AttentionLayerBase):
+    def __init__(
+        self,
+        config: InklingModelConfig,
+        *,
+        num_heads: int,
+        num_kv_heads: int,
+        head_dim: int,
+        rel_extent: int,
+        local_extent: int,
+        is_local: bool,
+        prefix: str,
+        quant_config: QuantizationConfig | None = None,
+        conv_owner: InklingConvState,
+    ) -> None:
+        super().__init__()
+        self.prefix = prefix
+        self.is_local = is_local
+        self.hidden_size = config.hidden_size
+        self.head_dim = head_dim
+        self.d_rel = config.d_rel
+        self.log_scaling_n_floor = config.log_scaling_n_floor
+        self.log_scaling_alpha = config.log_scaling_alpha
+        # q/k are per-head RMS-normed (unit norm), so Inkling scales by 1/head_dim.
+        self.scaling = 1.0 / head_dim
+
+        tp_size = get_tensor_model_parallel_world_size()
+        self.num_total_heads = num_heads
+        self.num_total_kv_heads = num_kv_heads
+        assert self.num_total_heads % tp_size == 0
+        self.num_heads = self.num_total_heads // tp_size
+        if self.num_total_kv_heads >= tp_size:
+            assert self.num_total_kv_heads % tp_size == 0
+        else:
+            assert tp_size % self.num_total_kv_heads == 0
+        self.num_kv_heads = max(1, self.num_total_kv_heads // tp_size)
+        # When tp_size > num_kv_heads the K/V projections are padded up to
+        # tp_size heads so each rank gets at least one (GQA replication).
+        kv_total_for_sizing = max(self.num_total_kv_heads, tp_size)
+
+        self.qkvr = MergedColumnParallelLinear(
+            input_size=config.hidden_size,
+            output_sizes=[
+                head_dim * self.num_total_heads,
+                head_dim * kv_total_for_sizing,
+                head_dim * kv_total_for_sizing,
+                self.d_rel * self.num_total_heads,
+            ],
+            bias=config.q_bias,
+            quant_config=quant_config,
+            prefix=f"{prefix}.qkvr",
+        )
+        self.wo_ud = RowParallelLinear(
+            input_size=head_dim * self.num_total_heads,
+            output_size=config.hidden_size,
+            bias=config.o_bias,
+            quant_config=quant_config,
+            # reduce_results=False: the partial output is all-reduced below
+            # (one-shot custom AR) so the attention-output sconv can run on the
+            # full hidden width fused with the residual add + rmsnorm.
+            reduce_results=False,
+            prefix=f"{prefix}.wo_ud",
+        )
+        self.rel_extent = local_extent if is_local else rel_extent
+        self.local_extent = local_extent if is_local else None
+        self.rel_logits_proj = RelLogitsProj(self.d_rel, self.rel_extent)
+        self.q_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps)
+        self.k_norm = InklingRMSNorm(head_dim, eps=config.rms_norm_eps)
+
+        # Short convolution on the K/V streams (per-head-width, TP sharded),
+        # applied after the qkvr projection and before q/k norm.
+        kv_conv_dim = self.num_kv_heads * head_dim
+        self.conv_owner = conv_owner
+        self.k_sconv = InklingShortConv(
+            kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_K
+        )
+        self.v_sconv = InklingShortConv(
+            kv_conv_dim, config.sconv_kernel_size, owner=conv_owner, stream_idx=_V
+        )
+
+        # FA4 left/right window; right=0 keeps it causal. local_extent-1 mirrors
+        # the source (sliding_window_size - 1).
+        self.window_size: tuple[int, int] = (
+            (local_extent - 1, 0) if is_local else (-1, -1)
+        )
+        # Static per-layer-type KV length bound for the split heuristic: local
+        # layers never see more than the sliding window.
+        vllm_config = get_current_vllm_config()
+        self._max_kv_len = (
+            local_extent if is_local else vllm_config.model_config.max_model_len
+        )
+
+        # ---- KV-cache wiring (reuse FlashAttentionBackend for metadata) ----
+        cache_config = vllm_config.cache_config
+        self.kv_cache_dtype = (
+            cache_config.cache_dtype if cache_config is not None else "auto"
+        )
+        self.kv_cache_torch_dtype = kv_cache_dtype_str_to_dtype(
+            self.kv_cache_dtype, vllm_config.model_config
+        )
+        self.register_buffer("k_scale", torch.ones((), dtype=torch.float32))
+        self.register_buffer("v_scale", torch.ones((), dtype=torch.float32))
+
+        compilation_config = vllm_config.compilation_config
+        if prefix in compilation_config.static_forward_context:
+            raise ValueError(f"Duplicate layer name: {prefix}")
+        compilation_config.static_forward_context[prefix] = self
+        self.kv_cache = torch.tensor([])  # replaced by bind_kv_cache
+
+        register_fa4_warmup(
+            InklingFA4WarmupConfig(
+                num_heads=self.num_heads,
+                num_kv_heads=self.num_kv_heads,
+                head_dim=self.head_dim,
+                rel_extent=self.rel_extent,
+                window_size=self.window_size,
+                is_local=self.is_local,
+                max_kv_len=self._max_kv_len,
+                dtype=vllm_config.model_config.dtype,
+                kv_dtype=self.kv_cache_torch_dtype,
+                block_size=vllm_config.cache_config.block_size,
+                max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
+                max_num_batched_tokens=(
+                    vllm_config.scheduler_config.max_num_batched_tokens
+                ),
+            )
+        )
+
+    def get_attn_backend(self) -> type[AttentionBackend]:
+        return FlashAttentionBackend
+
+    def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
+        block_size = vllm_config.cache_config.block_size
+        if self.is_local:
+            assert self.local_extent is not None
+            return SlidingWindowSpec(
+                block_size=block_size,
+                num_kv_heads=self.num_kv_heads,
+                head_size=self.head_dim,
+                dtype=self.kv_cache_torch_dtype,
+                sliding_window=self.local_extent,
+            )
+        return FullAttentionSpec(
+            block_size=block_size,
+            num_kv_heads=self.num_kv_heads,
+            head_size=self.head_dim,
+            dtype=self.kv_cache_torch_dtype,
+        )
+
+    def _split_kv_cache(self) -> tuple[torch.Tensor, torch.Tensor]:
+        key_cache, value_cache = self.kv_cache.transpose(1, 2).split(
+            self.head_dim, dim=-1
+        )
+        return (
+            canonicalize_singleton_dim_strides(key_cache),
+            canonicalize_singleton_dim_strides(value_cache),
+        )
+
+    def forward(
+        self,
+        positions: torch.Tensor,
+        hidden_states: torch.Tensor,
+        log_scaling: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        num_tokens = hidden_states.shape[0]
+        qkvr, _ = self.qkvr(hidden_states)
+
+        attn_metadata = get_forward_context().attn_metadata
+        attn_output = torch.empty(
+            (num_tokens, self.num_heads, self.head_dim),
+            dtype=qkvr.dtype,
+            device=qkvr.device,
+        )
+        if not isinstance(attn_metadata, dict):
+            attn_output.zero_()
+        else:
+            conv_meta = attn_metadata[self.conv_owner.prefix]
+            md = attn_metadata[self.prefix]
+            assert isinstance(conv_meta, InklingSconvMetadata)
+            fa_md = cast(FlashAttentionMetadata, md)
+            assert self.kv_cache.numel() > 0
+            assert self.conv_owner.kv_cache.numel() > 0
+            # One launch: K/V sconv (conv-cache insert + conv + residual),
+            # Q/K per-head rmsnorm, and the attention KV-cache write. K/V are
+            # consumed via the KV cache; only normed q is materialized.
+            key_cache, value_cache = self._split_kv_cache()
+            off_k, _ = self.conv_owner.stream_ranges[_K]
+            off_v, _ = self.conv_owner.stream_ranges[_V]
+            q, rel_logits = fused_qkvr_prep(
+                qkvr,
+                self.k_sconv.weight.squeeze(1),
+                self.v_sconv.weight.squeeze(1),
+                self.q_norm.weight,
+                self.k_norm.weight,
+                self.rel_logits_proj.proj,
+                self.q_norm.variance_epsilon,
+                self.num_heads,
+                self.num_kv_heads,
+                self.head_dim,
+                self.d_rel,
+                self.conv_owner.kv_cache,
+                key_cache,
+                value_cache,
+                positions,
+                conv_meta.block_table,
+                conv_meta.seq_idx,
+                conv_meta.slot_mapping,
+                conv_meta.query_start,
+                fa_md.slot_mapping,
+                off_k,
+                off_v,
+                self.conv_owner.block_size,
+                log_scaling if not self.is_local else None,
+            )
+            q = q.view(num_tokens, self.num_heads, self.head_dim)
+            self._attention(q, rel_logits, attn_output)
+
+        flat = attn_output.view(num_tokens, -1)
+        output, _ = self.wo_ud(flat)
+        return output
+
+    def _attention(
+        self,
+        q: torch.Tensor,
+        rel_logits: torch.Tensor,
+        output: torch.Tensor,
+    ) -> None:
+        attn_metadata = get_forward_context().attn_metadata
+        assert isinstance(attn_metadata, dict)
+        md = cast(FlashAttentionMetadata, attn_metadata[self.prefix])
+
+        nt = md.num_actual_tokens
+        key_cache, value_cache = self._split_kv_cache()
+        max_seqlen_q = bucket_max_seqlen_q(md.max_query_len)
+        num_splits = inkling_fa4_num_splits(
+            is_local=self.is_local,
+            batch_size=md.seq_lens.shape[0],
+            max_query_len=max_seqlen_q,
+            num_heads=self.num_heads,
+            num_kv_heads=self.num_kv_heads,
+            max_kv_len=self._max_kv_len,
+        )
+        inkling_fa4_rel_attention(
+            q[:nt],
+            key_cache,
+            value_cache,
+            block_table=md.block_table,
+            cache_seqlens=md.seq_lens,
+            cu_seqlens_q=md.query_start_loc,
+            max_seqlen_q=max_seqlen_q,
+            softmax_scale=self.scaling,
+            causal=True,
+            window_size=self.window_size,
+            rel_extent=self.rel_extent,
+            rel_logits=rel_logits[:nt],
+            num_splits=num_splits,
+            out=output[:nt],
+        )

vllm/models/inkling/nvidia/layernorm.py

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling RMSNorm (no bias, weight-scaled), backed by the vendored Triton kernel."""
+
+from __future__ import annotations
+
+import torch
+from torch import nn
+
+from .ops import rmsnorm
+
+
+class InklingRMSNorm(nn.Module):
+    def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+        self.hidden_size = hidden_size
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if x.numel() == 0:
+            return x
+        original_shape = x.shape
+        x_2d = x.contiguous().view(-1, self.hidden_size)
+        y = rmsnorm(x_2d, self.weight, self.variance_epsilon)
+        return y.view(original_shape)

vllm/models/inkling/nvidia/logits_processor.py

@@ -0,0 +1,73 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling logits processor with muP output scaling."""
+
+from __future__ import annotations
+
+import torch
+
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+
+
+class InklingLogitsProcessor(LogitsProcessor):
+    """``LogitsProcessor`` that applies Inkling's muP logits width multiplier.
+
+    Args:
+        vocab_size: Padded vocabulary size.
+        org_vocab_size: Unpadded vocabulary size (defaults to ``vocab_size``).
+        scale: Base logits scale (kept ``1.0`` for the served checkpoint).
+        logits_as_input: Whether the input is already logits.
+        soft_cap: Optional logit soft cap (``None`` for the served checkpoint).
+        logits_mup_width_multiplier: muP width divisor for the final logits;
+            ``None`` or ``0`` disables it.
+    """
+
+    def __init__(
+        self,
+        vocab_size: int,
+        org_vocab_size: int | None = None,
+        scale: float = 1.0,
+        logits_as_input: bool = False,
+        soft_cap: float | None = None,
+        logits_mup_width_multiplier: float | None = None,
+    ) -> None:
+        super().__init__(
+            vocab_size=vocab_size,
+            org_vocab_size=org_vocab_size,
+            scale=scale,
+            logits_as_input=logits_as_input,
+            soft_cap=soft_cap,
+        )
+        self.logits_mup_width_multiplier = logits_mup_width_multiplier
+        self._logits_zero: torch.Tensor | None = None
+
+    def forward(
+        self,
+        lm_head: VocabParallelEmbedding,
+        hidden_states: torch.Tensor,
+        embedding_bias: torch.Tensor | None = None,
+    ) -> torch.Tensor | None:
+        mup = self.logits_mup_width_multiplier
+        if not mup:
+            return super().forward(lm_head, hidden_states, embedding_bias)
+        # Fold the muP width divisor into the lm_head GEMM alpha (fp32 epilogue):
+        # no separate elementwise kernel, no bf16 rounding of scaled logits, and
+        # no weight mutation. Overfit to the served checkpoint: bf16 lm_head, no
+        # soft cap, unit logits scale.
+        assert self.soft_cap is None
+        assert self.scale == 1.0
+        w = lm_head.weight
+        if self._logits_zero is None:
+            self._logits_zero = w.new_zeros(1)
+        logits = torch.addmm(
+            self._logits_zero,
+            hidden_states,
+            w.t(),
+            beta=0.0,
+            alpha=1.0 / mup,
+        )
+        logits = self._gather_logits(logits)
+        if logits is not None:
+            logits = logits[..., : self.org_vocab_size]
+        return logits

vllm/models/inkling/nvidia/mlp.py

@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling dense SwiGLU MLP (also used as the MoE shared expert).
+
+The checkpoint stores the gate/up projection as a single fused, *interleaved*
+weight (``[gate0, up0, gate1, up1, ...]``), so we use a plain
+``ColumnParallelLinear`` whose contiguous row-sharding keeps each gate/up pair
+together, and an interleaved SwiGLU activation.
+"""
+
+from __future__ import annotations
+
+import torch
+from torch import nn
+
+from vllm.model_executor.layers.linear import (
+    ColumnParallelLinear,
+    RowParallelLinear,
+)
+from vllm.model_executor.layers.quantization import QuantizationConfig
+
+
+class InklingDenseMLP(nn.Module):
+    def __init__(
+        self,
+        hidden_size: int,
+        intermediate_size: int,
+        *,
+        use_global_scale: bool = False,
+        quant_config: QuantizationConfig | None = None,
+        prefix: str = "",
+    ) -> None:
+        super().__init__()
+        self.gate_up_proj = ColumnParallelLinear(
+            hidden_size,
+            2 * intermediate_size,
+            bias=False,
+            quant_config=quant_config,
+            prefix=f"{prefix}.gate_up_proj",
+        )
+        self.down_proj = RowParallelLinear(
+            intermediate_size,
+            hidden_size,
+            bias=False,
+            quant_config=quant_config,
+            reduce_results=False,
+            prefix=f"{prefix}.down_proj",
+        )
+        if use_global_scale:
+            self.global_scale = nn.Parameter(torch.empty(1), requires_grad=False)
+        else:
+            self.global_scale = None
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        from .ops import silu_and_mul_triton
+
+        gate_up, _ = self.gate_up_proj(x)
+        x = silu_and_mul_triton(gate_up)
+        x, _ = self.down_proj(x)
+        if self.global_scale is not None:
+            x = x * self.global_scale
+        # TP-partial output: the layer's reduce-scatter fallback consumes it.
+        return x

vllm/models/inkling/nvidia/model.py

@@ -0,0 +1,678 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling model implementation for NVIDIA GPUs."""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+from typing import Any
+
+import regex as re
+import torch
+from torch import nn
+
+from vllm.config import VllmConfig
+from vllm.distributed import (
+    get_pp_group,
+    get_tensor_model_parallel_rank,
+    get_tensor_model_parallel_world_size,
+    tensor_model_parallel_all_gather,
+    tensor_model_parallel_reduce_scatter,
+)
+from vllm.forward_context import get_forward_context
+from vllm.model_executor.layers.quantization import QuantizationConfig
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
+from vllm.model_executor.models.interfaces import (
+    MultiModalEmbeddings,
+    SupportsMultiModal,
+    SupportsPP,
+)
+from vllm.model_executor.models.utils import (
+    AutoWeightsLoader,
+    WeightsMapper,
+    make_empty_intermediate_tensors_factory,
+    make_layers,
+    maybe_prefix,
+)
+from vllm.models.inkling.common.mm_preprocess import (
+    InklingDummyInputsBuilder,
+    InklingMultiModalProcessor,
+    InklingProcessingInfo,
+    inkling_audio_enabled,
+    inkling_vision_enabled,
+)
+from vllm.models.inkling.common.towers import InklingAudio, InklingVision
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.sequence import IntermediateTensors
+
+from ..configs import InklingMMConfig, InklingModelConfig
+from ..nvfp4 import InklingNvfp4Config
+from .attention import InklingAttention, compute_log_scaling_tau
+from .layernorm import InklingRMSNorm
+from .logits_processor import InklingLogitsProcessor
+from .mlp import InklingDenseMLP
+from .moe import InklingMoE
+from .ops.lamport import get_lamport_rs_conv, initialize_lamport_rs_conv
+from .ops.norm import add_rmsnorm, embed_rmsnorm
+from .sconv_swa_attn import _ATTN, _MLP, InklingConvState, InklingSconvMetadata
+from .short_conv import InklingShortConv
+
+
+def _layer_id(name: str) -> int | None:
+    m = re.search(r"\.layers\.(\d+)\.", name)
+    return int(m.group(1)) if m else None
+
+
+def _sconv_add_norm(
+    delta: torch.Tensor,
+    hidden: torch.Tensor,
+    sconv: InklingShortConv,
+    norm: InklingRMSNorm | None,
+    positions: torch.Tensor,
+) -> tuple[torch.Tensor | None, torch.Tensor]:
+    """``h = hidden + sconv(TP-sum(delta)); y = rmsnorm(h)``.
+
+    The Lamport path performs reduce-scatter + shard sconv + all-gather +
+    residual add + norm. The NCCL path handles unsupported configurations."""
+    attn_metadata = get_forward_context().attn_metadata
+    m = (
+        attn_metadata.get(sconv.owner.prefix)
+        if isinstance(attn_metadata, dict)
+        else None
+    )
+    cache = sconv.owner.kv_cache
+    off_s, ws = sconv.owner.stream_ranges[sconv.stream_idx]
+    norm_w = norm.weight if norm is not None else None
+    eps = norm.variance_epsilon if norm is not None else 0.0
+
+    mm = get_lamport_rs_conv(hidden.shape[-1], sconv.kernel_size)
+    if mm is not None and mm.usable(delta.shape[0]) and m is not None:
+        assert cache.numel() > 0
+        assert isinstance(m, InklingSconvMetadata)
+        return mm.rs_sconv_ag_add_norm(
+            delta,
+            hidden,
+            sconv.weight.squeeze(1),
+            norm_w,
+            eps,
+            cache,
+            positions,
+            m.block_table,
+            m.seq_idx,
+            m.slot_mapping,
+            off_s,
+            ws,
+            sconv.owner.block_size,
+        )
+
+    # Fallback: NCCL RS -> shard sconv -> AG -> fused add(+rmsnorm).
+    shard = tensor_model_parallel_reduce_scatter(delta, dim=-1)
+    shard = sconv(shard.contiguous(), positions)
+    full = tensor_model_parallel_all_gather(shard, dim=-1)
+    if norm is None:
+        return None, hidden + full
+    return add_rmsnorm(hidden, full, norm_w, eps)
+
+
+class InklingDecoderLayer(nn.Module):
+    def __init__(
+        self,
+        config: InklingModelConfig,
+        layer_id: int,
+        is_local: bool,
+        quant_config: QuantizationConfig | None,
+        prefix: str,
+        nvfp4_config: InklingNvfp4Config | None = None,
+    ) -> None:
+        super().__init__()
+        # Per-layer owner of the conv state as a paged SWA cache. The 4 sconv
+        # streams (K/V/attn/mlp) are packed head-major into one block and share
+        # it. Built first so the attention layer can wire its K/V sconv to it.
+        self.conv_state = InklingConvState(
+            num_kv_heads=(
+                config.swa_num_key_value_heads
+                if is_local
+                else config.num_key_value_heads
+            ),
+            head_dim=config.swa_head_dim if is_local else config.head_dim,
+            hidden_size=config.hidden_size,
+            kernel_size=config.sconv_kernel_size,
+            prefix=f"{prefix}.conv_state",
+        )
+        self.attn_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.attn = InklingAttention(
+            config,
+            num_heads=(
+                config.swa_num_attention_heads
+                if is_local
+                else config.num_attention_heads
+            ),
+            num_kv_heads=(
+                config.swa_num_key_value_heads
+                if is_local
+                else config.num_key_value_heads
+            ),
+            head_dim=config.swa_head_dim if is_local else config.head_dim,
+            rel_extent=config.rel_extent,
+            local_extent=config.sliding_window_size,
+            is_local=is_local,
+            prefix=f"{prefix}.attn",
+            quant_config=quant_config,
+            conv_owner=self.conv_state,
+        )
+        self.mlp_norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        if layer_id < config.dense_mlp_idx:
+            self.mlp: nn.Module = InklingDenseMLP(
+                hidden_size=config.hidden_size,
+                intermediate_size=config.dense_intermediate_size,
+                use_global_scale=config.use_global_scale,
+                quant_config=quant_config,
+                prefix=f"{prefix}.mlp",
+            )
+        else:
+            # InklingMoE decides per layer (from the checkpoint exclude list)
+            # whether the routed experts are NVFP4 or bf16; the shared sink
+            # experts are always bf16.
+            self.mlp = InklingMoE(
+                config,
+                layer_id,
+                prefix=f"{prefix}.mlp",
+                nvfp4_config=nvfp4_config,
+            )
+
+        # Short convolution on the attention-output and MLP-output residual
+        # streams, hidden-sharded: the sublayer outputs are reduce-scattered
+        # to [T, H/tp], the sconv runs on the shard, and an all-gather
+        # restores the full residual — all fused with the residual add + next
+        # rmsnorm via the Lamport P2P kernels for decode-sized batches.
+        tp_size = get_tensor_model_parallel_world_size()
+        sconv_dim = config.hidden_size // tp_size
+        self.attn_sconv = InklingShortConv(
+            sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_ATTN
+        )
+        self.mlp_sconv = InklingShortConv(
+            sconv_dim, config.sconv_kernel_size, owner=self.conv_state, stream_idx=_MLP
+        )
+
+    def forward(
+        self,
+        positions: torch.Tensor,
+        hidden_states: torch.Tensor,
+        pending: tuple[torch.Tensor | None, InklingShortConv] | None = None,
+        attn_in: torch.Tensor | None = None,
+        log_scaling: torch.Tensor | None = None,
+    ) -> tuple[torch.Tensor, tuple[torch.Tensor | None, InklingShortConv]]:
+        # The previous sublayer's (pre-reduce, pre-sconv) delta is folded in
+        # fused with its RS/sconv/AG and this layer's pre-attention rmsnorm.
+        # A None delta means the partials sit in the NVLS symm buffer.
+        if pending is None:
+            if attn_in is None:
+                # First layer; on the text path attn_norm comes fused with
+                # the embedding gather (chain_weight in embed_rmsnorm).
+                attn_in = self.attn_norm(hidden_states)
+        else:
+            attn_in, hidden_states = _sconv_add_norm(
+                pending[0], hidden_states, pending[1], self.attn_norm, positions
+            )
+        attn_output = self.attn(positions, attn_in, log_scaling)
+        mlp_in, hidden_states = _sconv_add_norm(
+            attn_output, hidden_states, self.attn_sconv, self.mlp_norm, positions
+        )
+        mlp_output = self.mlp(mlp_in)
+        # The caller folds mlp_output (pre-reduce, pre-sconv) into the next
+        # fused sconv+add+rmsnorm.
+        return hidden_states, (mlp_output, self.mlp_sconv)
+
+
+class InklingReplicatedEmbedding(nn.Module):
+    """Full-vocab embedding table replicated on every TP rank.
+
+    Trades the full table per rank (~2.3 GiB at V=201k / H=6144 bf16, vs a
+    1/tp shard) for no masked lookup or per-lookup TP all-reduce, and keeps the
+    full table on-rank for the fused gather+norm kernel. Bit-exact vs
+    vocab-parallel: the all-reduce there only ever summed one real row against
+    exact zeros. The LM head stays vocab-sharded.
+    """
+
+    def __init__(self, num_embeddings: int, embedding_dim: int) -> None:
+        super().__init__()
+        self.weight = nn.Parameter(
+            torch.empty(num_embeddings, embedding_dim, dtype=torch.get_default_dtype()),
+            requires_grad=False,
+        )
+
+    def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return embed_rmsnorm(input_ids, self.weight, None, 0.0)
+
+
+class InklingModel(nn.Module):
+    def __init__(
+        self,
+        *,
+        config: InklingModelConfig,
+        quant_config: QuantizationConfig | None,
+        prefix: str,
+        nvfp4_config: InklingNvfp4Config | None = None,
+    ) -> None:
+        super().__init__()
+        self.config = config
+        self.embed_tokens = InklingReplicatedEmbedding(
+            config.padded_vocab_size, config.hidden_size
+        )
+        self.embed_norm = (
+            InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+            if config.use_embed_norm
+            else None
+        )
+        local_ids = set(config.local_layer_ids)
+
+        def get_layer(prefix: str) -> InklingDecoderLayer:
+            idx = _layer_id(prefix + ".") or int(prefix.split(".")[-1])
+            return InklingDecoderLayer(
+                config, idx, idx in local_ids, quant_config, prefix, nvfp4_config
+            )
+
+        self.start_layer, self.end_layer, self.layers = make_layers(
+            config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
+        )
+        self.norm = InklingRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+        self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
+            ["hidden_states"], config.hidden_size
+        )
+
+    def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+        # Row gather + embed_norm in one launch.
+        norm = self.embed_norm
+        return embed_rmsnorm(
+            input_ids,
+            self.embed_tokens.weight,
+            norm.weight if norm is not None else None,
+            norm.variance_epsilon if norm is not None else 0.0,
+        )
+
+    def forward(
+        self,
+        input_ids: torch.Tensor | None,
+        positions: torch.Tensor,
+        intermediate_tensors: IntermediateTensors | None,
+        inputs_embeds: torch.Tensor | None = None,
+    ) -> torch.Tensor | IntermediateTensors:
+        attn_in0: torch.Tensor | None = None
+        if get_pp_group().is_first_rank:
+            if inputs_embeds is not None:
+                # embed_norm was already applied when producing inputs_embeds.
+                hidden_states = inputs_embeds
+            else:
+                # Gather + embed_norm + the first layer's attn_norm, one launch.
+                norm = self.embed_norm
+                hidden_states, attn_in0 = embed_rmsnorm(
+                    input_ids,
+                    self.embed_tokens.weight,
+                    norm.weight if norm is not None else None,
+                    self.config.rms_norm_eps,
+                    chain_weight=self.layers[self.start_layer].attn_norm.weight,
+                )
+        else:
+            assert intermediate_tensors is not None
+            hidden_states = intermediate_tensors["hidden_states"]
+        hidden_states = hidden_states.view(-1, hidden_states.shape[-1])
+        log_scaling = None
+        if self.config.log_scaling_n_floor is not None:
+            log_scaling = compute_log_scaling_tau(
+                positions,
+                self.config.log_scaling_n_floor,
+                self.config.log_scaling_alpha,
+            )
+
+        pending: tuple[torch.Tensor | None, InklingShortConv] | None = None
+        for layer in self.layers[self.start_layer : self.end_layer]:
+            hidden_states, pending = layer(
+                positions,
+                hidden_states,
+                pending=pending,
+                attn_in=attn_in0,
+                log_scaling=log_scaling,
+            )
+            attn_in0 = None
+
+        if not get_pp_group().is_last_rank:
+            if pending is not None:
+                hidden_states = _sconv_add_norm(
+                    pending[0], hidden_states, pending[1], None, positions
+                )[1]
+            return IntermediateTensors({"hidden_states": hidden_states})
+        if pending is not None:
+            # Final RS/sconv/AG + residual add fused with the final rmsnorm.
+            norm_out = _sconv_add_norm(
+                pending[0], hidden_states, pending[1], self.norm, positions
+            )[0]
+            assert norm_out is not None
+            return norm_out
+        return self.norm(hidden_states)
+
+
+class _TmlForCausalLMBase(nn.Module, SupportsPP):
+    """Shared text-backbone causal-LM scaffolding for both entry classes."""
+
+    hf_to_vllm_mapper = WeightsMapper(
+        orig_to_new_substr={
+            ".w13_dn": ".gate_up_proj",
+            ".w2_md": ".down_proj",
+        },
+        orig_to_new_stacked={
+            ".attn.wq_du.": (".attn.qkvr.", 0),
+            ".attn.wk_dv.": (".attn.qkvr.", 1),
+            ".attn.wv_dv.": (".attn.qkvr.", 2),
+            ".attn.wr_du.": (".attn.qkvr.", 3),
+        },
+        orig_to_new_prefix={
+            "model.llm.layers.": "model.layers.",
+            "model.llm.embed_norm": "model.embed_norm",
+            "model.llm.embed": "model.embed_tokens",
+            "model.llm.norm": "model.norm",
+            "model.llm.unembed": "lm_head",
+        },
+        orig_to_new_suffix={
+            # NVFP4 scale
+            ".w13_weight.scale": ".w13_weight_scale",
+            ".w13_weight.scale2": ".w13_weight_scale_2",
+            ".w2_weight.scale": ".w2_weight_scale",
+            ".w2_weight.scale2": ".w2_weight_scale_2",
+        },
+    )
+
+    def _build(
+        self,
+        vllm_config: VllmConfig,
+        text_config: InklingModelConfig,
+        prefix: str,
+    ) -> None:
+        quant_config = vllm_config.quant_config
+        self.config = text_config
+        # NVFP4 experts are detected directly from the checkpoint quant config;
+        # only the MoE experts are quantized (attention/dense MLP stay bf16).
+        self.nvfp4_config = InklingNvfp4Config.from_hf_config(
+            vllm_config.model_config.hf_config
+        )
+        # Read by the MRV2 runner to publish per-request short-conv metadata.
+        # Short convolution is intrinsic to Inkling, so this is always set.
+        self.uses_sconv = True
+        self.model = InklingModel(
+            config=text_config,
+            quant_config=quant_config,
+            prefix=maybe_prefix(prefix, "model"),
+            nvfp4_config=self.nvfp4_config,
+        )
+        initialize_lamport_rs_conv(
+            text_config.hidden_size,
+            text_config.sconv_kernel_size,
+            vllm_config.scheduler_config.max_num_batched_tokens,
+        )
+        self.lm_head = ParallelLMHead(
+            text_config.padded_vocab_size,
+            text_config.hidden_size,
+            org_num_embeddings=text_config.padded_vocab_size,
+            quant_config=quant_config,
+            prefix=maybe_prefix(prefix, "lm_head"),
+        )
+        self.logits_processor = InklingLogitsProcessor(
+            text_config.padded_vocab_size,
+            org_vocab_size=text_config.vocab_size,
+            soft_cap=text_config.final_logit_softcapping,
+            logits_mup_width_multiplier=text_config.logits_mup_width_multiplier,
+        )
+        self.make_empty_intermediate_tensors = (  # type: ignore[method-assign]
+            self.model.make_empty_intermediate_tensors
+        )
+
+    def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
+        return self.model.embed_input_ids(input_ids)
+
+    def forward(
+        self,
+        input_ids: torch.Tensor | None,
+        positions: torch.Tensor,
+        intermediate_tensors: IntermediateTensors | None = None,
+        inputs_embeds: torch.Tensor | None = None,
+        **kwargs: object,
+    ) -> torch.Tensor | IntermediateTensors:
+        return self.model(
+            input_ids,
+            positions,
+            intermediate_tensors,
+            inputs_embeds,
+        )
+
+    def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
+        return self.logits_processor(self.lm_head, hidden_states)
+
+    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+        return _load_inkling_weights(self, weights, self.config)
+
+
+class InklingForCausalLM(_TmlForCausalLMBase):
+    """Text-only entry point (``inkling_model`` checkpoints)."""
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
+        super().__init__()
+        self._build(vllm_config, vllm_config.model_config.hf_config, prefix)
+
+
+@MULTIMODAL_REGISTRY.register_processor(
+    InklingMultiModalProcessor,
+    info=InklingProcessingInfo,
+    dummy_inputs=InklingDummyInputsBuilder,
+)
+class InklingForConditionalGeneration(_TmlForCausalLMBase, SupportsMultiModal):
+    """Top-level (multimodal) entry point.
+
+    Builds the vision + audio towers on top of the shared text backbone. Inkling has
+    NO cross-modal fusion (the vision tower emits one token per patch, the audio
+    tower one token per frame), so generation reuses the inherited backbone
+    ``forward`` / ``compute_logits`` (the latter already applies muP) and this
+    class only adds multimodal embedding + merge.
+    """
+
+    hf_to_vllm_mapper = _TmlForCausalLMBase.hf_to_vllm_mapper | WeightsMapper(
+        orig_to_new_prefix={
+            "model.audio.": "audio.",
+            "model.visual.": "visual.vision_encoder.",
+        },
+    )
+
+    @classmethod
+    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
+        if modality.startswith("image"):
+            return "<|content_image|>"
+        if modality.startswith("audio"):
+            return "<|content_audio_input|>"
+        raise ValueError("Only image or audio modality is supported")
+
+    def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
+        super().__init__()
+        config: InklingMMConfig = vllm_config.model_config.hf_config
+
+        self.visual = (
+            InklingVision(config.vision_config, prefix=maybe_prefix(prefix, "visual"))
+            if inkling_vision_enabled(config)
+            else None
+        )
+        self.audio = (
+            InklingAudio(config.audio_config, prefix=maybe_prefix(prefix, "audio"))
+            if inkling_audio_enabled(config)
+            else None
+        )
+
+        self._build(vllm_config, config.text_config, prefix)
+
+    # -- multimodal embedding -------------------------------------------
+
+    def _process_image_input(
+        self, pixel_values: Any, num_patches: Any
+    ) -> tuple[torch.Tensor, ...]:
+        assert self.visual is not None
+        # pixel_values is a list (per item) of [P_i, 2, P, P, 3] tensors,
+        # or a single concatenated tensor. Normalize to a flat batch, run the
+        # tower once, then split back per item.
+        if isinstance(pixel_values, (list, tuple)):
+            if not pixel_values:
+                return ()
+            sizes = [int(p.shape[0]) for p in pixel_values]
+            patches = torch.cat(list(pixel_values), dim=0)
+        else:
+            patches = pixel_values
+            sizes = self._sizes_from(num_patches, patches.shape[0])
+
+        patches = patches.to(device=self.visual.device, dtype=self.visual.dtype)
+        embeds = self.visual(patches)  # [total_patches, D]
+        return tuple(embeds.split(sizes))
+
+    def _process_audio_input(
+        self, input_audio_features: Any, num_audio_tokens: Any
+    ) -> tuple[torch.Tensor, ...]:
+        assert self.audio is not None
+        if isinstance(input_audio_features, (list, tuple)):
+            if not input_audio_features:
+                return ()
+            sizes = [int(d.shape[0]) for d in input_audio_features]
+            dmel = torch.cat(list(input_audio_features), dim=0)
+        else:
+            dmel = input_audio_features
+            sizes = self._sizes_from(num_audio_tokens, dmel.shape[0])
+
+        dmel = dmel.to(device=self.audio.device)
+        embeds = self.audio(dmel)  # [total_frames, D]
+        return tuple(embeds.split(sizes))
+
+    @staticmethod
+    def _sizes_from(counts: Any, total: int) -> list[int]:
+        if counts is None:
+            return [total]
+        if isinstance(counts, torch.Tensor):
+            return [int(c) for c in counts.flatten().tolist()]
+        if isinstance(counts, (list, tuple)):
+            flat: list[int] = []
+            for c in counts:
+                flat.append(int(c.item()) if isinstance(c, torch.Tensor) else int(c))
+            return flat
+        return [int(counts)]
+
+    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
+        # Iterate modalities in a stable order so the returned per-item tensors
+        # line up with their appearance order; the positional merge in
+        # embed_input_ids handles actual placement.
+        pixel_values = kwargs.get("pixel_values")
+        num_patches = kwargs.get("num_patches")
+        input_audio_features = kwargs.get("input_audio_features")
+        num_audio_tokens = kwargs.get("num_audio_tokens")
+
+        embeddings: tuple[torch.Tensor, ...] = ()
+        if pixel_values is not None and self.visual is not None:
+            embeddings += self._process_image_input(pixel_values, num_patches)
+        if input_audio_features is not None and self.audio is not None:
+            embeddings += self._process_audio_input(
+                input_audio_features, num_audio_tokens
+            )
+        return embeddings
+
+    def embed_input_ids(
+        self,
+        input_ids: torch.Tensor,
+        multimodal_embeddings: MultiModalEmbeddings | None = None,
+        *,
+        is_multimodal: torch.Tensor | None = None,
+    ) -> torch.Tensor:
+        # Override the base's 1-arg embed_input_ids: the runner calls this 3-arg
+        # signature for multimodal models. Text embeddings come from the shared
+        # backbone (which applies embed_norm); MM embeddings are scattered in.
+        from vllm.model_executor.models.utils import _merge_multimodal_embeddings
+
+        # Placeholder ids use unused vocabulary slots and these positions are
+        # overwritten by MM embeds below.
+        inputs_embeds = self.model.embed_input_ids(input_ids)
+        if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
+            return inputs_embeds
+        assert is_multimodal is not None
+        return _merge_multimodal_embeddings(
+            inputs_embeds=inputs_embeds,
+            multimodal_embeddings=multimodal_embeddings,
+            is_multimodal=is_multimodal,
+        )
+
+    def get_language_model(self) -> nn.Module:
+        # This class IS the causal LM (the towers are side branches), so the
+        # language model is self — callers expect a module exposing ``.model``
+        # and ``.lm_head``.
+        return self
+
+
+# ===========================================================================
+# Weight loading
+# ===========================================================================
+
+
+_MOE_EXPERT_WEIGHT_RE = re.compile(
+    r"^(?P<mlp>.*\.mlp)\.(?P<rest>(?:shared_)?experts\..+)$"
+)
+
+
+def _load_inkling_weights(
+    module: nn.Module,
+    weights: Iterable[tuple[str, torch.Tensor]],
+    config: InklingModelConfig,
+) -> set[str]:
+    moe_modules = {
+        name: mod for name, mod in module.named_modules() if isinstance(mod, InklingMoE)
+    }
+    loaded: set[str] = set()
+    tp_size = get_tensor_model_parallel_world_size()
+    tp_rank = get_tensor_model_parallel_rank()
+    local_ids = set(config.local_layer_ids)
+
+    def _iter_loadable_weights() -> Iterable[tuple[str, torch.Tensor]]:
+        for name, weight in module.hf_to_vllm_mapper.apply(weights):
+            shard_id = getattr(weight, "shard_id", None)
+            # Replicate K/V conv-free GQA heads when tp_size > num_kv_heads.
+            if (
+                shard_id in (1, 2)
+                and name.endswith(".attn.qkvr.weight")
+                and weight.shape[0] > 0
+            ):
+                lid = _layer_id(name)
+                if lid is not None:
+                    is_local = lid in local_ids
+                    n_kv = (
+                        config.swa_num_key_value_heads
+                        if is_local
+                        else config.num_key_value_heads
+                    )
+                    head_dim = config.swa_head_dim if is_local else config.head_dim
+                    if tp_size > n_kv and weight.shape[0] == n_kv * head_dim:
+                        kv_idx = (tp_rank * n_kv) // tp_size
+                        weight = weight.narrow(0, kv_idx * head_dim, head_dim)
+                        weight.shard_id = shard_id
+
+            # MoE expert tensors (fused stacked, routed + shared sink): translate
+            # the checkpoint layout to per-expert FusedMoE loads.
+            moe_match = _MOE_EXPERT_WEIGHT_RE.match(name)
+            if moe_match is not None and moe_match.group("mlp") in moe_modules:
+                moe = moe_modules[moe_match.group("mlp")]
+                for rel in moe.load_expert_weight(moe_match.group("rest"), weight):
+                    loaded.add(f"{moe_match.group('mlp')}.{rel}")
+                continue
+
+            yield name, weight
+
+    # The release checkpoint also carries auxiliary prediction-head weights;
+    # they are not part of the causal LM served by this implementation.
+    loader = AutoWeightsLoader(module, skip_prefixes=["model.mtp."])
+    loaded |= loader.load_weights(_iter_loadable_weights())
+
+    # Post-load MoE fixups (default input scales, zeroed EP-padding experts).
+    for moe_name, moe in moe_modules.items():
+        for rel in moe.finalize_load():
+            loaded.add(f"{moe_name}.{rel}")
+    return loaded
+
+
+EntryClass = [InklingForCausalLM, InklingForConditionalGeneration]

vllm/models/inkling/nvidia/moe.py

@@ -0,0 +1,578 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling mixture-of-experts on vLLM's FusedMoE abstraction.
+
+Overfit to the served checkpoint: sigmoid gate (+ selection bias) top-k over
+the routed experts, log-sigmoid renormalization over the k routed + S shared
+"sink" logits, scaled by route_scale * global_scale. The routed top-k goes
+through vLLM's FusedMoE (which handles TP/EP); the sink experts run in
+:class:`InklingSinkExperts` -- replicated across EP ranks (every token
+activates every sink) and always bf16 (the checkpoint excludes every
+``shared_experts`` from quantization).
+
+NVFP4 routed experts reuse vLLM's ModelOpt NVFP4 fused-MoE method; excluded
+(bf16) layers fall back to the unquantized method. The checkpoint's fused
+stacked tensors (interleaved gate/up rows, ``.scale`` / ``.scale2`` /
+``.input_amax`` aux tensors) are translated to the standard per-expert loads
+in :meth:`InklingMoE.load_expert_weight`.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import TYPE_CHECKING
+
+import torch
+from torch import nn
+from torch.nn.parameter import Parameter
+
+import vllm.envs as envs
+from vllm.config import get_current_vllm_config
+from vllm.distributed import (
+    get_dp_group,
+    get_pcp_group,
+    get_tensor_model_parallel_rank,
+    get_tensor_model_parallel_world_size,
+)
+from vllm.model_executor.kernels.linear.cute_dsl import ll_bf16
+from vllm.model_executor.layers.fused_moe import FusedMoE
+from vllm.model_executor.utils import set_weight_attrs
+from vllm.platforms import current_platform
+from vllm.triton_utils import tl, tldevice, triton
+from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
+from vllm.utils.torch_utils import aux_stream
+
+from ..configs import InklingModelConfig
+from ..nvfp4 import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
+
+if TYPE_CHECKING:
+    from vllm.model_executor.layers.fused_moe.routed_experts import (
+        RoutedExperts,
+    )
+
+    from ..nvfp4 import InklingNvfp4Config
+
+# ---------------------------------------------------------------------------
+# Gate / expert selection
+# ---------------------------------------------------------------------------
+
+_INKLING_LL_BF16_MAX_TOKENS = 64
+
+
+def _linear_with_fp32_out(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor:
+    leading = list(x.shape[:-1])
+    flat = x.flatten(0, -2)
+    if (
+        flat.shape[0] <= _INKLING_LL_BF16_MAX_TOKENS
+        and flat.dtype == torch.bfloat16
+        and weight.dtype == torch.bfloat16
+        and flat.is_cuda
+        and flat.is_contiguous()
+        and weight.is_contiguous()
+        and flat.shape[1] % 8 == 0
+        and current_platform.has_device_capability(90)
+        and ll_bf16.is_available()
+    ):
+        out = ll_bf16.ll_bf16_gemm(flat, weight)
+    else:
+        out = torch.mm(flat, weight.T, out_dtype=torch.float32)
+    return out.view(*leading, weight.shape[0])
+
+
+@triton.jit(do_not_specialize=["T", "route_scale"])
+def _inkling_gate_select_kernel(
+    logits_ptr,  # [T, G] fp32 gate logits (stride_logits_0 may include pad)
+    bias_ptr,  # [R] fp32 selection bias (or 0 ptr if HAS_BIAS=False)
+    global_scale_ptr,  # [1] fp32 (or unused if HAS_GSCALE=False)
+    ids_ptr,  # [T, K + S] int32 out: selected expert ids
+    weights_ptr,  # [T, K + S] fp32 out: renormalized weights
+    route_scale,
+    T,
+    G: tl.constexpr,  # total gate experts (routed + shared)
+    stride_logits_0,
+    R: tl.constexpr,  # routed experts
+    K: tl.constexpr,  # top-k routed
+    S: tl.constexpr,  # shared (sink) experts
+    HAS_BIAS: tl.constexpr,
+    HAS_GSCALE: tl.constexpr,
+    BLOCK_G: tl.constexpr,
+):
+    pid = tl.program_id(0).to(tl.int64)
+    if pid >= T:
+        return
+    offs = tl.arange(0, BLOCK_G)
+    mask_r = offs < R
+    logits = tl.load(
+        logits_ptr + pid * stride_logits_0 + offs,
+        mask=offs < G,
+        other=float("-inf"),
+    ).to(tl.float32)
+
+    # Selection scores: sigmoid(routed logits) (+ bias), non-routed lanes -inf.
+    sel = tl.where(mask_r, tl.sigmoid(logits), float("-inf"))
+    if HAS_BIAS:
+        bias = tl.load(bias_ptr + offs, mask=mask_r, other=0.0).to(tl.float32)
+        sel = tl.where(mask_r, sel + bias, float("-inf"))
+
+    scale = route_scale
+    if HAS_GSCALE:
+        scale = scale * tl.load(global_scale_ptr).to(tl.float32)
+
+    # Iterative top-K (K is small); argmax tie-breaks to the lowest index
+    # (stable ordering).
+    A: tl.constexpr = K + S
+    offs_a = tl.arange(0, A)
+    top_ids = tl.zeros([A], dtype=tl.int32)
+    active = tl.zeros([A], dtype=tl.float32)
+    for kk in tl.static_range(K):
+        idx = tl.argmax(sel, axis=0).to(tl.int32)
+        raw = tl.max(tl.where(offs == idx, logits, float("-inf")), axis=0)
+        top_ids = tl.where(offs_a == kk, idx, top_ids)
+        active = tl.where(offs_a == kk, raw, active)
+        sel = tl.where(offs == idx, float("-inf"), sel)
+    if S > 0:
+        # Shared sink logits sit at the tail of the gate output; their expert
+        # ids continue after the routed range (R + j).
+        for jj in tl.static_range(S):
+            raw = tl.max(tl.where(offs == R + jj, logits, float("-inf")), axis=0)
+            top_ids = tl.where(offs_a == K + jj, tl.full([], R + jj, tl.int32), top_ids)
+            active = tl.where(offs_a == K + jj, raw, active)
+
+    # Log-sigmoid renormalization over the K + S active logits.
+    abs_l = tl.abs(active)
+    min_l = tl.minimum(active, 0.0)
+    log_probs = min_l - tldevice.log1p(tldevice.exp(-abs_l))
+    max_lp = tl.max(log_probs, axis=0)
+    exp_shifted = tldevice.exp(log_probs - max_lp)
+    sum_exp = tl.sum(exp_shifted, axis=0)
+    weights = exp_shifted / sum_exp * scale
+
+    tl.store(ids_ptr + pid * A + offs_a, top_ids)
+    tl.store(weights_ptr + pid * A + offs_a, weights)
+
+
+def inkling_gate_select(
+    logits: torch.Tensor,  # [T, >=G] fp32 (rows may carry GEMM padding)
+    n_gate_experts: int,
+    n_routed_experts: int,
+    topk: int,
+    n_shared_experts: int,
+    bias: torch.Tensor | None,
+    route_scale: float,
+    global_scale: torch.Tensor | None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Sigmoid + bias + top-k + log-sigmoid renorm; returns (weights, ids)."""
+    assert logits.dtype == torch.float32
+    tokens = logits.shape[0]
+    active = topk + n_shared_experts
+    topk_ids = torch.empty((tokens, active), dtype=torch.int32, device=logits.device)
+    topk_weights = torch.empty(
+        (tokens, active), dtype=torch.float32, device=logits.device
+    )
+    if tokens == 0:
+        return topk_weights, topk_ids
+    _inkling_gate_select_kernel[(tokens,)](
+        logits,
+        bias if bias is not None else logits,
+        global_scale if global_scale is not None else logits,
+        topk_ids,
+        topk_weights,
+        route_scale,
+        tokens,
+        n_gate_experts,
+        logits.stride(0),
+        n_routed_experts,
+        topk,
+        n_shared_experts,
+        HAS_BIAS=bias is not None,
+        HAS_GSCALE=global_scale is not None,
+        BLOCK_G=triton.next_power_of_2(n_gate_experts),
+    )
+    return topk_weights, topk_ids
+
+
+class InklingGate(nn.Module):
+    """Sigmoid gate with selection bias, log-sigmoid renorm after top-k, and
+    global scale (the served checkpoint's only configuration)."""
+
+    def __init__(
+        self,
+        d_model: int,
+        n_routed_experts: int,
+        n_shared_experts: int,
+        experts_per_token: int,
+        route_scale: float,
+        *,
+        use_global_scale: bool = False,
+        use_gate_bias: bool = False,
+    ) -> None:
+        super().__init__()
+        self.n_routed_experts = n_routed_experts
+        self.n_shared_experts = n_shared_experts
+        self.n_total_experts = n_routed_experts + n_shared_experts
+        self.topk = experts_per_token
+        self.route_scale = route_scale
+
+        padded_experts = self.n_total_experts + (-self.n_total_experts) % 8
+        self.weight = Parameter(
+            torch.empty(padded_experts, d_model), requires_grad=False
+        )
+        set_weight_attrs(self.weight, {"weight_loader": self._load_weight})
+        if use_global_scale:
+            self.global_scale = Parameter(
+                torch.empty(1, dtype=torch.float32), requires_grad=False
+            )
+        else:
+            self.global_scale = None
+        if use_gate_bias:
+            self.bias = Parameter(
+                torch.empty(n_routed_experts, dtype=torch.float32),
+                requires_grad=False,
+            )
+        else:
+            self.bias = None
+
+    @staticmethod
+    def _load_weight(param: Parameter, loaded_weight: torch.Tensor) -> None:
+        param.data.zero_()
+        param.data[: loaded_weight.shape[0]].copy_(loaded_weight)
+
+    def compute_logits(self, x: torch.Tensor) -> torch.Tensor:
+        """fp32 gate logits [T, n_total_experts + pad] (pad columns are junk)."""
+        return _linear_with_fp32_out(x, self.weight)
+
+    def select_experts(
+        self, gating_output: torch.Tensor
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Full selection: (weights, ids) of [T, K + S]. The first K entries
+        are the routed top-k; the S trailing entries are the sink gammas."""
+        return inkling_gate_select(
+            gating_output,
+            self.n_total_experts,
+            self.n_routed_experts,
+            self.topk,
+            self.n_shared_experts,
+            self.bias,
+            self.route_scale,
+            self.global_scale,
+        )
+
+
+# ---------------------------------------------------------------------------
+# MoE layer
+# ---------------------------------------------------------------------------
+
+
+def _inkling_moe_ep_size() -> int:
+    """EP size the FusedMoE layer will run with (mirrors
+    FusedMoEParallelConfig.make: experts shard over tp * dp * pcp when
+    expert parallelism is enabled)."""
+    parallel_config = get_current_vllm_config().parallel_config
+    if not parallel_config.enable_expert_parallel:
+        return 1
+    world = (
+        get_tensor_model_parallel_world_size()
+        * get_dp_group().world_size
+        * get_pcp_group().world_size
+    )
+    return world if world > 1 else 1
+
+
+class InklingSinkExperts(nn.Module):
+    """Shared "sink" experts with per-token gammas, in bf16.
+
+    Replicated across EP ranks (every token activates every sink, so
+    EP-sharding them would hotspot the owning rank) and TP-sharded on the
+    intermediate dim so the output remains a TP-partial sum like the routed
+    output. The sinks are always bf16 (the checkpoint excludes every
+    ``shared_experts`` from quantization): the experts concatenate into two
+    plain dense GEMMs with the fused sink epilogue between them.
+    """
+
+    def __init__(
+        self, n_experts: int, d_model: int, d_mlp: int, *, prefix: str = ""
+    ) -> None:
+        super().__init__()
+        self.n_experts = n_experts
+        tp_size = get_tensor_model_parallel_world_size()
+        self.tp_rank = get_tensor_model_parallel_rank()
+        intermediate_pp = d_mlp // tp_size
+        self.w13_weight = Parameter(
+            torch.empty(n_experts, 2 * intermediate_pp, d_model),
+            requires_grad=False,
+        )
+        self.w2_weight = Parameter(
+            torch.empty(d_model, n_experts * intermediate_pp),
+            requires_grad=False,
+        )
+        self._unit: torch.Tensor | None = None
+
+    def load_weight(self, key: str, weight: torch.Tensor) -> list[str]:
+        """Load one checkpoint sink tensor (stacked over the S experts)."""
+        if key == "w13_weight":
+            if weight.shape != self.w13_weight.shape:
+                shard = self.w13_weight.shape[1]
+                weight = weight.narrow(1, self.tp_rank * shard, shard)
+            self.w13_weight.data.copy_(weight)
+            return [key]
+
+        assert key == "w2_weight"
+        shard = self.w2_weight.shape[1] // self.n_experts
+        shard_start = 0 if weight.shape[2] == shard else self.tp_rank * shard
+        for expert_idx, expert_weight in enumerate(weight):
+            local_weight = expert_weight.narrow(1, shard_start, shard)
+            start = expert_idx * shard
+            self.w2_weight.data[:, start : start + shard].copy_(local_weight)
+        return [key]
+
+    def forward(self, x: torch.Tensor, gammas: torch.Tensor) -> torch.Tensor:
+        """``sum_e gammas[:, e] * MLP_e(x)`` (TP-partial along d_mlp)."""
+        from .ops import sink_silu_mul_epilogue
+
+        # One GEMM over the experts' stacked w13 (a view), fused epilogue,
+        # then one GEMM whose K-reduction over the K-concatenated w2 performs
+        # the expert sum.
+        if self._unit is None or self._unit.device != x.device:
+            self._unit = torch.ones(
+                self.n_experts, dtype=torch.float32, device=x.device
+            )
+        raw = x @ self.w13_weight.view(-1, x.shape[-1]).T  # (T, S*2F)
+        h = sink_silu_mul_epilogue(
+            raw, self._unit, gammas, self._unit, self.n_experts, x.dtype
+        )
+        return h @ self.w2_weight.T  # (T, D)
+
+
+class InklingMoE(nn.Module):
+    def __init__(
+        self,
+        config: InklingModelConfig,
+        layer_id: int,
+        *,
+        prefix: str = "",
+        nvfp4_config: InklingNvfp4Config | None = None,
+    ) -> None:
+        super().__init__()
+        # Overfit to the served checkpoint: sigmoid gate renormalized after
+        # top-k, shared sink experts, interleaved gate/up checkpoint rows.
+        assert config.gate_activation == "sigmoid" and config.norm_after_topk
+        assert config.n_shared_experts > 0 and config.shared_expert_sink
+        assert config.inference_moe_w13_interleaved
+        n_routed = config.n_routed_experts
+        n_shared = config.n_shared_experts
+        self.n_routed_experts = n_routed
+        self.gate = InklingGate(
+            d_model=config.hidden_size,
+            n_routed_experts=n_routed,
+            n_shared_experts=n_shared,
+            experts_per_token=config.num_experts_per_tok,
+            route_scale=config.route_scale,
+            use_global_scale=config.use_global_scale,
+            use_gate_bias=config.use_gate_bias,
+        )
+
+        moe_quant_config = None
+        if nvfp4_config is not None and nvfp4_config.experts_quantized(layer_id):
+            from vllm.model_executor.layers.quantization.modelopt import (
+                ModelOptNvFp4Config,
+            )
+
+            # The Inkling checkpoint is ModelOpt NVFP4; exclusion is decided per
+            # layer right here, so no exclude list is needed.
+            moe_quant_config = ModelOptNvFp4Config(
+                quant_method="NVFP4",
+                is_checkpoint_nvfp4_serialized=True,
+                kv_cache_quant_algo=None,
+                exclude_modules=[],
+                group_size=nvfp4_config.group_size,
+            )
+
+        # TRTLLM MoE kernels assume equal, contiguous per-rank expert slabs
+        # (local_expert_offset = ep_rank * local_num_experts), so pad the
+        # expert count to a multiple of the EP size. A no-op for the usual
+        # power-of-two EP sizes (n_routed is a power of two).
+        num_experts = n_routed + (-n_routed) % _inkling_moe_ep_size()
+
+        self.experts = FusedMoE(
+            num_experts=num_experts,
+            top_k=config.num_experts_per_tok,
+            hidden_size=config.hidden_size,
+            intermediate_size=config.intermediate_size,
+            renormalize=False,
+            quant_config=moe_quant_config,
+            prefix=f"{prefix}.experts",
+            custom_routing_function=self._select_routed,
+            router_logits_dtype=torch.float32,
+            activation="silu",
+        )
+        # The decoder layer reduce-scatters the MoE delta into the sconv
+        # stream itself (RS -> shard sconv -> AG); the runner must return the
+        # per-rank partial sum instead of all-reducing.
+        self.experts.moe_config.skip_final_all_reduce = True
+
+        self._routed_sel: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None
+        # The sinks are always bf16; fail loudly on a checkpoint that
+        # quantizes them instead of silently misloading.
+        assert nvfp4_config is None or not nvfp4_config.shared_experts_quantized(
+            layer_id
+        ), f"layer {layer_id}: NVFP4 shared experts are not supported"
+
+        self.sink_experts = InklingSinkExperts(
+            n_experts=n_shared,
+            d_model=config.hidden_size,
+            d_mlp=config.intermediate_size,
+            prefix=f"{prefix}.shared_experts",
+        )
+
+        # Sink chain overlaps the routed MoE call on the aux stream for
+        # decode-sized batches (same pattern as the runner's SharedExperts
+        # multi-stream overlap). The routed GEMM runs on the default stream and
+        # the sink chain on the aux stream, joined via these two events by
+        # ``maybe_execute_in_parallel``.
+        self._sink_stream: torch.cuda.Stream | None = aux_stream()
+        self._sink_events = (torch.cuda.Event(), torch.cuda.Event())
+
+    def _select_routed(
+        self,
+        hidden_states: torch.Tensor,
+        gating_output: torch.Tensor,
+        topk: int,
+        renormalize: bool,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """FusedMoE ``custom_routing_function``: the routed top-k slice of the
+        full (routed + sink) selection.
+
+        forward() stashes its selection (keyed by logits identity) so the
+        gate select runs once per layer; the fallback covers paths where the
+        runner re-derives the logits (e.g. naive DP dispatch).
+        """
+        del hidden_states, renormalize
+        assert topk == self.gate.topk
+        cached = self._routed_sel
+        self._routed_sel = None
+        if cached is not None and cached[0] is gating_output:
+            return cached[1], cached[2]
+        weights, ids = self.gate.select_experts(gating_output)
+        return weights[:, :topk].contiguous(), ids[:, :topk].contiguous()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor | None:
+        router_logits = self.gate.compute_logits(x)
+        num_tokens = x.shape[0]
+        # One gate select per layer: the routed slice is stashed for the
+        # routing function inside the FusedMoE op; the sink gammas are the
+        # trailing columns.
+        k = self.gate.topk
+        weights, ids = self.gate.select_experts(router_logits)
+        self._routed_sel = (
+            router_logits,
+            weights[:, :k].contiguous(),
+            ids[:, :k].contiguous(),
+        )
+        gammas = weights[:, k:]
+
+        out, sink_out = maybe_execute_in_parallel(
+            lambda: self.experts(hidden_states=x, router_logits=router_logits),
+            lambda: self.sink_experts(x, gammas),
+            self._sink_events[0],
+            self._sink_events[1],
+            self._sink_stream
+            if num_tokens <= envs.VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD
+            else None,
+        )
+        self._routed_sel = None
+
+        return out + sink_out
+
+    # -- weight loading ----------------------------------------------------
+
+    def _local_expert_slots(self) -> dict[int, int]:
+        """Global expert id -> local slot for this rank's expert partition."""
+        manager = self.experts.routed_experts.expert_map_manager
+        if manager.expert_map is None:
+            return {g: g for g in range(manager.global_num_experts)}
+        emap = manager.expert_map.tolist()
+        return {g: slot for g, slot in enumerate(emap) if slot >= 0}
+
+    def load_expert_weight(self, name: str, weight: torch.Tensor) -> list[str]:
+        """Load one checkpoint expert tensor.
+
+        ``name`` is relative to the mlp module: ``experts.<t>`` (routed
+        stack) or ``shared_experts.shared_<t>`` (sink experts). Returns the
+        loaded param names (relative to this module).
+        """
+        if name.startswith("shared_experts."):
+            key = name.split(".", 1)[1].replace("shared_", "", 1)
+            return [
+                f"sink_experts.{p}" for p in self.sink_experts.load_weight(key, weight)
+            ]
+
+        experts: RoutedExperts = self.experts.routed_experts
+        key = name.split(".", 1)[1]
+
+        # original_shape is unused by the vLLM serving layout.
+        if key.endswith(".original_shape"):
+            return []
+        if key.endswith(".input_amax"):
+            projection = "w13" if key.startswith("w13") else "w2"
+            amax = float(weight.max())
+            assert math.isfinite(amax) and amax > 0, (
+                f"bad {projection} input_amax: {amax}"
+            )
+            input_scale = getattr(experts, f"{projection}_input_scale")
+            input_scale.data.fill_(amax / (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX))
+            return [f"experts.routed_experts.{projection}_input_scale"]
+
+        param = getattr(experts, key)
+        slots = self._local_expert_slots()
+        gids = sorted(slots)
+        lids = [slots[g] for g in gids]
+        tp_rank = experts.moe_config.moe_parallel_config.tp_rank
+
+        if key.endswith("_scale_2"):
+            # Per-expert scalars, vectorized over the local experts. The
+            # fused w13 param carries one slot per gate/up half.
+            vals = weight[gids].float().to(param.device)
+            param.data[lids] = vals[:, None] if param.data.ndim == 2 else vals
+        elif key.startswith("w13"):
+            # Checkpoint w13 rows are interleaved [g0, u0, g1, u1, ...]; the
+            # fused param layout is [w1(gate); w3(up)]. The TP-local rows form
+            # one contiguous slab of the interleaved tensor, so upload just
+            # that slab (a single bounded synchronous H2D; pre-uploading whole
+            # untrimmed tensors pins the mmap pages of the entire checkpoint
+            # and OOMs the host) and de-interleave on device.
+            half = param.shape[1] // 2
+            for gid, lid in slots.items():
+                slab = weight[gid].narrow(0, tp_rank * 2 * half, 2 * half)
+                slab = slab.to(param.device)
+                param.data[lid, :half].copy_(slab[0::2])
+                param.data[lid, half:].copy_(slab[1::2])
+        else:
+            # w2: shard the packed intermediate (last) dim.
+            shard = param.shape[2]
+            for gid, lid in slots.items():
+                param.data[lid].copy_(weight[gid].narrow(1, tp_rank * shard, shard))
+        return [f"experts.routed_experts.{key}"]
+
+    def finalize_load(self) -> list[str]:
+        """Post-load fixups for zeroed padding experts."""
+        experts = self.experts.routed_experts
+        out: list[str] = []
+        # Zero the EP-alignment padding experts (if any) so their
+        # (never-routed) slots hold defined values.
+        slots = self._local_expert_slots()
+        for gid in range(self.n_routed_experts, experts.global_num_experts):
+            lid = slots.get(gid)
+            if lid is None:
+                continue
+            for pname in (
+                "w13_weight",
+                "w2_weight",
+                "w13_weight_scale",
+                "w2_weight_scale",
+                "w13_weight_scale_2",
+                "w2_weight_scale_2",
+            ):
+                p = getattr(experts, pname, None)
+                if p is not None:
+                    p.data[lid].zero_()
+        return out

vllm/models/inkling/nvidia/ops/__init__.py

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling kernels (NVIDIA).
+
+``rmsnorm`` / ``sconv`` import eagerly. The SwiGLU kernels and the FA4
+relative-attention wrapper are exposed lazily to keep this package's
+import path lightweight.
+"""
+
+from typing import TYPE_CHECKING
+
+from .norm import add_rmsnorm, rmsnorm
+from .sconv import fused_sconv
+
+_LAZY_EXPORTS = {
+    "silu_and_mul_triton": "silu_and_mul",
+    "sink_silu_mul_epilogue": "silu_and_mul",
+    "inkling_fa4_rel_attention": "fa4_rel_attention",
+}
+
+if TYPE_CHECKING:
+    from .fa4_rel_attention import inkling_fa4_rel_attention  # noqa: F401
+    from .silu_and_mul import (  # noqa: F401
+        silu_and_mul_triton,
+        sink_silu_mul_epilogue,
+    )
+
+__all__ = [
+    "add_rmsnorm",
+    "rmsnorm",
+    "fused_sconv",
+    *sorted(_LAZY_EXPORTS),
+]
+
+
+def __getattr__(name: str):
+    module = _LAZY_EXPORTS.get(name)
+    if module is not None:
+        import importlib
+
+        mod = importlib.import_module(f".{module}", __name__)
+        return getattr(mod, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

vllm/models/inkling/nvidia/ops/fa4_rel_attention.py

@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from __future__ import annotations
+
+import torch
+
+from vllm.platforms import current_platform
+
+
+def bucket_max_seqlen_q(max_seqlen_q: int) -> int:
+    """Round the FA4 scheduling bound up to a power of two."""
+    return 1 << max(0, max_seqlen_q - 1).bit_length()
+
+
+def inkling_fa4_num_splits(
+    *,
+    is_local: bool,
+    batch_size: int,
+    max_query_len: int,
+    num_heads: int,
+    num_kv_heads: int,
+    max_kv_len: int,
+) -> int:
+    """Return the split-KV cap for FA4 with sheared relative bias."""
+    capability = current_platform.get_device_capability()
+    if capability is not None and capability.major == 9:
+        return 1
+    if is_local:
+        return 1
+
+    q_rows = max_query_len * (num_heads // num_kv_heads)
+    q_tiles = (q_rows + 255) // 256
+    base_ctas = batch_size * num_kv_heads * q_tiles
+    # Shearing makes split/combine overhead more visible. Multi-tile causal
+    # prefill saturates around 64 CTAs. Batch-1 decode at very long context is
+    # memory-bound and uses a TP-specific cap measured through 1M KV tokens.
+    target_ctas = (
+        256 if q_tiles == 1 and batch_size == 1 else (128 if q_tiles == 1 else 64)
+    )
+    max_splits = 128
+    if q_tiles == 1 and batch_size == 1:
+        if num_kv_heads == 8:
+            max_splits = 16
+        elif num_kv_heads == 4 or max_kv_len <= 8192:
+            max_splits = 32
+        elif max_kv_len <= 65536:
+            max_splits = 64
+        else:
+            max_splits = 128
+    return max(
+        1,
+        min(target_ctas // base_ctas, max_splits, (max_kv_len + 127) // 128),
+    )
+
+
+def inkling_fa4_rel_attention(
+    q: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    *,
+    block_table: torch.Tensor,
+    cache_seqlens: torch.Tensor,
+    cu_seqlens_q: torch.Tensor,
+    max_seqlen_q: int,
+    softmax_scale: float,
+    causal: bool,
+    window_size: tuple[int, int],
+    rel_extent: int,
+    rel_logits: torch.Tensor,
+    num_splits: int = 32,
+    out: torch.Tensor | None = None,
+) -> torch.Tensor:
+    """Paged varlen FA4 over the bound K/V cache with the Inkling relative bias.
+
+    ``q`` is ``(num_tokens, num_heads, head_dim)``; ``key_cache`` / ``value_cache``
+    are the paged caches ``(num_blocks, block_size, num_kv_heads, head_dim)``;
+    ``block_table`` is the per-request page table and ``cache_seqlens`` the
+    per-request KV lengths (``seqused_k``). ``rel_logits`` is
+    ``(num_tokens, num_heads, rel_extent)``.
+
+    The bias uses tml-fa4's sheared relative-bias layout.
+    """
+    from vllm.third_party.tml_fa4 import flash_attn_varlen_func
+
+    # cute uses (None, None) to mean "no window".
+    cute_window = (None, None) if window_size == (-1, -1) else window_size
+
+    ret = flash_attn_varlen_func(
+        q=q,
+        k=key_cache,
+        v=value_cache,
+        cu_seqlens_q=cu_seqlens_q,
+        seqused_k=cache_seqlens,
+        max_seqlen_q=max_seqlen_q,
+        page_table=block_table,
+        softmax_scale=softmax_scale,
+        causal=causal,
+        window_size=cute_window,
+        num_splits=num_splits,
+        return_lse=False,
+        out=out,
+        rel_bias=rel_logits.contiguous(),
+    )
+    if isinstance(ret, tuple):
+        return ret[0]
+    return ret

vllm/models/inkling/nvidia/ops/fa4_warmup.py

@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Startup compilation of the FA4 kernels used by Inkling."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from dataclasses import dataclass
+from functools import partial
+
+import torch
+
+from vllm.model_executor.warmup.cutedsl_warmup import (
+    CuTeDSLCompileUnit,
+    register_cutedsl_warmup_provider,
+)
+
+from .fa4_rel_attention import (
+    bucket_max_seqlen_q,
+    inkling_fa4_num_splits,
+    inkling_fa4_rel_attention,
+)
+
+
+@dataclass(frozen=True)
+class InklingFA4WarmupConfig:
+    num_heads: int
+    num_kv_heads: int
+    head_dim: int
+    rel_extent: int
+    window_size: tuple[int, int]
+    is_local: bool
+    max_kv_len: int
+    dtype: torch.dtype
+    kv_dtype: torch.dtype
+    block_size: int
+    max_num_reqs: int
+    max_num_batched_tokens: int
+
+
+def _num_warps_bucket(num_reqs: int) -> int:
+    num_warps = min((num_reqs + 30) // 31, 32)
+    return 1 << (num_warps - 1).bit_length()
+
+
+def _compile(config: InklingFA4WarmupConfig, max_seqlen_q: int, num_reqs: int) -> None:
+    from torch._subclasses.fake_tensor import FakeTensorMode
+
+    num_splits = inkling_fa4_num_splits(
+        is_local=config.is_local,
+        batch_size=num_reqs,
+        max_query_len=max_seqlen_q,
+        num_heads=config.num_heads,
+        num_kv_heads=config.num_kv_heads,
+        max_kv_len=config.max_kv_len,
+    )
+    with FakeTensorMode():
+        device = torch.accelerator.current_accelerator()
+        total_q = max_seqlen_q + num_reqs - 1
+        q = torch.empty(
+            total_q,
+            config.num_heads,
+            config.head_dim,
+            dtype=config.dtype,
+            device=device,
+        )
+        kv = torch.empty(
+            1,
+            2,
+            config.block_size,
+            config.num_kv_heads,
+            config.head_dim,
+            dtype=config.kv_dtype,
+            device=device,
+        )
+        key_cache, value_cache = kv.unbind(1)
+        inkling_fa4_rel_attention(
+            q,
+            key_cache,
+            value_cache,
+            block_table=torch.empty(num_reqs, 1, dtype=torch.int32, device=device),
+            cache_seqlens=torch.empty(num_reqs, dtype=torch.int32, device=device),
+            cu_seqlens_q=torch.empty(num_reqs + 1, dtype=torch.int32, device=device),
+            max_seqlen_q=max_seqlen_q,
+            softmax_scale=1.0 / config.head_dim,
+            causal=True,
+            window_size=config.window_size,
+            rel_extent=config.rel_extent,
+            rel_logits=torch.empty(
+                total_q,
+                config.num_heads,
+                config.rel_extent,
+                dtype=config.dtype,
+                device=device,
+            ),
+            num_splits=num_splits,
+            out=torch.empty_like(q),
+        )
+
+
+def _iter_compile_units(
+    config: InklingFA4WarmupConfig,
+) -> Iterator[CuTeDSLCompileUnit]:
+    max_bucket = bucket_max_seqlen_q(config.max_num_batched_tokens)
+    max_seqlen_q = 1
+    while max_seqlen_q <= max_bucket:
+        min_query_len = max_seqlen_q // 2 + 1
+        max_num_reqs = min(
+            config.max_num_reqs,
+            config.max_num_batched_tokens - min_query_len + 1,
+        )
+        seen: set[tuple[int, int, int | None, bool]] = set()
+        for num_reqs in range(1, max_num_reqs + 1):
+            num_splits = inkling_fa4_num_splits(
+                is_local=config.is_local,
+                batch_size=num_reqs,
+                max_query_len=max_seqlen_q,
+                num_heads=config.num_heads,
+                num_kv_heads=config.num_kv_heads,
+                max_kv_len=config.max_kv_len,
+            )
+            key = (
+                max_seqlen_q,
+                num_splits,
+                _num_warps_bucket(num_reqs) if num_splits > 1 else None,
+                num_reqs > 1024,
+            )
+            if key in seen:
+                continue
+            seen.add(key)
+            yield CuTeDSLCompileUnit(
+                name="inkling_fa4",
+                key=("inkling_fa4", config, key),
+                compile=partial(_compile, config, max_seqlen_q, num_reqs),
+            )
+        max_seqlen_q *= 2
+
+
+class _WarmupProvider:
+    def __init__(self) -> None:
+        self.configs: set[InklingFA4WarmupConfig] = set()
+
+    def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]:
+        return tuple(
+            unit for config in self.configs for unit in _iter_compile_units(config)
+        )
+
+
+_PROVIDER = _WarmupProvider()
+
+
+def register_fa4_warmup(config: InklingFA4WarmupConfig) -> None:
+    _PROVIDER.configs.add(config)
+    register_cutedsl_warmup_provider(_PROVIDER)

vllm/models/inkling/nvidia/ops/lamport.py

@@ -0,0 +1,766 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Deadlock-free fused RS + short-conv + AG + residual + RMSNorm.
+
+The public integration surface is ``LamportRSConv.rs_sconv_ag_add_norm``.
+
+Liveness
+--------
+Large grids are deliberately split at the two communication dependencies:
+
+  1. ``_publish_input_kernel`` only publishes rank partials.
+  2. ``_reduce_insert_kernel`` waits, reduces, and inserts the local shard.
+  3. ``_sconv_publish_kernel`` only computes and publishes the local result.
+  4. ``_gather_norm_kernel`` waits, gathers, and normalizes.
+
+Without PDL, CUDA stream order completes a producer before its consumer.  With
+PDL, every producer CTA posts all peer stores before triggering its dependent,
+and the consumer executes ``gdc_wait`` before polling.  A consumer therefore
+waits only for stores from a producer that is already running (or complete) on
+another GPU.  It never waits for another block in its own grid.  Consequently
+spinning consumers cannot occupy resources needed by any producer, and the
+wait-for graph has no cycle.  The proof is independent of grid size, block
+dispatch order, and occupancy.
+
+For one token, the first three phases use eight independent channel slices to
+expose enough CTA parallelism for decode latency.  Each slice has exclusive
+ownership of its cache and Lamport columns; the gather/RMSNorm phase retains
+one CTA per token so no cross-CTA reduction or completion counter is needed.
+
+Immediate buffer reuse (including replay of a captured CUDA graph) is also
+safe.  Rank R cannot republish input for call n+1 until its gather for n has
+finished; that gather waited for owner O's output, which O publishes only
+after consuming R's input for n.  Likewise, R cannot republish output for n+1
+until its reduction for n+1 has observed destination D's input; D publishes
+that input only after its gather consumed R's output for n.  Thus every prior
+read happens-before a same-slot rewrite.  Three generations reduce incidental
+coupling for ordinary launches, but correctness does not depend on rotation.
+
+The payload itself is the Lamport flag.  A 32-bit store publishes two bf16s
+atomically; 0x80008000 (two negative zeroes) denotes an empty pair.  Real
+negative zeroes are changed to positive zero before publication.  Consumers
+use volatile 32-bit loads and restore the sentinel after consuming a slot.
+"""
+
+from __future__ import annotations
+
+import os
+
+import torch
+
+from vllm.distributed import get_tp_group
+from vllm.distributed.parallel_state import in_the_same_node_as
+from vllm.logger import init_logger
+from vllm.triton_utils import HAS_TRITON, tl, triton
+
+logger = init_logger(__name__)
+
+_MAX_TOKENS = 16384
+_EMPTY_PAIR = tl.constexpr(0x80008000) if HAS_TRITON else 0x80008000
+
+
+@triton.jit
+def _pack_bf16_pairs(values):
+    """Pack bf16 values into atomic u32 pairs and reserve negative zero."""
+    lo, hi = tl.split(values.reshape([values.shape[0] // 2, 2]))
+    lo = lo.to(tl.uint16, bitcast=True)
+    hi = hi.to(tl.uint16, bitcast=True)
+    lo = tl.where(lo == 0x8000, 0, lo).to(tl.uint32)
+    hi = tl.where(hi == 0x8000, 0, hi).to(tl.uint32)
+    return lo | (hi << 16)
+
+
+@triton.jit
+def _unpack_bf16_pairs(values):
+    lo = (values & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
+    hi = (values >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
+    return tl.interleave(lo, hi)
+
+
+@triton.jit
+def _wait_pairs(ptr, offsets, mask):
+    values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True)
+    while tl.max(tl.where(mask & (values == _EMPTY_PAIR), 1, 0)) != 0:
+        values = tl.load(ptr + offsets, mask=mask, other=0, volatile=True)
+    return values
+
+
+@triton.jit
+def _publish_input_kernel(
+    stage_ptr,
+    peer_ptrs,
+    peer_offset_u32,
+    stride_stage_t,
+    C: tl.constexpr,
+    CS: tl.constexpr,
+    CS_P2: tl.constexpr,
+    SPLITS: tl.constexpr,
+    RANK: tl.constexpr,
+    WORLD: tl.constexpr,
+    USE_PDL: tl.constexpr,
+    launch_pdl: tl.constexpr,
+):
+    """Publish this rank's full partial row into every shard owner's slots."""
+    token = tl.program_id(0).to(tl.int64)
+    split = tl.program_id(1).to(tl.int64)
+    CSS: tl.constexpr = CS // SPLITS
+    pair = tl.arange(0, CS_P2 // 2)
+    pair_mask = pair < CSS // 2
+    elem = tl.arange(0, CS_P2)
+    elem_mask = elem < CSS
+    ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64))
+    # Address generation is independent of the preceding stream kernel.  The
+    # acquire remains before stage/peer loads, which may consume its writes.
+    if USE_PDL:
+        tl.extra.cuda.gdc_wait()
+
+    for owner in tl.static_range(WORLD):
+        values = tl.load(
+            stage_ptr + token * stride_stage_t + owner * CS + split * CSS + elem,
+            mask=elem_mask,
+            other=0.0,
+        )
+        packed = _pack_bf16_pairs(values)
+        base = tl.load(ptrs + owner).to(tl.pointer_type(tl.uint32))
+        dst = (token * WORLD + RANK) * (CS // 2) + split * (CSS // 2) + pair
+        tl.store(base + peer_offset_u32 + dst, packed, mask=pair_mask)
+    if USE_PDL:
+        tl.extra.cuda.gdc_launch_dependents()
+
+
+@triton.jit
+def _reduce_insert_kernel(
+    input_peer_ptrs,
+    input_peer_offset_u32,
+    cache_ptr,
+    slot_ptr,
+    stride_cache_block,
+    stride_cache_head,
+    stride_cache_token,
+    stride_cache_dim,
+    block_size,
+    C: tl.constexpr,
+    CS: tl.constexpr,
+    CS_P2: tl.constexpr,
+    SPLITS: tl.constexpr,
+    HEAD_SIZE: tl.constexpr,
+    CACHE_OFFSET: tl.constexpr,
+    RANK: tl.constexpr,
+    WORLD: tl.constexpr,
+    USE_PDL: tl.constexpr,
+    launch_pdl: tl.constexpr,
+):
+    """Consume all partials for this rank and insert the reduced cache row."""
+    token = tl.program_id(0).to(tl.int64)
+    split = tl.program_id(1).to(tl.int64)
+    CSS: tl.constexpr = CS // SPLITS
+    source = tl.arange(0, WORLD)
+    pair = tl.arange(0, CS_P2 // 2)
+    pair_mask = pair < CSS // 2
+    offsets = (
+        (token * WORLD + source)[:, None] * (CS // 2)
+        + split * (CSS // 2)
+        + pair[None, :]
+    )
+    mask = tl.full([WORLD], True, tl.int1)[:, None] & pair_mask[None, :]
+    input_ptrs = input_peer_ptrs.to(tl.pointer_type(tl.uint64))
+    input_u32 = tl.load(input_ptrs + RANK).to(tl.pointer_type(tl.uint32))
+    input_u32 += input_peer_offset_u32
+    # Slot metadata and the cache destination do not depend on input publish.
+    slot = tl.load(slot_ptr + token)
+    valid = slot >= 0
+    channel = tl.arange(0, CS_P2)
+    channel_mask = channel < CSS
+    global_channel = split * CSS + channel
+    head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1)
+    dim = CACHE_OFFSET + global_channel % HEAD_SIZE
+    safe_slot = tl.maximum(slot, 0).to(tl.int64)
+    dst = (
+        cache_ptr
+        + (safe_slot // block_size) * stride_cache_block
+        + head * stride_cache_head
+        + (safe_slot % block_size) * stride_cache_token
+        + dim * stride_cache_dim
+    )
+    if USE_PDL:
+        tl.extra.cuda.gdc_wait()
+    packed = _wait_pairs(input_u32, offsets, mask)
+
+    lo = (packed & 0xFFFF).to(tl.uint16).to(tl.bfloat16, bitcast=True)
+    hi = (packed >> 16).to(tl.uint16).to(tl.bfloat16, bitcast=True)
+    reduced = tl.interleave(
+        tl.sum(lo.to(tl.float32), axis=0).to(tl.bfloat16),
+        tl.sum(hi.to(tl.float32), axis=0).to(tl.bfloat16),
+    )
+    tl.store(
+        input_u32 + offsets,
+        tl.full([WORLD, CS_P2 // 2], _EMPTY_PAIR, tl.uint32),
+        mask=mask,
+    )
+
+    tl.store(dst, reduced, mask=valid & channel_mask)
+    if USE_PDL:
+        tl.extra.cuda.gdc_launch_dependents()
+
+
+@triton.jit
+def _sconv_publish_kernel(
+    peer_ptrs,
+    peer_offset_u32,
+    residual_ptr,
+    weight_ptr,
+    cache_ptr,
+    position_ptr,
+    sequence_ptr,
+    slot_ptr,
+    block_table_ptr,
+    stride_residual_t,
+    stride_cache_block,
+    stride_cache_head,
+    stride_cache_token,
+    stride_cache_dim,
+    stride_block_table_r,
+    max_blocks,
+    block_size,
+    C: tl.constexpr,
+    CS: tl.constexpr,
+    CS_P2: tl.constexpr,
+    SPLITS: tl.constexpr,
+    HEAD_SIZE: tl.constexpr,
+    CACHE_OFFSET: tl.constexpr,
+    RANK: tl.constexpr,
+    WORLD: tl.constexpr,
+    WINDOW: tl.constexpr,
+    USE_PDL: tl.constexpr,
+    launch_pdl: tl.constexpr,
+):
+    """Compute this rank's shard, then publish it to every rank."""
+    tl.static_assert(WINDOW == 4)
+    token = tl.program_id(0).to(tl.int64)
+    split = tl.program_id(1).to(tl.int64)
+    CSS: tl.constexpr = CS // SPLITS
+    channel = tl.arange(0, CS_P2)
+    channel_mask = channel < CSS
+    global_channel = split * CSS + channel
+    head = tl.minimum(global_channel // HEAD_SIZE, CS // HEAD_SIZE - 1)
+    dim = CACHE_OFFSET + global_channel % HEAD_SIZE
+    slot = tl.load(slot_ptr + token)
+    valid = slot >= 0
+    position = tl.load(position_ptr + token)
+    sequence = tl.load(sequence_ptr + token)
+    safe_slot = tl.maximum(slot, 0).to(tl.int64)
+    own_ptr = (
+        cache_ptr
+        + (safe_slot // block_size) * stride_cache_block
+        + head * stride_cache_head
+        + (safe_slot % block_size) * stride_cache_token
+        + dim * stride_cache_dim
+    )
+    # These loads were made visible before reduce/insert could signal us and
+    # are independent of its cache store.  Hoisting them gives PDL useful work
+    # to overlap while retaining the acquire before every cache read.
+    residual = tl.load(
+        residual_ptr + token * stride_residual_t + RANK * CS + global_channel,
+        mask=channel_mask,
+        other=0.0,
+    )
+    weight0 = tl.load(
+        weight_ptr + global_channel * WINDOW,
+        mask=channel_mask,
+        other=0.0,
+    ).to(tl.float32)
+    weight1 = tl.load(
+        weight_ptr + global_channel * WINDOW + 1,
+        mask=channel_mask,
+        other=0.0,
+    ).to(tl.float32)
+    weight2 = tl.load(
+        weight_ptr + global_channel * WINDOW + 2,
+        mask=channel_mask,
+        other=0.0,
+    ).to(tl.float32)
+    weight3 = tl.load(
+        weight_ptr + global_channel * WINDOW + 3,
+        mask=channel_mask,
+        other=0.0,
+    ).to(tl.float32)
+    ptrs = peer_ptrs.to(tl.pointer_type(tl.uint64))
+    if USE_PDL:
+        tl.extra.cuda.gdc_wait()
+    current = tl.load(own_ptr, mask=valid & channel_mask, other=0.0)
+
+    conv = tl.zeros([CS_P2], tl.float32)
+    for tap_idx in tl.static_range(WINDOW):
+        source_position = position - (WINDOW - 1) + tap_idx
+        take = valid & (source_position >= 0)
+        if tap_idx == WINDOW - 1:
+            value = tl.where(take, current.to(tl.float32), 0.0)
+        else:
+            safe_position = tl.maximum(source_position, 0)
+            logical_block = tl.minimum(safe_position // block_size, max_blocks - 1)
+            physical_block = tl.load(
+                block_table_ptr + sequence * stride_block_table_r + logical_block,
+                mask=take,
+                other=0,
+            ).to(tl.int64)
+            source_ptr = (
+                cache_ptr
+                + physical_block * stride_cache_block
+                + head * stride_cache_head
+                + (safe_position % block_size) * stride_cache_token
+                + dim * stride_cache_dim
+            )
+            cached = tl.load(source_ptr, mask=take & channel_mask, other=0.0)
+            value = tl.where(take, cached.to(tl.float32), 0.0)
+        if tap_idx == 0:
+            weight = weight0
+        elif tap_idx == 1:
+            weight = weight1
+        elif tap_idx == 2:
+            weight = weight2
+        else:
+            weight = weight3
+        conv += value * weight
+
+    # Preserve both bf16 rounding points of the original sublayer.
+    short_conv_with_skip = (conv + current.to(tl.float32)).to(tl.bfloat16)
+    output = (residual.to(tl.float32) + short_conv_with_skip.to(tl.float32)).to(
+        tl.bfloat16
+    )
+
+    pair = tl.arange(0, CS_P2 // 2)
+    pair_mask = pair < CSS // 2
+    packed = _pack_bf16_pairs(output)
+    row_offset = token * (C // 2) + RANK * (CS // 2) + split * (CSS // 2) + pair
+
+    for destination in tl.static_range(WORLD):
+        base = tl.load(ptrs + destination).to(tl.pointer_type(tl.uint32))
+        tl.store(base + peer_offset_u32 + row_offset, packed, mask=pair_mask)
+    if USE_PDL:
+        tl.extra.cuda.gdc_launch_dependents()
+
+
+@triton.jit
+def _gather_norm_kernel(
+    output_peer_ptrs,
+    output_peer_offset_u32,
+    norm_weight_ptr,
+    normed_ptr,
+    residual_out_ptr,
+    eps,
+    stride_output_t,
+    C: tl.constexpr,
+    C_P2: tl.constexpr,
+    RANK: tl.constexpr,
+    HAS_NORM: tl.constexpr,
+    USE_PDL: tl.constexpr,
+    launch_pdl: tl.constexpr,
+):
+    """Consume a complete row; one CTA owns all outputs for one token."""
+    token = tl.program_id(0).to(tl.int64)
+    pair = tl.arange(0, C_P2 // 2)
+    pair_mask = pair < C // 2
+    output_ptrs = output_peer_ptrs.to(tl.pointer_type(tl.uint64))
+    output_u32 = tl.load(output_ptrs + RANK).to(tl.pointer_type(tl.uint32))
+    output_u32 += output_peer_offset_u32
+    offsets = token * (C // 2) + pair
+    channel = tl.arange(0, C_P2)
+    channel_mask = channel < C
+    # Gamma is independent of the preceding sconv publication.  Keep the
+    # acquire immediately before polling the Lamport output slots.
+    if HAS_NORM:
+        weight = tl.load(norm_weight_ptr + channel, mask=channel_mask, other=0.0)
+    if USE_PDL:
+        tl.extra.cuda.gdc_wait()
+    packed = _wait_pairs(output_u32, offsets, pair_mask)
+    row = _unpack_bf16_pairs(packed)
+    tl.store(
+        residual_out_ptr + token * stride_output_t + channel, row, mask=channel_mask
+    )
+    if HAS_NORM:
+        row_f32 = tl.where(channel_mask, row.to(tl.float32), 0.0)
+        inv_rms = tl.rsqrt(tl.sum(row_f32 * row_f32, axis=0) / C + eps)
+        tl.store(
+            normed_ptr + token * stride_output_t + channel,
+            (row_f32 * inv_rms * weight.to(tl.float32)).to(tl.bfloat16),
+            mask=channel_mask,
+        )
+    tl.store(
+        output_u32 + offsets,
+        tl.full([C_P2 // 2], _EMPTY_PAIR, tl.uint32),
+        mask=pair_mask,
+    )
+    if USE_PDL:
+        tl.extra.cuda.gdc_launch_dependents()
+
+
+@triton.jit
+def _validate_lamport_init_kernel(
+    input_peer_ptrs,
+    output_peer_ptrs,
+    bad_ptr,
+    num_pairs,
+    RANK: tl.constexpr,
+    BLOCK: tl.constexpr,
+):
+    """Validate both complete local allocations through their fabric pointers."""
+    offsets = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+    mask = offsets < num_pairs
+    ptrs_in = input_peer_ptrs.to(tl.pointer_type(tl.uint64))
+    ptrs_out = output_peer_ptrs.to(tl.pointer_type(tl.uint64))
+    local_in = tl.load(ptrs_in + RANK).to(tl.pointer_type(tl.uint32))
+    local_out = tl.load(ptrs_out + RANK).to(tl.pointer_type(tl.uint32))
+    value_in = tl.load(local_in + offsets, mask=mask, other=_EMPTY_PAIR)
+    value_out = tl.load(local_out + offsets, mask=mask, other=_EMPTY_PAIR)
+    bad = tl.max(
+        tl.where(mask & ((value_in != _EMPTY_PAIR) | (value_out != _EMPTY_PAIR)), 1, 0)
+    )
+    if bad != 0:
+        tl.atomic_max(bad_ptr, 1)
+
+
+class LamportRSConv:
+    """Persistent symmetric buffers for one TP group."""
+
+    def _initialize_lamport_buffers(self) -> None:
+        """Arm every slot and collectively verify the exact sentinel bits.
+
+        Do not replace this with a zero-fill: Lamport readiness distinguishes
+        the bf16 bit pattern for -0.0 (0x8000) from every published payload.
+        The validation is deliberately collective so one rank cannot enter the
+        first polling kernel while another rank still has an unarmed buffer.
+        """
+        if not self.buf_in.is_contiguous() or not self.buf_out.is_contiguous():
+            raise RuntimeError("Lamport symmetric buffers must be contiguous")
+        if self.buf_in.numel() % 2 or self.buf_out.numel() % 2:
+            raise RuntimeError("Lamport buffers must contain whole uint32 pairs")
+
+        # Fill through int16 so each bf16 lane receives the exact -0.0 bits.
+        # This covers all three generations and all max-token slots, including
+        # slots that a smaller first invocation does not touch.
+        self.buf_in.view(torch.int16).fill_(-0x8000)
+        self.buf_out.view(torch.int16).fill_(-0x8000)
+        torch.accelerator.synchronize(self.device)
+        self.tp.barrier()
+
+        # Scan the complete local allocations.  Since every rank performs the
+        # scan and participates in MAX, success proves every symmetric backing
+        # allocation was armed before any rank is allowed to use generation 0.
+        bad = torch.logical_or(
+            self.buf_in.view(torch.int16).ne(-0x8000).any(),
+            self.buf_out.view(torch.int16).ne(-0x8000).any(),
+        ).to(dtype=torch.float32)
+        bad = self.tp.all_reduce(bad)
+        if int(bad.item()) != 0:
+            raise RuntimeError("Lamport sentinel initialization failed on a TP rank")
+        self.tp.barrier()
+
+    def _initialize_mnnvl_buffers(self) -> None:
+        """Initialize and validate FlashInfer fabric-mapped Lamport storage."""
+        self._mnnvl_input_handle.lamport_initialize(self.rank, torch.bfloat16)
+        self._mnnvl_output_handle.lamport_initialize(self.rank, torch.bfloat16)
+        torch.accelerator.synchronize(self.device)
+        self.tp.barrier()
+
+        bad = torch.zeros((), dtype=torch.int32, device=self.device)
+        num_pairs = self.num_buffers * self.max_tokens * self.hidden_size // 2
+        _validate_lamport_init_kernel[(triton.cdiv(num_pairs, 256),)](
+            self.input_peer_ptrs,
+            self.output_peer_ptrs,
+            bad,
+            num_pairs,
+            RANK=self.rank,
+            BLOCK=256,
+            num_warps=4,
+        )
+        bad = self.tp.all_reduce(bad.to(torch.float32))
+        if int(bad.item()) != 0:
+            raise RuntimeError("MNNVL Lamport sentinel initialization failed")
+        self.tp.barrier()
+
+    def __init__(
+        self, hidden_size: int, window_size: int, max_tokens: int = _MAX_TOKENS
+    ) -> None:
+        import torch.distributed._symmetric_memory as symm_mem
+
+        tp = get_tp_group()
+        self.tp = tp
+        self.group = tp.device_group
+        self.world_size = tp.world_size
+        self.rank = tp.rank_in_group
+        self.device = torch.device(tp.device)
+        if self.world_size not in (2, 4, 8):
+            raise ValueError(f"TP world size must be 2, 4, or 8, got {self.world_size}")
+        if hidden_size % (2 * self.world_size) != 0:
+            raise ValueError("hidden size must produce an even shard on every rank")
+        if window_size != 4:
+            raise ValueError(f"short-conv window size must be 4, got {window_size}")
+        if max_tokens < 1 or max_tokens > _MAX_TOKENS:
+            raise ValueError(f"max_tokens must be in [1, {_MAX_TOKENS}]")
+
+        is_cross_node = not all(in_the_same_node_as(tp.cpu_group))
+        self.hidden_size = hidden_size
+        self.window_size = window_size
+        self.max_tokens = max_tokens
+        self.shard_size = hidden_size // self.world_size
+        # Three generations follow FlashInfer's Lamport layout.  A generation
+        # is reused only after two intervening collective calls.
+        self.num_buffers = 3
+        self.input_generation_bytes = max_tokens * hidden_size * 2
+        self.output_generation_bytes = max_tokens * hidden_size * 2
+        self.use_pdl = torch.cuda.get_device_capability(self.device)[0] >= 9
+        if is_cross_node:
+            try:
+                from flashinfer.comm.mnnvl import (
+                    McastGPUBuffer,
+                    TorchDistBackend,
+                    is_mnnvl_fabric_supported,
+                )
+            except ImportError as error:
+                raise RuntimeError(
+                    "cross-node TP requires FlashInfer MNNVL support"
+                ) from error
+
+            local_supported = int(
+                is_mnnvl_fabric_supported(torch.accelerator.current_device_index())
+            )
+            unsupported = torch.tensor(
+                1 - local_supported, dtype=torch.float32, device=self.device
+            )
+            unsupported = tp.all_reduce(unsupported)
+            if int(unsupported.item()) != 0:
+                raise RuntimeError("cross-node TP is supported only on MNNVL fabric")
+
+            comm_backend = TorchDistBackend(self.group)
+            allocation_bytes = self.num_buffers * max_tokens * hidden_size * 2
+            self._mnnvl_input_handle = McastGPUBuffer(
+                allocation_bytes,
+                self.world_size,
+                self.rank,
+                self.device,
+                comm_backend,
+            )
+            self._mnnvl_output_handle = McastGPUBuffer(
+                allocation_bytes,
+                self.world_size,
+                self.rank,
+                self.device,
+                comm_backend,
+            )
+            self.input_peer_ptrs = self._mnnvl_input_handle.get_buffer_ptrs_dev()
+            self.output_peer_ptrs = self._mnnvl_output_handle.get_buffer_ptrs_dev()
+            self._initialize_mnnvl_buffers()
+            logger.info("using FlashInfer fabric-mapped MNNVL Lamport buffers")
+        else:
+            self.buf_in = symm_mem.empty(
+                self.num_buffers,
+                max_tokens,
+                self.world_size,
+                self.shard_size,
+                dtype=torch.bfloat16,
+                device=self.device,
+            )
+            self.buf_out = symm_mem.empty(
+                self.num_buffers,
+                max_tokens,
+                hidden_size,
+                dtype=torch.bfloat16,
+                device=self.device,
+            )
+            group_name = self.group.group_name
+            input_handle = symm_mem.rendezvous(self.buf_in, group_name)
+            output_handle = symm_mem.rendezvous(self.buf_out, group_name)
+            self.input_peer_ptrs = input_handle.buffer_ptrs_dev
+            self.output_peer_ptrs = output_handle.buffer_ptrs_dev
+            self._input_handle = input_handle
+            self._output_handle = output_handle
+            self._initialize_lamport_buffers()
+        self.generation = 0
+
+    def usable(self, num_tokens: int) -> bool:
+        return 0 < num_tokens <= self.max_tokens
+
+    def rs_sconv_ag_add_norm(
+        self,
+        input_tensor: torch.Tensor,
+        residual: torch.Tensor,
+        conv_weight: torch.Tensor,
+        norm_weight: torch.Tensor | None,
+        eps: float,
+        cache: torch.Tensor,
+        positions: torch.Tensor,
+        block_table: torch.Tensor,
+        seq_idx: torch.Tensor,
+        slot_mapping: torch.Tensor,
+        off_s: int,
+        ws: int,
+        block_size: int,
+    ) -> tuple[torch.Tensor | None, torch.Tensor]:
+        """Return ``(normed | None, new_residual)``, both shaped ``[T, 6144]``."""
+        tokens, hidden_size = residual.shape
+        if not self.usable(tokens):
+            raise ValueError(f"num_tokens must be in [1, {self.max_tokens}]")
+        if hidden_size != self.hidden_size or residual.dtype != torch.bfloat16:
+            raise ValueError("residual must be bf16 [T, 6144]")
+        if (
+            input_tensor.shape != residual.shape
+            or input_tensor.dtype != torch.bfloat16
+            or input_tensor.stride(1) != 1
+        ):
+            raise ValueError("input_tensor must be channel-contiguous bf16 [T, 6144]")
+        shard_size = hidden_size // self.world_size
+        if conv_weight.shape != (shard_size, self.window_size):
+            raise ValueError(
+                f"conv_weight must have shape [{shard_size}, {self.window_size}]"
+            )
+        if (
+            conv_weight.dtype != torch.bfloat16
+            or conv_weight.stride(0) != self.window_size
+        ):
+            raise ValueError("conv_weight must be contiguous bf16")
+        if norm_weight is not None and (
+            norm_weight.shape != (hidden_size,) or norm_weight.dtype != torch.bfloat16
+        ):
+            raise ValueError("norm_weight must be bf16 [6144] or None")
+        if cache.dtype != torch.bfloat16 or cache.ndim != 4:
+            raise ValueError("cache must be a 4-D bf16 tensor")
+        if shard_size % ws != 0 or cache.shape[1] != shard_size // ws:
+            raise ValueError("cache head layout is inconsistent with ws")
+        if off_s < 0 or off_s + ws > cache.shape[3]:
+            raise ValueError("cache channel offset is out of bounds")
+
+        index = self.generation
+        input_offset = index * self.input_generation_bytes // 4
+        output_offset = index * self.output_generation_bytes // 4
+        normed = torch.empty_like(residual) if norm_weight is not None else None
+        residual_out = torch.empty_like(residual)
+        phase_splits = 8 if tokens == 1 and shard_size % 8 == 0 else 1
+        phase_tile_p2 = triton.next_power_of_2(shard_size // phase_splits)
+        phase_grid = (tokens, phase_splits)
+        # Wide CTAs win before the grid saturates; smaller CTAs reduce register
+        # pressure once high-throughput batches provide enough parallelism.
+        if 128 <= tokens <= 2048:
+            phase_warps = 16
+        elif tokens > 2048:
+            phase_warps = 8
+        else:
+            phase_warps = 4
+        gather_warps = 4 if tokens >= 256 else 8
+        _publish_input_kernel[phase_grid](
+            input_tensor,
+            self.input_peer_ptrs,
+            input_offset,
+            input_tensor.stride(0),
+            C=hidden_size,
+            CS=shard_size,
+            CS_P2=phase_tile_p2,
+            SPLITS=phase_splits,
+            RANK=self.rank,
+            WORLD=self.world_size,
+            USE_PDL=self.use_pdl,
+            launch_pdl=self.use_pdl,
+            num_warps=phase_warps,
+        )
+        _reduce_insert_kernel[phase_grid](
+            self.input_peer_ptrs,
+            input_offset,
+            cache,
+            slot_mapping,
+            cache.stride(0),
+            cache.stride(1),
+            cache.stride(2),
+            cache.stride(3),
+            block_size,
+            C=hidden_size,
+            CS=shard_size,
+            CS_P2=phase_tile_p2,
+            SPLITS=phase_splits,
+            HEAD_SIZE=ws,
+            CACHE_OFFSET=off_s,
+            RANK=self.rank,
+            WORLD=self.world_size,
+            USE_PDL=self.use_pdl,
+            launch_pdl=self.use_pdl,
+            num_warps=phase_warps,
+        )
+        _sconv_publish_kernel[phase_grid](
+            self.output_peer_ptrs,
+            output_offset,
+            residual,
+            conv_weight,
+            cache,
+            positions,
+            seq_idx,
+            slot_mapping,
+            block_table,
+            residual.stride(0),
+            cache.stride(0),
+            cache.stride(1),
+            cache.stride(2),
+            cache.stride(3),
+            block_table.stride(0),
+            block_table.shape[1],
+            block_size,
+            C=hidden_size,
+            CS=shard_size,
+            CS_P2=phase_tile_p2,
+            SPLITS=phase_splits,
+            HEAD_SIZE=ws,
+            CACHE_OFFSET=off_s,
+            RANK=self.rank,
+            WORLD=self.world_size,
+            WINDOW=self.window_size,
+            USE_PDL=self.use_pdl,
+            launch_pdl=self.use_pdl,
+            num_warps=phase_warps,
+        )
+        _gather_norm_kernel[(tokens,)](
+            self.output_peer_ptrs,
+            output_offset,
+            norm_weight if norm_weight is not None else residual,
+            normed if normed is not None else residual_out,
+            residual_out,
+            eps,
+            residual.stride(0),
+            C=hidden_size,
+            C_P2=triton.next_power_of_2(hidden_size),
+            RANK=self.rank,
+            HAS_NORM=norm_weight is not None,
+            USE_PDL=self.use_pdl,
+            launch_pdl=self.use_pdl,
+            num_warps=gather_warps,
+        )
+        self.generation = (index + 1) % self.num_buffers
+        return normed, residual_out
+
+
+_STATE: LamportRSConv | None = None
+_STATE_FAILED = False
+
+
+def initialize_lamport_rs_conv(
+    hidden_size: int, window_size: int, max_num_batched_tokens: int
+) -> None:
+    """Collectively initialize the TP-group state during model construction."""
+    global _STATE, _STATE_FAILED
+    if _STATE is not None:
+        if _STATE.hidden_size != hidden_size or _STATE.window_size != window_size:
+            raise RuntimeError("all Lamport users must share hidden and window sizes")
+        return
+    if _STATE_FAILED or os.environ.get("LAMPORT_RS_SCONV", "1") == "0":
+        return
+    try:
+        max_tokens = min(_MAX_TOKENS, max_num_batched_tokens)
+        _STATE = LamportRSConv(hidden_size, window_size, max_tokens=max_tokens)
+    except Exception:
+        _STATE_FAILED = True
+        logger.exception("fused collective unavailable; use the NCCL fallback")
+
+
+def get_lamport_rs_conv(hidden_size: int, window_size: int) -> LamportRSConv | None:
+    """Return the state initialized with the model, or ``None`` for fallback."""
+    if _STATE is not None and (
+        _STATE.hidden_size != hidden_size or _STATE.window_size != window_size
+    ):
+        raise RuntimeError("all Lamport users must share hidden and window sizes")
+    return _STATE

vllm/models/inkling/nvidia/ops/mm_towers.py

@@ -0,0 +1,190 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Fused CUDA kernels for the Inkling vision/audio towers.
+
+Both kernels keep the reference paths' fp32 accumulation and per-op bf16
+rounding points (native ``rms_norm`` / ``F.gelu``); outputs are frequently
+bit-identical and otherwise differ by 1-2 bf16 ulps from reduction-order
+(real-checkpoint-weight cosine vs reference > 0.9999998).
+"""
+
+from __future__ import annotations
+
+import torch
+
+from vllm.triton_utils import tl, tldevice, triton
+
+from .norm import _get_num_warps_from_block_size
+
+
+@triton.jit
+def _dmel_embed_sum_norm_kernel(
+    idx_ptr,  # [T, NB] int32 dMel bin indices (values in [0, VOCAB))
+    w_ptr,  # [NB * VOCAB, D] bf16 embedding table
+    norm_w_ptr,  # [D] (unused if HAS_NORM=False)
+    out_ptr,  # [T, D] bf16
+    eps,
+    D,
+    stride_idx_t,
+    NB: tl.constexpr,
+    VOCAB: tl.constexpr,
+    D_P2: tl.constexpr,
+    HAS_NORM: tl.constexpr,
+):
+    t = tl.program_id(0).to(tl.int64)
+    offs = tl.arange(0, D_P2)
+    mask = offs < D
+    # One embedding row per mel bin (bin b uses table rows [b*VOCAB, (b+1)*VOCAB)),
+    # summed in fp32 (matches torch's fp32-accumulated bf16 .sum()).
+    acc = tl.zeros([D_P2], dtype=tl.float32)
+    for b in tl.static_range(NB):
+        v = tl.load(idx_ptr + t * stride_idx_t + b)
+        row = (b * VOCAB + v).to(tl.int64)
+        acc += tl.load(w_ptr + row * D + offs, mask=mask, other=0.0).to(tl.float32)
+    h = acc.to(tl.bfloat16)
+    if HAS_NORM:
+        # Match ir.ops.rms_norm: fp32 variance/normalize, then a single-rounded
+        # bf16 multiply with the bf16 weight.
+        x32 = h.to(tl.float32)
+        var = tl.sum(tl.where(mask, x32 * x32, 0.0), axis=0) / D
+        xn = (x32 * tl.math.rsqrt(var + eps)).to(tl.bfloat16)
+        w = tl.load(norm_w_ptr + offs, mask=mask, other=0.0)
+        h = xn * w
+    tl.store(out_ptr + t * D + offs, h, mask=mask)
+
+
+def dmel_embed_sum_norm(
+    dmel_idx: torch.Tensor,  # [T, NB] int32
+    weight: torch.Tensor,  # [NB * VOCAB, D] bf16
+    norm_weight: torch.Tensor | None,
+    eps: float,
+) -> torch.Tensor:
+    """``rmsnorm(sum_b weight[b * VOCAB + idx[:, b]])`` in one launch (no
+    [T, NB, D] intermediate)."""
+    T, nb = dmel_idx.shape
+    D = weight.shape[1]
+    assert weight.shape[0] % nb == 0
+    vocab = weight.shape[0] // nb
+    out = torch.empty((T, D), dtype=weight.dtype, device=weight.device)
+    if T == 0:
+        return out
+    d_p2 = triton.next_power_of_2(D)
+    _dmel_embed_sum_norm_kernel[(T,)](
+        dmel_idx,
+        weight,
+        norm_weight if norm_weight is not None else weight,
+        out,
+        eps,
+        D,
+        dmel_idx.stride(0),
+        NB=nb,
+        VOCAB=vocab,
+        D_P2=d_p2,
+        HAS_NORM=norm_weight is not None,
+        # Swept on GB200: 8 warps beats the block-size heuristic's 16 by ~4%
+        # at large T (the 80-row gather chain is latency- not lane-bound).
+        num_warps=8,
+    )
+    return out
+
+
+@triton.jit
+def _rmsnorm_gelu_kernel(
+    x_ptr,  # [R, D] bf16
+    w_ptr,  # [D]
+    out_ptr,  # [R, D] bf16 (or the folded layout when FOLD)
+    eps,
+    R,
+    D,
+    D_P2: tl.constexpr,
+    BLOCK_M: tl.constexpr,  # rows per block (>1 for small D)
+    HAS_GELU: tl.constexpr,
+    FOLD: tl.constexpr,
+    # fold geometry: input rows index [N, T, H, W]; the store scatters each
+    # row to (out_row, slot) of fold_timespace_to_depth's output layout.
+    FT: tl.constexpr,
+    FH: tl.constexpr,
+    FW: tl.constexpr,
+    TF: tl.constexpr,
+    HF: tl.constexpr,
+):
+    pid = tl.program_id(0).to(tl.int64)
+    rows = pid * BLOCK_M + tl.arange(0, BLOCK_M)
+    rmask = rows < R
+    offs = tl.arange(0, D_P2)
+    mask = rmask[:, None] & (offs < D)[None, :]
+    x32 = tl.load(x_ptr + rows[:, None] * D + offs[None, :], mask=mask, other=0.0).to(
+        tl.float32
+    )
+    var = tl.sum(x32 * x32, axis=1) / D
+    xn = (x32 * tl.math.rsqrt(var + eps)[:, None]).to(tl.bfloat16)
+    w = tl.load(w_ptr + offs, mask=offs < D, other=0.0)
+    h = xn * w[None, :]  # bf16 multiply, matching ir.ops.rms_norm
+    if HAS_GELU:
+        # Exact (erf) GELU on the bf16-rounded norm output, fp32 math,
+        # matching F.gelu's opmath on a bf16 tensor.
+        g32 = h.to(tl.float32)
+        h = (0.5 * g32 * (1.0 + tldevice.erf(g32 * 0.7071067811865476))).to(tl.bfloat16)
+    if FOLD:
+        # Store directly into the next layer's folded layout (a pure
+        # permutation — replaces the separate fold copy pass).
+        t = (rows // (FH * FW)) % FT
+        hh = (rows // FW) % FH
+        ww = rows % FW
+        n = rows // (FT * FH * FW)
+        slot = ((t % TF) * HF + hh % HF) * HF + ww % HF
+        out_row = ((n * (FT // TF) + t // TF) * (FH // HF) + hh // HF) * (
+            FW // HF
+        ) + ww // HF
+        base = (out_row * (TF * HF * HF) + slot) * D
+        tl.store(out_ptr + base[:, None] + offs[None, :], h, mask=mask)
+    else:
+        tl.store(out_ptr + rows[:, None] * D + offs[None, :], h, mask=mask)
+
+
+def rmsnorm_gelu(
+    x: torch.Tensor,  # [..., D] bf16 contiguous
+    weight: torch.Tensor,
+    eps: float,
+    gelu: bool = True,
+    fold: tuple[int, int] | None = None,  # (t_fold, hw_fold) of the NEXT fold
+) -> torch.Tensor:
+    """Fused ``gelu(rmsnorm(x))`` (or plain rmsnorm); multiple rows per block
+    when D is small. With ``fold``, x must be [N, T, H, W, D] and the output
+    comes back as ``fold_timespace_to_depth(result, *fold)``."""
+    D = x.shape[-1]
+    flat = x.reshape(-1, D)
+    assert flat.stride(1) == 1 and flat.stride(0) == D
+    R = flat.shape[0]
+    if fold is None:
+        out = torch.empty_like(flat)
+        ft = fh = fw = tf = hf = 1
+        out_shape = x.shape
+    else:
+        tf, hf = fold
+        N, ft, fh, fw, _ = x.shape
+        out_shape = (N, ft // tf, fh // hf, fw // hf, tf * hf * hf * D)
+        out = torch.empty(out_shape, dtype=x.dtype, device=x.device)
+    if R == 0:
+        return out.reshape(out_shape)
+    d_p2 = triton.next_power_of_2(D)
+    block_m = max(1, 4096 // d_p2)
+    _rmsnorm_gelu_kernel[(triton.cdiv(R, block_m),)](
+        flat,
+        weight,
+        out,
+        eps,
+        R,
+        D,
+        D_P2=d_p2,
+        BLOCK_M=block_m,
+        HAS_GELU=gelu,
+        FOLD=fold is not None,
+        FT=ft,
+        FH=fh,
+        FW=fw,
+        TF=tf,
+        HF=hf,
+        num_warps=_get_num_warps_from_block_size(d_p2 * block_m),
+    )
+    return out.reshape(out_shape)

vllm/models/inkling/nvidia/ops/norm.py

@@ -0,0 +1,315 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from __future__ import annotations
+
+from functools import lru_cache
+
+import torch
+
+from vllm.triton_utils import tl, triton
+
+_MAX_FUSED_SIZE = 65536
+
+
+def _get_num_warps_from_block_size(block_size: int) -> int:
+    if block_size >= 32768:
+        return 32
+    if block_size >= 8192:
+        return 16
+    if block_size >= 2048:
+        return 8
+    return 4
+
+
+def _largest_power_of_2(n: int) -> int:
+    assert n > 0, f"{n=}"
+    return 1 << (n.bit_length() - 1)
+
+
+@lru_cache(maxsize=128)
+def _get_grid_size_for_mem_bw_kernel(device: torch.device, factor: int = 8) -> int:
+    num_sms = torch.cuda.get_device_properties(device).multi_processor_count
+    return _largest_power_of_2(num_sms) * factor
+
+
+@triton.jit
+def _rmsnorm_fwd_kernel(
+    x_ptr,
+    weight_ptr,
+    y_ptr,
+    rstd_ptr,
+    eps,
+    x_stride_0,
+    y_stride_0,
+    n_cols,
+    block_size_n: tl.constexpr,
+):
+    pid_m = tl.program_id(0).to(tl.int64)
+    offs_n = tl.arange(0, block_size_n)
+    mask_n = offs_n < n_cols
+
+    weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
+    x = tl.load(x_ptr + pid_m * x_stride_0 + offs_n, mask=mask_n, other=0.0).to(
+        tl.float32
+    )
+    row_var = tl.sum(x * x, axis=0) / n_cols
+    rstd = tl.math.rsqrt(row_var + eps)
+    tl.store(rstd_ptr + pid_m, rstd)
+
+    y = x * rstd * weight
+    tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n)
+
+
+@triton.jit(do_not_specialize=["n_rows"])
+def _rmsnorm_fwd_kernel_block_m(
+    x_ptr,
+    weight_ptr,
+    y_ptr,
+    rstd_ptr,
+    eps,
+    x_stride_0,
+    y_stride_0,
+    n_rows,
+    n_cols,
+    block_size_m: tl.constexpr,
+    block_size_n: tl.constexpr,
+):
+    pid_m = tl.program_id(0).to(tl.int64)
+    offs_n = tl.arange(0, block_size_n)
+    mask_n = offs_n < n_cols
+
+    num_blocks_m = tl.cdiv(n_rows, block_size_m)
+    blocks_per_pid = tl.cdiv(num_blocks_m, tl.num_programs(0))
+    block_id_start = pid_m * blocks_per_pid
+    block_id_end = min(block_id_start + blocks_per_pid, num_blocks_m)
+
+    weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
+
+    for block_id in range(block_id_start, block_id_end):
+        offs_m = block_id * block_size_m + tl.arange(0, block_size_m)
+        mask_m = offs_m < n_rows
+        mask_mn = mask_m[:, None] & mask_n[None, :]
+        x = tl.load(
+            x_ptr + offs_m[:, None] * x_stride_0 + offs_n[None, :],
+            mask=mask_mn,
+            other=0.0,
+        ).to(tl.float32)
+        row_var = tl.sum(x * x, axis=1) / n_cols
+        rstd = tl.math.rsqrt(row_var + eps)
+        tl.store(rstd_ptr + offs_m, rstd, mask=mask_m)
+
+        y = x * rstd[:, None] * weight
+        tl.store(
+            y_ptr + offs_m[:, None] * y_stride_0 + offs_n[None, :],
+            y,
+            mask=mask_mn,
+        )
+
+
+@triton.jit
+def _add_rmsnorm_fwd_kernel(
+    res_ptr,  # [T, N] residual (read)
+    delta_ptr,  # [T, N] delta to add (read)
+    weight_ptr,
+    y_ptr,  # [T, N] normed output
+    res_out_ptr,  # [T, N] updated residual output
+    eps,
+    res_stride_0,
+    delta_stride_0,
+    y_stride_0,
+    res_out_stride_0,
+    n_cols,
+    block_size_n: tl.constexpr,
+):
+    pid_m = tl.program_id(0).to(tl.int64)
+    offs_n = tl.arange(0, block_size_n)
+    mask_n = offs_n < n_cols
+
+    weight = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
+    r = tl.load(res_ptr + pid_m * res_stride_0 + offs_n, mask=mask_n, other=0.0).to(
+        tl.float32
+    )
+    d = tl.load(delta_ptr + pid_m * delta_stride_0 + offs_n, mask=mask_n, other=0.0).to(
+        tl.float32
+    )
+    # Round the sum to the residual dtype first (matches the eager
+    # `residual + delta` then rmsnorm-on-bf16 sequence bit-for-bit).
+    s = (r + d).to(res_out_ptr.dtype.element_ty)
+    tl.store(res_out_ptr + pid_m * res_out_stride_0 + offs_n, s, mask=mask_n)
+    x = s.to(tl.float32)
+    row_var = tl.sum(x * x, axis=0) / n_cols
+    rstd = tl.math.rsqrt(row_var + eps)
+    y = x * rstd * weight
+    tl.store(y_ptr + pid_m * y_stride_0 + offs_n, y, mask=mask_n)
+
+
+def add_rmsnorm(
+    residual: torch.Tensor,
+    delta: torch.Tensor,
+    weight: torch.Tensor,
+    eps: float,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    """Fused ``res = residual + delta; y = rmsnorm(res)``.
+
+    Returns ``(y, res)``; both are fresh tensors (cudagraph-friendly, no
+    in-place update of the inputs).
+    """
+    assert residual.ndim == 2 and delta.ndim == 2, (residual.shape, delta.shape)
+    n_rows, n_cols = residual.shape
+    assert weight.shape[0] == n_cols
+    y = torch.empty_like(residual)
+    res_out = torch.empty_like(residual)
+    if n_rows == 0:
+        return y, res_out
+
+    block_size_n = triton.next_power_of_2(n_cols)
+    max_block_size_n = _MAX_FUSED_SIZE // residual.element_size()
+    if max_block_size_n < block_size_n:
+        raise RuntimeError(f"Large {n_cols=} is not supported")
+    num_warps = _get_num_warps_from_block_size(block_size_n)
+    _add_rmsnorm_fwd_kernel[(n_rows,)](
+        residual,
+        delta,
+        weight,
+        y,
+        res_out,
+        eps,
+        residual.stride(0),
+        delta.stride(0),
+        y.stride(0),
+        res_out.stride(0),
+        n_cols,
+        block_size_n,
+        num_warps=num_warps,
+    )
+    return y, res_out
+
+
+@triton.jit
+def _embed_rmsnorm_kernel(
+    ids_ptr,  # [T] token ids
+    table_ptr,  # [V, N] embedding table
+    weight_ptr,  # [N] (HAS_NORM only)
+    chain_weight_ptr,  # [N] (HAS_CHAIN only)
+    out_ptr,  # [T, N] rmsnorm(table[ids], weight)
+    chain_out_ptr,  # [T, N] rmsnorm(out, chain_weight) (HAS_CHAIN only)
+    eps,
+    table_stride_0,
+    n_cols,
+    block_size_n: tl.constexpr,
+    HAS_NORM: tl.constexpr,
+    HAS_CHAIN: tl.constexpr,
+):
+    pid_m = tl.program_id(0).to(tl.int64)
+    offs_n = tl.arange(0, block_size_n)
+    mask_n = offs_n < n_cols
+    row = tl.load(ids_ptr + pid_m).to(tl.int64)
+    x = tl.load(table_ptr + row * table_stride_0 + offs_n, mask=mask_n, other=0.0)
+    if HAS_NORM:
+        xf = x.to(tl.float32)
+        rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps)
+        w = tl.load(weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
+        # Round to the output dtype so the chained norm is bit-exact vs the
+        # unfused pair (which stores bf16 in between).
+        x = (xf * rstd * w).to(out_ptr.dtype.element_ty)
+    tl.store(out_ptr + pid_m * n_cols + offs_n, x, mask=mask_n)
+    if HAS_CHAIN:
+        xf = x.to(tl.float32)
+        rstd = tl.math.rsqrt(tl.sum(xf * xf, axis=0) / n_cols + eps)
+        w = tl.load(chain_weight_ptr + offs_n, mask=mask_n, other=0.0).to(tl.float32)
+        tl.store(
+            chain_out_ptr + pid_m * n_cols + offs_n,
+            (xf * rstd * w).to(chain_out_ptr.dtype.element_ty),
+            mask=mask_n,
+        )
+
+
+def embed_rmsnorm(
+    input_ids: torch.Tensor,
+    embed_table: torch.Tensor,
+    weight: torch.Tensor | None,
+    eps: float,
+    chain_weight: torch.Tensor | None = None,
+) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
+    """Fused ``rmsnorm(embed_table[input_ids], weight)`` row gather + norm.
+
+    Requires the full vocab on-rank (replicated or tp_size == 1).
+    ``weight=None`` skips the norm (``use_embed_norm=False``), leaving a pure
+    embedding-table gather.
+    ``chain_weight`` additionally emits ``rmsnorm(out, chain_weight)`` (the
+    first decoder layer's pre-attention norm) as a second output, still one
+    launch. Bit-exact vs the unfused module sequence."""
+    ids = input_ids.view(-1)
+    (T,) = ids.shape
+    n = embed_table.shape[1]
+    out = torch.empty(
+        (*input_ids.shape, n), dtype=embed_table.dtype, device=embed_table.device
+    )
+    chain_out = torch.empty_like(out) if chain_weight is not None else None
+    if T > 0:
+        block_size_n = triton.next_power_of_2(n)
+        _embed_rmsnorm_kernel[(T,)](
+            ids,
+            embed_table,
+            weight if weight is not None else embed_table,
+            chain_weight if chain_weight is not None else embed_table,
+            out,
+            chain_out if chain_out is not None else out,
+            eps,
+            embed_table.stride(0),
+            n,
+            block_size_n,
+            HAS_NORM=weight is not None,
+            HAS_CHAIN=chain_weight is not None,
+            num_warps=_get_num_warps_from_block_size(block_size_n),
+        )
+    if chain_out is not None:
+        return out, chain_out
+    return out
+
+
+def rmsnorm(x: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor:
+    assert x.ndim == 2, f"{x.shape=}"
+    assert weight.ndim == 1, f"{weight.shape=}"
+    n_rows, n_cols = x.shape
+    assert weight.shape[0] == n_cols, f"{weight.shape=} {x.shape=}"
+    y = torch.empty_like(x)
+    rstd = torch.empty((n_rows,), dtype=torch.float32, device=x.device)
+
+    block_size_n = triton.next_power_of_2(n_cols)
+    max_block_size_n = _MAX_FUSED_SIZE // x.element_size()
+    if max_block_size_n < block_size_n:
+        raise RuntimeError(f"Large {n_cols=} is not supported")
+    block_size_m = max(1, 4096 // block_size_n)
+    num_warps = _get_num_warps_from_block_size(block_size_n)
+
+    if block_size_m == 1:
+        _rmsnorm_fwd_kernel[(n_rows,)](
+            x,
+            weight,
+            y,
+            rstd,
+            eps,
+            x.stride(0),
+            y.stride(0),
+            n_cols,
+            block_size_n,
+            num_warps=num_warps,
+        )
+    else:
+        grid_size = _get_grid_size_for_mem_bw_kernel(x.device)
+        _rmsnorm_fwd_kernel_block_m[(grid_size,)](
+            x,
+            weight,
+            y,
+            rstd,
+            eps,
+            x.stride(0),
+            y.stride(0),
+            n_rows,
+            n_cols,
+            block_size_m,
+            block_size_n,
+            num_warps=num_warps,
+        )
+    return y

vllm/models/inkling/nvidia/ops/qkvr_prep.py

@@ -0,0 +1,918 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import torch
+
+from vllm.triton_utils import tl, triton
+from vllm.utils.torch_utils import aux_stream
+
+LOW_BLOCK_M = 32
+LOW_BLOCK_N = 64
+LOW_NUM_WARPS = 4
+THROUGHPUT_BLOCK_M = 32
+THROUGHPUT_BLOCK_N = 128
+THROUGHPUT_GROUP_M = 2
+THROUGHPUT_NUM_WARPS = 4
+SMALL_TOKEN_THRESHOLD = 128
+SMALL_NUM_WARPS = 2
+Q_BLOCK_ROWS = 8
+Q_NUM_WARPS = 2
+KV_BLOCK_ROWS = 4
+KV_NUM_WARPS = 2
+
+
+@triton.jit(do_not_specialize=["rows"])
+def _rel_proj_low_latency_kernel(
+    qkvr_ptr,
+    rel_proj_ptr,
+    rel_out_ptr,
+    log_scaling_ptr,
+    rows,
+    stride_x_t,
+    R_OFFSET: tl.constexpr,
+    NUM_Q_HEADS: tl.constexpr,
+    REL_EXTENT: tl.constexpr,
+    BLOCK_M: tl.constexpr,
+    BLOCK_N: tl.constexpr,
+    APPLY_LOG_SCALING: tl.constexpr,
+):
+    row = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
+    col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
+    inner = tl.arange(0, 16)
+    token = row // NUM_Q_HEADS
+    head = row % NUM_Q_HEADS
+    relative = tl.load(
+        qkvr_ptr
+        + token[:, None] * stride_x_t
+        + R_OFFSET
+        + head[:, None] * 16
+        + inner[None, :],
+        mask=row[:, None] < rows,
+        other=0.0,
+    )
+    projection = tl.load(
+        rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :],
+        mask=col[None, :] < REL_EXTENT,
+        other=0.0,
+    )
+    values = tl.dot(relative, projection, out_dtype=tl.float32).to(
+        rel_out_ptr.dtype.element_ty
+    )
+    values = values.to(tl.float32)
+    if APPLY_LOG_SCALING:
+        values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[:, None]
+    tl.store(
+        rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :],
+        values.to(rel_out_ptr.dtype.element_ty),
+        mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT),
+    )
+
+
+@triton.jit(do_not_specialize=["rows"])
+def _rel_proj_throughput_kernel(
+    qkvr_ptr,
+    rel_proj_ptr,
+    rel_out_ptr,
+    log_scaling_ptr,
+    rows,
+    stride_x_t,
+    R_OFFSET: tl.constexpr,
+    NUM_Q_HEADS: tl.constexpr,
+    REL_EXTENT: tl.constexpr,
+    BLOCK_M: tl.constexpr,
+    BLOCK_N: tl.constexpr,
+    GROUP_M: tl.constexpr,
+    APPLY_LOG_SCALING: tl.constexpr,
+):
+    row_group = tl.program_id(0)
+    col = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
+    inner = tl.arange(0, 16)
+    projection = tl.load(
+        rel_proj_ptr + inner[:, None] * REL_EXTENT + col[None, :],
+        mask=col[None, :] < REL_EXTENT,
+        other=0.0,
+    )
+    row_offsets = tl.arange(0, BLOCK_M)
+    for group_offset in tl.static_range(GROUP_M):
+        row = (row_group * GROUP_M + group_offset) * BLOCK_M + row_offsets
+        token = row // NUM_Q_HEADS
+        head = row % NUM_Q_HEADS
+        relative = tl.load(
+            qkvr_ptr
+            + token[:, None] * stride_x_t
+            + R_OFFSET
+            + head[:, None] * 16
+            + inner[None, :],
+            mask=row[:, None] < rows,
+            other=0.0,
+        )
+        values = tl.dot(relative, projection, out_dtype=tl.float32).to(
+            rel_out_ptr.dtype.element_ty
+        )
+        values = values.to(tl.float32)
+        if APPLY_LOG_SCALING:
+            values *= tl.load(log_scaling_ptr + token, mask=row < rows, other=1.0)[
+                :, None
+            ]
+        tl.store(
+            rel_out_ptr + row[:, None] * REL_EXTENT + col[None, :],
+            values.to(rel_out_ptr.dtype.element_ty),
+            mask=(row[:, None] < rows) & (col[None, :] < REL_EXTENT),
+        )
+
+
+def use_rel_proj_throughput(rows: int, rel_extent: int) -> bool:
+    min_rows = 8192 if rel_extent == 512 else 2048
+    return rows >= min_rows
+
+
+def qkvr_rel_proj(
+    qkvr: torch.Tensor,
+    rel_proj: torch.Tensor,
+    rel_out: torch.Tensor,
+    log_scaling: torch.Tensor | None,
+    *,
+    num_q_heads: int,
+    num_kv_heads: int,
+    head_dim: int,
+    d_rel: int,
+) -> None:
+    rows = qkvr.shape[0] * num_q_heads
+    rel_extent = rel_proj.shape[1]
+    assert d_rel == 16 and rel_proj.shape[0] == 16
+    r_offset = num_q_heads * head_dim + 2 * num_kv_heads * head_dim
+    log_scaling_ptr = log_scaling if log_scaling is not None else qkvr
+    common = dict(
+        R_OFFSET=r_offset,
+        NUM_Q_HEADS=num_q_heads,
+        REL_EXTENT=rel_extent,
+        APPLY_LOG_SCALING=log_scaling is not None,
+    )
+
+    if use_rel_proj_throughput(rows, rel_extent):
+        grid = (
+            triton.cdiv(rows, THROUGHPUT_BLOCK_M * THROUGHPUT_GROUP_M),
+            triton.cdiv(rel_extent, THROUGHPUT_BLOCK_N),
+        )
+        _rel_proj_throughput_kernel[grid](
+            qkvr,
+            rel_proj,
+            rel_out,
+            log_scaling_ptr,
+            rows,
+            qkvr.stride(0),
+            BLOCK_M=THROUGHPUT_BLOCK_M,
+            BLOCK_N=THROUGHPUT_BLOCK_N,
+            GROUP_M=THROUGHPUT_GROUP_M,
+            num_warps=THROUGHPUT_NUM_WARPS,
+            **common,
+        )
+        return
+
+    grid = (
+        triton.cdiv(rows, LOW_BLOCK_M),
+        triton.cdiv(rel_extent, LOW_BLOCK_N),
+    )
+    _rel_proj_low_latency_kernel[grid](
+        qkvr,
+        rel_proj,
+        rel_out,
+        log_scaling_ptr,
+        rows,
+        qkvr.stride(0),
+        BLOCK_M=LOW_BLOCK_M,
+        BLOCK_N=LOW_BLOCK_N,
+        num_warps=LOW_NUM_WARPS,
+        **common,
+    )
+
+
+@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"])
+def _qkvr_qkv_kernel(
+    qkvr_ptr,
+    q_norm_weight_ptr,
+    q_out_ptr,
+    rel_proj_ptr,
+    rel_out_ptr,
+    k_weight_ptr,
+    v_weight_ptr,
+    k_norm_weight_ptr,
+    conv_cache_ptr,
+    key_cache_ptr,
+    value_cache_ptr,
+    positions_ptr,
+    seq_idx_ptr,
+    conv_slot_mapping_ptr,
+    conv_block_table_ptr,
+    query_start_ptr,
+    attention_slot_mapping_ptr,
+    log_scaling_ptr,
+    tokens,
+    eps,
+    stride_x_t,
+    stride_cc_block,
+    stride_cc_head,
+    stride_cc_token,
+    stride_cc_dim,
+    stride_kc_block,
+    stride_kc_token,
+    stride_kc_head,
+    stride_vc_block,
+    stride_vc_token,
+    stride_vc_head,
+    stride_block_table_req,
+    max_blocks,
+    conv_block_size,
+    attention_page_size,
+    Q_WIDTH: tl.constexpr,
+    KV_WIDTH: tl.constexpr,
+    NUM_Q_HEADS: tl.constexpr,
+    NUM_KV_HEADS: tl.constexpr,
+    HEAD_DIM: tl.constexpr,
+    WINDOW_SIZE: tl.constexpr,
+    OFF_K: tl.constexpr,
+    OFF_V: tl.constexpr,
+    APPLY_LOG_SCALING: tl.constexpr,
+    D_REL: tl.constexpr,
+    REL_EXTENT: tl.constexpr,
+    REL_EXTENT_PADDED: tl.constexpr,
+):
+    block = tl.program_id(0)
+    num_q_rows = tokens * NUM_Q_HEADS
+    dims = tl.arange(0, HEAD_DIM)
+
+    if block < num_q_rows:
+        row = block
+        token = row // NUM_Q_HEADS
+        head = row % NUM_Q_HEADS
+        values = tl.load(
+            qkvr_ptr + token * stride_x_t + head * HEAD_DIM + dims,
+        ).to(tl.float32)
+        weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32)
+        rstd = tl.rsqrt(tl.sum(values * values, axis=0) / HEAD_DIM + eps)
+        normalized = values * rstd * weight
+        if APPLY_LOG_SCALING:
+            normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32)
+            normalized *= tl.load(log_scaling_ptr + token)
+        tl.store(
+            q_out_ptr + row * HEAD_DIM + dims,
+            normalized.to(q_out_ptr.dtype.element_ty),
+        )
+        rel_cols = tl.arange(0, REL_EXTENT_PADDED)
+        rel_mask = rel_cols < REL_EXTENT
+        projected = tl.zeros([REL_EXTENT_PADDED], dtype=tl.float32)
+        rel_offset = Q_WIDTH + 2 * KV_WIDTH + head * D_REL
+        for rel_dim in tl.static_range(D_REL):
+            rel_value = tl.load(
+                qkvr_ptr + token * stride_x_t + rel_offset + rel_dim
+            ).to(tl.float32)
+            proj = tl.load(
+                rel_proj_ptr + rel_dim * REL_EXTENT + rel_cols,
+                mask=rel_mask,
+                other=0.0,
+            ).to(tl.float32)
+            projected += rel_value * proj
+        projected = projected.to(rel_out_ptr.dtype.element_ty).to(tl.float32)
+        if APPLY_LOG_SCALING:
+            projected *= tl.load(log_scaling_ptr + token)
+        tl.store(
+            rel_out_ptr + row * REL_EXTENT + rel_cols,
+            projected.to(rel_out_ptr.dtype.element_ty),
+            mask=rel_mask,
+        )
+    else:
+        row = block - num_q_rows
+        if row < tokens * NUM_KV_HEADS:
+            token = row // NUM_KV_HEADS
+            head = row % NUM_KV_HEADS
+            position = tl.load(positions_ptr + token)
+            request = tl.load(seq_idx_ptr + token)
+            conv_slot = tl.load(conv_slot_mapping_ptr + token)
+            query_start = tl.load(query_start_ptr + token)
+            attention_slot = tl.load(attention_slot_mapping_ptr + token)
+            valid = conv_slot >= 0
+
+            k_col = Q_WIDTH + head * HEAD_DIM
+            v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM
+            k_value = tl.load(qkvr_ptr + token * stride_x_t + k_col + dims)
+            v_value = tl.load(qkvr_ptr + token * stride_x_t + v_col + dims)
+
+            safe_slot = tl.maximum(conv_slot, 0)
+            cache_base = (
+                conv_cache_ptr
+                + (safe_slot // conv_block_size) * stride_cc_block
+                + head * stride_cc_head
+                + (safe_slot % conv_block_size) * stride_cc_token
+            )
+            tl.store(
+                cache_base + (OFF_K + dims) * stride_cc_dim,
+                k_value,
+                mask=valid,
+            )
+            tl.store(
+                cache_base + (OFF_V + dims) * stride_cc_dim,
+                v_value,
+                mask=valid,
+            )
+
+            acc_k = tl.zeros([HEAD_DIM], dtype=tl.float32)
+            acc_v = tl.zeros([HEAD_DIM], dtype=tl.float32)
+            for tap in tl.static_range(WINDOW_SIZE):
+                source_position = position - (WINDOW_SIZE - 1) + tap
+                source_row = token - (WINDOW_SIZE - 1) + tap
+                in_window = valid & (source_position >= 0)
+                intra = in_window & (source_row >= query_start)
+                cached = in_window & (source_row < query_start)
+                safe_row = tl.maximum(source_row, 0)
+                source_k = tl.load(
+                    qkvr_ptr + safe_row * stride_x_t + k_col + dims,
+                    mask=intra,
+                    other=0.0,
+                ).to(tl.float32)
+                source_v = tl.load(
+                    qkvr_ptr + safe_row * stride_x_t + v_col + dims,
+                    mask=intra,
+                    other=0.0,
+                ).to(tl.float32)
+                safe_position = tl.maximum(source_position, 0)
+                logical_block = tl.minimum(
+                    safe_position // conv_block_size, max_blocks - 1
+                )
+                physical_block = tl.load(
+                    conv_block_table_ptr
+                    + request * stride_block_table_req
+                    + logical_block,
+                    mask=cached,
+                    other=0,
+                ).to(tl.int64)
+                tap_base = (
+                    conv_cache_ptr
+                    + physical_block * stride_cc_block
+                    + head * stride_cc_head
+                    + (safe_position % conv_block_size) * stride_cc_token
+                )
+                source_k += tl.load(
+                    tap_base + (OFF_K + dims) * stride_cc_dim,
+                    mask=cached,
+                    other=0.0,
+                ).to(tl.float32)
+                source_v += tl.load(
+                    tap_base + (OFF_V + dims) * stride_cc_dim,
+                    mask=cached,
+                    other=0.0,
+                ).to(tl.float32)
+                k_weight = tl.load(
+                    k_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap
+                ).to(tl.float32)
+                v_weight = tl.load(
+                    v_weight_ptr + (head * HEAD_DIM + dims) * WINDOW_SIZE + tap
+                ).to(tl.float32)
+                acc_k += source_k * k_weight
+                acc_v += source_v * v_weight
+
+            k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
+            v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
+            k_float = k_rounded.to(tl.float32)
+            k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32)
+            rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=0) / HEAD_DIM + eps)
+            k_normalized = (k_float * rstd * k_norm_weight).to(
+                qkvr_ptr.dtype.element_ty
+            )
+
+            safe_attention_slot = tl.maximum(attention_slot, 0)
+            attention_block = safe_attention_slot // attention_page_size
+            attention_offset = safe_attention_slot % attention_page_size
+            tl.store(
+                key_cache_ptr
+                + attention_block * stride_kc_block
+                + attention_offset * stride_kc_token
+                + head * stride_kc_head
+                + dims,
+                k_normalized,
+                mask=attention_slot >= 0,
+            )
+            tl.store(
+                value_cache_ptr
+                + attention_block * stride_vc_block
+                + attention_offset * stride_vc_token
+                + head * stride_vc_head
+                + dims,
+                v_rounded,
+                mask=attention_slot >= 0,
+            )
+
+
+@triton.jit(do_not_specialize=["num_rows"])
+def _q_kernel(
+    qkvr_ptr,
+    q_norm_weight_ptr,
+    q_out_ptr,
+    log_scaling_ptr,
+    num_rows,
+    stride_x_t,
+    eps,
+    NUM_Q_HEADS: tl.constexpr,
+    HEAD_DIM: tl.constexpr,
+    BLOCK_ROWS: tl.constexpr,
+    APPLY_LOG_SCALING: tl.constexpr,
+):
+    rows = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
+    dims = tl.arange(0, HEAD_DIM)
+    row_mask = rows < num_rows
+    tokens = rows // NUM_Q_HEADS
+    heads = rows % NUM_Q_HEADS
+    values = tl.load(
+        qkvr_ptr
+        + tokens[:, None] * stride_x_t
+        + heads[:, None] * HEAD_DIM
+        + dims[None, :],
+        mask=row_mask[:, None],
+        other=0.0,
+    ).to(tl.float32)
+    weight = tl.load(q_norm_weight_ptr + dims).to(tl.float32)
+    rstd = tl.rsqrt(tl.sum(values * values, axis=1) / HEAD_DIM + eps)
+    normalized = values * rstd[:, None] * weight[None, :]
+    if APPLY_LOG_SCALING:
+        normalized = normalized.to(q_out_ptr.dtype.element_ty).to(tl.float32)
+        tau = tl.load(log_scaling_ptr + tokens, mask=row_mask, other=1.0)
+        normalized *= tau[:, None]
+    tl.store(
+        q_out_ptr + rows[:, None] * HEAD_DIM + dims[None, :],
+        normalized.to(q_out_ptr.dtype.element_ty),
+        mask=row_mask[:, None],
+    )
+
+
+@triton.jit(do_not_specialize=["tokens", "stride_block_table_req", "max_blocks"])
+def _kv_kernel(
+    qkvr_ptr,
+    k_weight_ptr,
+    v_weight_ptr,
+    k_norm_weight_ptr,
+    conv_cache_ptr,
+    key_cache_ptr,
+    value_cache_ptr,
+    positions_ptr,
+    seq_idx_ptr,
+    conv_slot_mapping_ptr,
+    conv_block_table_ptr,
+    query_start_ptr,
+    attention_slot_mapping_ptr,
+    tokens,
+    eps,
+    stride_x_t,
+    stride_cc_block,
+    stride_cc_head,
+    stride_cc_token,
+    stride_cc_dim,
+    stride_kc_block,
+    stride_kc_token,
+    stride_kc_head,
+    stride_vc_block,
+    stride_vc_token,
+    stride_vc_head,
+    stride_block_table_req,
+    max_blocks,
+    conv_block_size,
+    attention_page_size,
+    Q_WIDTH: tl.constexpr,
+    KV_WIDTH: tl.constexpr,
+    NUM_KV_HEADS: tl.constexpr,
+    HEAD_DIM: tl.constexpr,
+    WINDOW_SIZE: tl.constexpr,
+    OFF_K: tl.constexpr,
+    OFF_V: tl.constexpr,
+    BLOCK_ROWS: tl.constexpr,
+):
+    token = tl.program_id(0) * BLOCK_ROWS + tl.arange(0, BLOCK_ROWS)
+    dims = tl.arange(0, HEAD_DIM)
+    row_mask = token < tokens
+    head_id = tl.program_id(1)
+    head = tl.full([BLOCK_ROWS], head_id, tl.int64)
+    position = tl.load(positions_ptr + token, mask=row_mask, other=0)
+    request = tl.load(seq_idx_ptr + token, mask=row_mask, other=0)
+    conv_slot = tl.load(conv_slot_mapping_ptr + token, mask=row_mask, other=-1)
+    query_start = tl.load(query_start_ptr + token, mask=row_mask, other=0)
+    attention_slot = tl.load(
+        attention_slot_mapping_ptr + token, mask=row_mask, other=-1
+    )
+    valid = row_mask & (conv_slot >= 0)
+
+    k_col = Q_WIDTH + head * HEAD_DIM
+    v_col = Q_WIDTH + KV_WIDTH + head * HEAD_DIM
+    k_value = tl.load(
+        qkvr_ptr + token[:, None] * stride_x_t + k_col[:, None] + dims[None, :],
+        mask=row_mask[:, None],
+        other=0.0,
+    )
+    v_value = tl.load(
+        qkvr_ptr + token[:, None] * stride_x_t + v_col[:, None] + dims[None, :],
+        mask=row_mask[:, None],
+        other=0.0,
+    )
+    safe_slot = tl.maximum(conv_slot, 0)
+    cache_base = (
+        conv_cache_ptr
+        + (safe_slot // conv_block_size) * stride_cc_block
+        + head * stride_cc_head
+        + (safe_slot % conv_block_size) * stride_cc_token
+    )
+    tl.store(
+        cache_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim,
+        k_value,
+        mask=valid[:, None],
+    )
+    tl.store(
+        cache_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim,
+        v_value,
+        mask=valid[:, None],
+    )
+
+    acc_k = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32)
+    acc_v = tl.zeros([BLOCK_ROWS, HEAD_DIM], dtype=tl.float32)
+    for tap in tl.static_range(WINDOW_SIZE):
+        source_position = position - (WINDOW_SIZE - 1) + tap
+        source_row = token - (WINDOW_SIZE - 1) + tap
+        in_window = valid & (source_position >= 0)
+        intra = in_window & (source_row >= query_start)
+        cached = in_window & (source_row < query_start)
+        safe_row = tl.maximum(source_row, 0)
+        source_k = tl.load(
+            qkvr_ptr + safe_row[:, None] * stride_x_t + k_col[:, None] + dims[None, :],
+            mask=intra[:, None],
+            other=0.0,
+        ).to(tl.float32)
+        source_v = tl.load(
+            qkvr_ptr + safe_row[:, None] * stride_x_t + v_col[:, None] + dims[None, :],
+            mask=intra[:, None],
+            other=0.0,
+        ).to(tl.float32)
+        safe_position = tl.maximum(source_position, 0)
+        logical_block = tl.minimum(safe_position // conv_block_size, max_blocks - 1)
+        physical_block = tl.load(
+            conv_block_table_ptr + request * stride_block_table_req + logical_block,
+            mask=cached,
+            other=0,
+        ).to(tl.int64)
+        tap_base = (
+            conv_cache_ptr
+            + physical_block * stride_cc_block
+            + head * stride_cc_head
+            + (safe_position % conv_block_size) * stride_cc_token
+        )
+        source_k += tl.load(
+            tap_base[:, None] + (OFF_K + dims[None, :]) * stride_cc_dim,
+            mask=cached[:, None],
+            other=0.0,
+        ).to(tl.float32)
+        source_v += tl.load(
+            tap_base[:, None] + (OFF_V + dims[None, :]) * stride_cc_dim,
+            mask=cached[:, None],
+            other=0.0,
+        ).to(tl.float32)
+        k_weight = tl.load(
+            k_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap
+        ).to(tl.float32)
+        v_weight = tl.load(
+            v_weight_ptr + (head_id * HEAD_DIM + dims) * WINDOW_SIZE + tap
+        ).to(tl.float32)
+        acc_k += source_k * k_weight[None, :]
+        acc_v += source_v * v_weight[None, :]
+
+    k_rounded = (acc_k + k_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
+    v_rounded = (acc_v + v_value.to(tl.float32)).to(qkvr_ptr.dtype.element_ty)
+    k_float = k_rounded.to(tl.float32)
+    k_norm_weight = tl.load(k_norm_weight_ptr + dims).to(tl.float32)
+    rstd = tl.rsqrt(tl.sum(k_float * k_float, axis=1) / HEAD_DIM + eps)
+    k_normalized = (k_float * rstd[:, None] * k_norm_weight[None, :]).to(
+        qkvr_ptr.dtype.element_ty
+    )
+
+    safe_attention_slot = tl.maximum(attention_slot, 0)
+    attention_block = safe_attention_slot // attention_page_size
+    attention_offset = safe_attention_slot % attention_page_size
+    attention_mask = row_mask & (attention_slot >= 0)
+    tl.store(
+        key_cache_ptr
+        + attention_block[:, None] * stride_kc_block
+        + attention_offset[:, None] * stride_kc_token
+        + head[:, None] * stride_kc_head
+        + dims[None, :],
+        k_normalized,
+        mask=attention_mask[:, None],
+    )
+    tl.store(
+        value_cache_ptr
+        + attention_block[:, None] * stride_vc_block
+        + attention_offset[:, None] * stride_vc_token
+        + head[:, None] * stride_vc_head
+        + dims[None, :],
+        v_rounded,
+        mask=attention_mask[:, None],
+    )
+
+
+def _run_tiled_q(
+    qkvr: torch.Tensor,
+    q_norm_weight: torch.Tensor,
+    q_out: torch.Tensor,
+    positions: torch.Tensor,
+    *,
+    eps: float,
+    num_q_heads: int,
+    head_dim: int,
+    log_scaling: torch.Tensor | None,
+) -> None:
+    num_rows = qkvr.shape[0] * num_q_heads
+    _q_kernel[(triton.cdiv(num_rows, Q_BLOCK_ROWS),)](
+        qkvr,
+        q_norm_weight,
+        q_out,
+        log_scaling if log_scaling is not None else positions,
+        num_rows,
+        qkvr.stride(0),
+        eps,
+        NUM_Q_HEADS=num_q_heads,
+        HEAD_DIM=head_dim,
+        BLOCK_ROWS=Q_BLOCK_ROWS,
+        APPLY_LOG_SCALING=log_scaling is not None,
+        num_warps=Q_NUM_WARPS,
+    )
+
+
+def _run_tiled_kv(
+    qkvr: torch.Tensor,
+    k_weight: torch.Tensor,
+    v_weight: torch.Tensor,
+    k_norm_weight: torch.Tensor,
+    conv_cache: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    positions: torch.Tensor,
+    seq_idx: torch.Tensor,
+    conv_slot_mapping: torch.Tensor,
+    conv_block_table: torch.Tensor,
+    query_start: torch.Tensor,
+    attention_slot_mapping: torch.Tensor,
+    *,
+    eps: float,
+    num_q_heads: int,
+    num_kv_heads: int,
+    head_dim: int,
+    off_k: int,
+    off_v: int,
+    conv_block_size: int,
+) -> None:
+    tokens = qkvr.shape[0]
+    _kv_kernel[(triton.cdiv(tokens, KV_BLOCK_ROWS), num_kv_heads)](
+        qkvr,
+        k_weight,
+        v_weight,
+        k_norm_weight,
+        conv_cache,
+        key_cache,
+        value_cache,
+        positions,
+        seq_idx,
+        conv_slot_mapping,
+        conv_block_table,
+        query_start,
+        attention_slot_mapping,
+        tokens,
+        eps,
+        qkvr.stride(0),
+        conv_cache.stride(0),
+        conv_cache.stride(1),
+        conv_cache.stride(2),
+        conv_cache.stride(3),
+        key_cache.stride(0),
+        key_cache.stride(1),
+        key_cache.stride(2),
+        value_cache.stride(0),
+        value_cache.stride(1),
+        value_cache.stride(2),
+        conv_block_table.stride(0),
+        conv_block_table.shape[1],
+        conv_block_size,
+        key_cache.shape[1],
+        Q_WIDTH=num_q_heads * head_dim,
+        KV_WIDTH=num_kv_heads * head_dim,
+        NUM_KV_HEADS=num_kv_heads,
+        HEAD_DIM=head_dim,
+        WINDOW_SIZE=k_weight.shape[1],
+        OFF_K=off_k,
+        OFF_V=off_v,
+        BLOCK_ROWS=KV_BLOCK_ROWS,
+        num_warps=KV_NUM_WARPS,
+    )
+
+
+def _run_fused_small(
+    qkvr: torch.Tensor,
+    q_norm_weight: torch.Tensor,
+    q_out: torch.Tensor,
+    rel_proj: torch.Tensor,
+    rel_out: torch.Tensor,
+    k_weight: torch.Tensor,
+    v_weight: torch.Tensor,
+    k_norm_weight: torch.Tensor,
+    conv_cache: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    positions: torch.Tensor,
+    seq_idx: torch.Tensor,
+    conv_slot_mapping: torch.Tensor,
+    conv_block_table: torch.Tensor,
+    query_start: torch.Tensor,
+    attention_slot_mapping: torch.Tensor,
+    *,
+    eps: float,
+    num_q_heads: int,
+    num_kv_heads: int,
+    head_dim: int,
+    off_k: int,
+    off_v: int,
+    conv_block_size: int,
+    log_scaling: torch.Tensor | None,
+) -> None:
+    tokens = qkvr.shape[0]
+
+    num_q_rows = tokens * num_q_heads
+    grid = (num_q_rows + tokens * num_kv_heads,)
+    _qkvr_qkv_kernel[grid](
+        qkvr,
+        q_norm_weight,
+        q_out,
+        rel_proj,
+        rel_out,
+        k_weight,
+        v_weight,
+        k_norm_weight,
+        conv_cache,
+        key_cache,
+        value_cache,
+        positions,
+        seq_idx,
+        conv_slot_mapping,
+        conv_block_table,
+        query_start,
+        attention_slot_mapping,
+        log_scaling if log_scaling is not None else positions,
+        tokens,
+        eps,
+        qkvr.stride(0),
+        conv_cache.stride(0),
+        conv_cache.stride(1),
+        conv_cache.stride(2),
+        conv_cache.stride(3),
+        key_cache.stride(0),
+        key_cache.stride(1),
+        key_cache.stride(2),
+        value_cache.stride(0),
+        value_cache.stride(1),
+        value_cache.stride(2),
+        conv_block_table.stride(0),
+        conv_block_table.shape[1],
+        conv_block_size,
+        key_cache.shape[1],
+        Q_WIDTH=num_q_heads * head_dim,
+        KV_WIDTH=num_kv_heads * head_dim,
+        NUM_Q_HEADS=num_q_heads,
+        NUM_KV_HEADS=num_kv_heads,
+        HEAD_DIM=head_dim,
+        WINDOW_SIZE=k_weight.shape[1],
+        OFF_K=off_k,
+        OFF_V=off_v,
+        APPLY_LOG_SCALING=log_scaling is not None,
+        D_REL=16,
+        REL_EXTENT=rel_proj.shape[1],
+        REL_EXTENT_PADDED=triton.next_power_of_2(rel_proj.shape[1]),
+        num_warps=SMALL_NUM_WARPS,
+    )
+
+
+def fused_qkvr_prep(
+    qkvr: torch.Tensor,
+    k_weight: torch.Tensor,
+    v_weight: torch.Tensor,
+    q_norm_weight: torch.Tensor,
+    k_norm_weight: torch.Tensor,
+    rel_proj: torch.Tensor,
+    eps: float,
+    num_q_heads: int,
+    num_kv_heads: int,
+    head_dim: int,
+    d_rel: int,
+    conv_cache: torch.Tensor,
+    key_cache: torch.Tensor,
+    value_cache: torch.Tensor,
+    positions: torch.Tensor,
+    conv_block_table: torch.Tensor,
+    seq_idx: torch.Tensor,
+    conv_slot_mapping: torch.Tensor,
+    query_start: torch.Tensor,
+    attention_slot_mapping: torch.Tensor,
+    off_k: int,
+    off_v: int,
+    conv_block_size: int,
+    log_scaling: torch.Tensor | None = None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+    assert d_rel == 16 and rel_proj.shape[0] == 16
+    assert head_dim == 128
+    assert qkvr.is_contiguous()
+    assert k_weight.stride() == (k_weight.shape[1], 1)
+    assert v_weight.stride() == (v_weight.shape[1], 1)
+    assert rel_proj.stride() == (rel_proj.shape[1], 1)
+    assert conv_cache.stride(3) == 1
+    assert key_cache.stride(3) == 1 and value_cache.stride(3) == 1
+    tokens = qkvr.shape[0]
+    q_out = torch.empty(
+        (tokens, num_q_heads * head_dim), dtype=qkvr.dtype, device=qkvr.device
+    )
+    rel_out = torch.empty(
+        (tokens, num_q_heads, rel_proj.shape[1]),
+        dtype=qkvr.dtype,
+        device=qkvr.device,
+    )
+    if tokens == 0:
+        return q_out, rel_out
+
+    if tokens < SMALL_TOKEN_THRESHOLD:
+        _run_fused_small(
+            qkvr,
+            q_norm_weight,
+            q_out,
+            rel_proj,
+            rel_out,
+            k_weight,
+            v_weight,
+            k_norm_weight,
+            conv_cache,
+            key_cache,
+            value_cache,
+            positions,
+            seq_idx,
+            conv_slot_mapping,
+            conv_block_table,
+            query_start,
+            attention_slot_mapping,
+            eps=eps,
+            num_q_heads=num_q_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_dim,
+            off_k=off_k,
+            off_v=off_v,
+            conv_block_size=conv_block_size,
+            log_scaling=log_scaling,
+        )
+        return q_out, rel_out
+
+    kv_stream = aux_stream()
+    assert kv_stream is not None
+    current_stream = torch.cuda.current_stream()
+    kv_stream.wait_stream(current_stream)
+    with torch.cuda.stream(kv_stream):
+        _run_tiled_kv(
+            qkvr,
+            k_weight,
+            v_weight,
+            k_norm_weight,
+            conv_cache,
+            key_cache,
+            value_cache,
+            positions,
+            seq_idx,
+            conv_slot_mapping,
+            conv_block_table,
+            query_start,
+            attention_slot_mapping,
+            eps=eps,
+            num_q_heads=num_q_heads,
+            num_kv_heads=num_kv_heads,
+            head_dim=head_dim,
+            off_k=off_k,
+            off_v=off_v,
+            conv_block_size=conv_block_size,
+        )
+    _run_tiled_q(
+        qkvr,
+        q_norm_weight,
+        q_out,
+        positions,
+        eps=eps,
+        num_q_heads=num_q_heads,
+        head_dim=head_dim,
+        log_scaling=log_scaling,
+    )
+    qkvr_rel_proj(
+        qkvr,
+        rel_proj,
+        rel_out,
+        log_scaling,
+        num_q_heads=num_q_heads,
+        num_kv_heads=num_kv_heads,
+        head_dim=head_dim,
+        d_rel=d_rel,
+    )
+    current_stream.wait_stream(kv_stream)
+    return q_out, rel_out

vllm/models/inkling/nvidia/ops/sconv.py

@@ -0,0 +1,290 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling short-convolution kernels backed by a paged sliding-window state cache.
+
+Each layer's 4 conv streams (K, V, attn-output, mlp-output) share one paged KV
+cache ``[num_blocks, H, N, D]`` (head-major; see ``sconv_swa_attn.py``). A stream
+occupies the contiguous D-sub-range ``[off_s, off_s + ws)`` across all ``H``
+heads, so its flat per-token width is ``H * ws`` and that is the conv channel
+dim. The cache stores the conv *input* at every absolute position.
+
+``fused_sconv`` is the single-launch path used by the model: per token it writes
+the current input to its slot and convolves the ``W`` taps ending at its
+absolute position. A tap landing inside the current forward is read from the
+immutable input ``x`` (row ``src - pos + pid_t``); only pre-forward taps are read
+from the paged cache (window position ``src`` -> physical block via
+``block_table[req, src // N]``). The just-written slot is never read back this
+step, so there is no write/read hazard within or across programs -- which is
+why this needs no decode-vs-prefill split and is valid for prefill / decode /
+spec alike.
+
+All kernels address the cache purely by ``(slot, absolute_position)`` and
+allocate nothing inside the captured region; their grids depend only on the
+token count (``fused_sconv`` on a fixed token/channel tiling), so the same
+decode path replays correctly under a full CUDA graph without any
+data-dependent shape or branch.
+"""
+
+from __future__ import annotations
+
+import torch
+
+from vllm.triton_utils import tl, triton
+
+
+@triton.jit
+def _fused_sconv_kernel(
+    x_ptr,  # [T, H*WS] head-major current-token inputs (also residual)
+    cache_ptr,  # [num_blocks, H, N, D] paged (page-strided view)
+    weight_ptr,  # [H*WS, W]
+    out_ptr,  # [T, H*WS]
+    pos_ptr,  # [T] int64 absolute position per token
+    seq_idx_ptr,  # [T] int32 token -> batch request
+    slot_ptr,  # [T] int64 flat slot (block*N + blk_off); < 0 => PAD
+    block_table_ptr,  # [num_reqs, max_blocks] int32 block_table
+    qstart_ptr,  # [T] int32 first x-row of the token's request
+    T,  # num tokens
+    stride_x_t,
+    stride_c_blk,
+    stride_c_h,
+    stride_c_n,
+    stride_c_d,
+    stride_w_d,
+    stride_w_w,
+    stride_bt_r,
+    MAX_BLOCKS,
+    N,  # block_size
+    W: tl.constexpr,
+    USE_SILU: tl.constexpr,
+    USE_RESIDUAL: tl.constexpr,
+    OFF_S: tl.constexpr,
+    WS: tl.constexpr,
+    H: tl.constexpr,
+    BT: tl.constexpr,
+    BLOCK_C: tl.constexpr,
+):
+    # Each program owns a [BT tokens, BLOCK_C channels] tile. The flat channel
+    # index packs all H heads head-major (head = c // WS, in-stream offset =
+    # c % WS), so one program spans heads -- no per-head launch and no
+    # next_power_of_2(WS) lane waste.
+    pid_t = tl.program_id(0)
+    pid_c = tl.program_id(1)
+    toff = pid_t * BT + tl.arange(0, BT)  # [BT] token rows
+    coff = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)  # [BLOCK_C] flat channels
+    C = H * WS
+    t_mask = toff < T
+    c_mask = coff < C
+    head = tl.minimum(coff // WS, H - 1)  # clamp keeps masked lanes in-buffer
+    cd = OFF_S + coff % WS  # cache D-index of the channel's stream slot
+
+    slot = tl.load(slot_ptr + toff, mask=t_mask, other=-1)  # [BT]
+    valid = slot >= 0
+    pos = tl.load(pos_ptr + toff, mask=t_mask, other=0)
+    req = tl.load(seq_idx_ptr + toff, mask=t_mask, other=0)
+    qstart = tl.load(qstart_ptr + toff, mask=t_mask, other=0)
+
+    tc_mask = t_mask[:, None] & c_mask[None, :]
+
+    # 1) Insert each token's input into its paged slot (skip PAD rows).
+    xv = tl.load(x_ptr + toff[:, None] * stride_x_t + coff[None, :], mask=tc_mask)
+    safe_slot = tl.maximum(slot, 0)
+    dst = (
+        cache_ptr
+        + (safe_slot // N)[:, None] * stride_c_blk
+        + head[None, :] * stride_c_h
+        + (safe_slot % N)[:, None] * stride_c_n
+        + cd[None, :] * stride_c_d
+    )
+    tl.store(dst, xv, mask=tc_mask & valid[:, None])
+
+    # 2) Convolve the W taps ending at each token's `pos`. Each tap is read from
+    # `x` if it falls inside this forward (row >= the request's first row), else
+    # from the paged cache. Exactly one source is unmasked per tap, so we sum both.
+    acc = tl.zeros([BT, BLOCK_C], dtype=tl.float32)
+    for iw in tl.static_range(W):
+        src = pos - (W - 1) + iw  # [BT] absolute window position
+        row = toff - (W - 1) + iw  # [BT] x-row of `src` (== src - pos + token)
+        in_win = valid & (src >= 0)
+        intra = in_win & (row >= qstart)
+        cached = in_win & (row < qstart)
+        # intra-forward tap: read the immutable input x (never the slot we just
+        # wrote), so there is no write/read hazard.
+        safe_row = tl.maximum(row, 0)
+        xt = tl.load(
+            x_ptr + safe_row[:, None] * stride_x_t + coff[None, :],
+            mask=c_mask[None, :] & intra[:, None],
+            other=0.0,
+        ).to(tl.float32)
+        # pre-forward tap: read from the paged cache via the block table. Clamp
+        # addressing terms; the load is masked off when out of window.
+        safe_src = tl.maximum(src, 0)
+        safe_lblk = tl.minimum(safe_src // N, MAX_BLOCKS - 1)
+        blk = tl.load(
+            block_table_ptr + req * stride_bt_r + safe_lblk, mask=cached, other=0
+        ).to(tl.int64)
+        cbase = (
+            cache_ptr
+            + blk[:, None] * stride_c_blk
+            + head[None, :] * stride_c_h
+            + (safe_src % N)[:, None] * stride_c_n
+            + cd[None, :] * stride_c_d
+        )
+        cv = tl.load(cbase, mask=c_mask[None, :] & cached[:, None], other=0.0).to(
+            tl.float32
+        )
+        wv = tl.load(
+            weight_ptr + coff * stride_w_d + iw * stride_w_w, mask=c_mask, other=0.0
+        ).to(tl.float32)
+        acc += (xt + cv) * wv[None, :]
+
+    if USE_SILU:
+        acc = acc * tl.sigmoid(acc)
+    if USE_RESIDUAL:
+        acc += xv.to(tl.float32)
+
+    tl.store(
+        out_ptr + toff[:, None] * stride_x_t + coff[None, :],
+        acc.to(out_ptr.dtype.element_ty),
+        mask=tc_mask,
+    )
+
+
+def fused_sconv(
+    x: torch.Tensor,  # [T, H*ws] head-major current-token inputs
+    weight: torch.Tensor,  # [H*ws, W]
+    cache: torch.Tensor,  # [num_blocks, H, N, D] paged
+    positions: torch.Tensor,  # [T] int64 absolute position per token
+    block_table: torch.Tensor,  # [num_reqs, max_blocks] int32
+    seq_idx: torch.Tensor,  # [T] int32 token -> batch request
+    slot_mapping: torch.Tensor,  # [T] int64 flat slot (PAD = -1 => skip)
+    query_start: torch.Tensor,  # [T] int32 first x-row of the token's request
+    off_s: int,
+    ws: int,
+    block_size: int,
+    activation: str | None = None,
+    use_residual: bool = True,
+) -> torch.Tensor:
+    """Single-launch insert + depthwise causal conv1d over the paged cache.
+
+    Reads same-forward taps from ``x`` and pre-forward taps from the cache, so
+    it is race-free in one launch for prefill / decode / spec and supports full
+    CUDA-graph capture for decode.
+    """
+    T = x.shape[0]
+    out = torch.empty_like(x)
+    if T == 0:
+        return out
+    assert x.is_contiguous()
+    assert cache.stride(3) == 1, "cache D-dim must be contiguous"
+    H = cache.shape[1]
+    W = weight.shape[1]
+    C = H * ws  # flat conv channel dim (all heads, head-major)
+    # Tile BT tokens x BLOCK_C channels per program: enough work per CTA to
+    # amortize the per-token addressing, while keeping the grid large for
+    # prefill. BLOCK_C spans heads so there is no per-head launch. A ~2K-element
+    # tile at 4 warps measured best on Blackwell; larger tiles spill registers.
+    BLOCK_C = min(triton.next_power_of_2(C), 256)
+    BT = 8
+    grid = (triton.cdiv(T, BT), triton.cdiv(C, BLOCK_C))
+    _fused_sconv_kernel[grid](
+        x,
+        cache,
+        weight,
+        out,
+        positions,
+        seq_idx,
+        slot_mapping,
+        block_table,
+        query_start,
+        T,
+        x.stride(0),
+        cache.stride(0),
+        cache.stride(1),
+        cache.stride(2),
+        cache.stride(3),
+        weight.stride(0),
+        weight.stride(1),
+        block_table.stride(0),
+        block_table.shape[1],
+        block_size,
+        W=W,
+        USE_SILU=activation in ("silu", "swish"),
+        USE_RESIDUAL=use_residual,
+        OFF_S=off_s,
+        WS=ws,
+        H=H,
+        BT=BT,
+        BLOCK_C=BLOCK_C,
+        num_warps=4,
+    )
+    return out
+
+
+@triton.jit
+def _seq_metadata_kernel(
+    qsl_ptr,  # [num_reqs + 1] int32 cumulative query start rows
+    seq_idx_ptr,  # [T] int32 out: token -> owning request
+    query_start_ptr,  # [T] int32 out: first x-row of the token's request
+    num_reqs,
+    num_actual_tokens,
+    num_padded_tokens,
+    n_iters,  # ceil(log2(num_reqs)): binary-search depth
+    BLOCK: tl.constexpr,
+):
+    offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+    tok = offs.to(tl.int32)
+    # Largest j in [0, num_reqs) with qsl[j] <= tok.
+    lo = tl.zeros([BLOCK], tl.int32)
+    hi = tl.full([BLOCK], num_reqs - 1, tl.int32)
+    for _ in range(n_iters):
+        mid = (lo + hi + 1) // 2
+        below = tl.load(qsl_ptr + mid) <= tok
+        lo = tl.where(below, mid, lo)
+        hi = tl.where(below, hi, mid - 1)
+    actual = offs < num_actual_tokens
+    padded = offs < num_padded_tokens
+    query_start = tl.load(qsl_ptr + lo)
+    tl.store(seq_idx_ptr + offs, tl.where(actual, lo, 0), mask=padded)
+    tl.store(
+        query_start_ptr + offs,
+        tl.where(actual, query_start, 0),
+        mask=padded,
+    )
+
+
+def sconv_seq_metadata(
+    query_start_loc: torch.Tensor,
+    num_reqs: int,
+    num_actual_tokens: int,
+    seq_idx_out: torch.Tensor,
+    query_start_out: torch.Tensor,
+    num_padded_tokens: int | None = None,
+) -> None:
+    """Fill static per-token seq_idx / query_start buffers in one launch.
+
+    Replaces the arange + searchsorted + clamp + gather + 2x copy chain of the
+    sconv metadata build with a single kernel writing both persistent buffers.
+    Padded rows are filled with zero and must have ``slot_mapping == -1``.
+    """
+    if num_padded_tokens is None:
+        num_padded_tokens = num_actual_tokens
+    if num_padded_tokens < num_actual_tokens:
+        raise ValueError("num_padded_tokens must cover all actual tokens")
+    if num_padded_tokens > seq_idx_out.shape[0]:
+        raise ValueError("seq_idx_out is too small for the padded token count")
+    if num_padded_tokens > query_start_out.shape[0]:
+        raise ValueError("query_start_out is too small for the padded token count")
+
+    BLOCK = 256
+    n_iters = (num_reqs - 1).bit_length()
+    grid = (triton.cdiv(num_padded_tokens, BLOCK),)
+    _seq_metadata_kernel[grid](
+        query_start_loc,
+        seq_idx_out,
+        query_start_out,
+        num_reqs,
+        num_actual_tokens,
+        num_padded_tokens,
+        n_iters,
+        BLOCK=BLOCK,
+    )

vllm/models/inkling/nvidia/ops/silu_and_mul.py

@@ -0,0 +1,197 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""SwiGLU kernels for the Inkling MLP layers.
+
+``silu_and_mul_triton``: SiLU-and-mul over the checkpoint's interleaved
+fused gate/up layout (dense MLP). ``sink_silu_mul_epilogue``: the sink-expert
+variant with the per-expert dequant scale and per-token gamma fused in.
+"""
+
+import torch
+
+from vllm.triton_utils import tl, triton
+
+
+@triton.jit(do_not_specialize=["M"])
+def _silu_and_mul_triton_kernel(
+    gateup_out_ptr,
+    down_inp_ptr,
+    M,
+    N: tl.constexpr,
+    GRID_SIZE: tl.constexpr,
+    NUM_STAGES: tl.constexpr,
+    BLOCK_SIZE_M: tl.constexpr,
+    BLOCK_SIZE_N: tl.constexpr,
+    EVEN_N: tl.constexpr,
+    INT64_INDEX: tl.constexpr,
+):
+    start_pid = tl.program_id(0)
+    if INT64_INDEX:
+        start_pid = start_pid.to(tl.int64)
+        M = M.to(tl.int64)
+
+    NUM_BLOCKS_N: tl.constexpr = tl.cdiv(N, BLOCK_SIZE_N)
+    num_blocks_mn = tl.cdiv(M, BLOCK_SIZE_M) * NUM_BLOCKS_N
+
+    for pid in tl.range(start_pid, num_blocks_mn, GRID_SIZE, num_stages=NUM_STAGES):
+        pid_m = pid // NUM_BLOCKS_N
+        pid_n = pid % NUM_BLOCKS_N
+
+        offs_m = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
+        offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+        mask_m = offs_m < M
+        mask_n = offs_n < N
+
+        # Interleaved fused gate/up: [g0, u0, g1, u1, ...].
+        mask_offs_2n = pid_n * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N) // 2
+        tl.static_assert(BLOCK_SIZE_N % 8 == 0, f"{BLOCK_SIZE_N=}")
+        mask_2n = mask_offs_2n < N
+        mask_2n = tl.max_constancy(mask_2n, [16])
+
+        offs_2n = pid_n * 2 * BLOCK_SIZE_N + tl.arange(0, 2 * BLOCK_SIZE_N)
+        offs_m2n = offs_m[:, None] * N * 2 + offs_2n[None, :]
+
+        if EVEN_N or pid_n * BLOCK_SIZE_N + BLOCK_SIZE_N <= N:
+            gateup_out = tl.load(
+                gateup_out_ptr + offs_m2n, mask=mask_m[:, None], other=0.0
+            )
+        else:
+            mask_m2n = mask_m[:, None] & mask_2n[None, :]
+            gateup_out = tl.load(gateup_out_ptr + offs_m2n, mask=mask_m2n, other=0.0)
+
+        gate_out, up_out = tl.split(
+            tl.reshape(gateup_out, (BLOCK_SIZE_M, BLOCK_SIZE_N, 2))
+        )
+        gate_out = gate_out.to(tl.float32)
+        up_out = up_out.to(tl.float32)
+
+        down_inp = gate_out * tl.sigmoid(gate_out) * up_out
+
+        mask_mn = mask_m[:, None] if EVEN_N else mask_m[:, None] & mask_n[None, :]
+        offs_mn = offs_m[:, None] * N + offs_n[None, :]
+        tl.store(down_inp_ptr + offs_mn, down_inp, mask=mask_mn)
+
+
+def silu_and_mul_triton(gateup_output: torch.Tensor) -> torch.Tensor:
+    """SiLU-and-mul for the interleaved fused gate/up layout.
+
+    Adapted from ``inkling_kernels.activation.silu_and_mul_fwd`` (without MXFP).
+    """
+    assert gateup_output.is_contiguous(), (
+        f"{gateup_output.shape=} {gateup_output.stride()=}"
+    )
+    assert gateup_output.ndim == 2, f"{gateup_output.shape=}"
+
+    M = gateup_output.shape[0]
+    hidden_size = gateup_output.shape[1]
+    assert hidden_size % 2 == 0, f"{hidden_size=}"
+    N = hidden_size // 2
+
+    down_input = torch.empty(
+        (M, N), device=gateup_output.device, dtype=gateup_output.dtype
+    )
+    if M == 0:
+        return down_input
+
+    BLOCK_SIZE_N = max(8, min(256, triton.next_power_of_2(N)))
+    if M <= 1:
+        BLOCK_SIZE_M = 4
+    elif M <= 256:
+        BLOCK_SIZE_M = 2
+    elif M < 4096:
+        BLOCK_SIZE_M = 4
+    else:
+        BLOCK_SIZE_M = 16
+        BLOCK_SIZE_N = max(8, min(128, triton.next_power_of_2(N)))
+    max_grid_size = triton.cdiv(M, BLOCK_SIZE_M) * triton.cdiv(N, BLOCK_SIZE_N)
+    num_sms = torch.cuda.get_device_properties(
+        gateup_output.device
+    ).multi_processor_count
+    grid_size = min(num_sms * 4, max_grid_size)
+
+    _silu_and_mul_triton_kernel[(grid_size,)](
+        gateup_out_ptr=gateup_output,
+        down_inp_ptr=down_input,
+        M=M,
+        N=N,
+        GRID_SIZE=grid_size,
+        NUM_STAGES=1,
+        BLOCK_SIZE_M=BLOCK_SIZE_M,
+        BLOCK_SIZE_N=BLOCK_SIZE_N,
+        EVEN_N=N % BLOCK_SIZE_N == 0,
+        INT64_INDEX=gateup_output.nbytes >= 2**31,
+        num_warps=8,
+    )
+
+    return down_input
+
+
+@triton.jit(do_not_specialize=["T"])
+def _sink_epilogue_kernel(
+    raw_ptr,  # [T, S * 2F] gemm1 output, interleaved g/u pairs per expert block
+    alpha_ptr,  # [S] fp32 per-expert pre-SiLU dequant scale
+    gamma_ptr,  # [T, S] fp32 per-token sink weights (may be strided)
+    ratio_ptr,  # [S] fp32 per-expert post-SiLU scale (gemm2 alpha ratio)
+    out_ptr,  # [T, S * F] output
+    T,
+    stride_raw_0,
+    stride_gamma_0,
+    F: tl.constexpr,
+    S: tl.constexpr,
+    BLOCK_F: tl.constexpr,
+):
+    pid_t = tl.program_id(0).to(tl.int64)
+    pid_sf = tl.program_id(1)
+    if pid_t >= T:
+        return
+    s = pid_sf // (F // BLOCK_F)
+    offs_f = (pid_sf % (F // BLOCK_F)) * BLOCK_F + tl.arange(0, BLOCK_F)
+
+    base = pid_t * stride_raw_0 + s * 2 * F
+    gate = tl.load(raw_ptr + base + 2 * offs_f).to(tl.float32)
+    up = tl.load(raw_ptr + base + 2 * offs_f + 1).to(tl.float32)
+    alpha = tl.load(alpha_ptr + s)
+    weight = tl.load(gamma_ptr + pid_t * stride_gamma_0 + s) * tl.load(ratio_ptr + s)
+
+    gate *= alpha
+    up *= alpha
+    h = gate * tl.sigmoid(gate) * up * weight
+    tl.store(out_ptr + pid_t * (S * F) + s * F + offs_f, h)
+
+
+def sink_silu_mul_epilogue(
+    raw: torch.Tensor,  # [T, S * 2F] gemm1 output (interleaved gate/up rows)
+    alphas: torch.Tensor,  # [S] fp32
+    gammas: torch.Tensor,  # [T, S] fp32
+    ratios: torch.Tensor,  # [S] fp32
+    n_experts: int,
+    out_dtype: torch.dtype,
+) -> torch.Tensor:
+    """Fused sink-expert epilogue: silu(g * a_e) * (u * a_e) * (gamma * r_e).
+
+    One kernel replaces the per-expert dequant column scale, the SwiGLU, and
+    the per-token gamma multiply between the two sink GEMMs.
+    """
+    tokens = raw.shape[0]
+    f = raw.shape[1] // (2 * n_experts)
+    out = torch.empty((tokens, n_experts * f), device=raw.device, dtype=out_dtype)
+    if tokens == 0:
+        return out
+    # raw may be a column-slice of a padded GEMM output (rows strided).
+    assert raw.stride(1) == 1 and gammas.stride(1) == 1
+    # Largest power-of-two divisor of f (f = 768 -> 256), capped at 512.
+    block_f = min(512, f & (-f))
+    _sink_epilogue_kernel[(tokens, n_experts * (f // block_f))](
+        raw,
+        alphas,
+        gammas,
+        ratios,
+        out,
+        tokens,
+        raw.stride(0),
+        gammas.stride(0),
+        F=f,
+        S=n_experts,
+        BLOCK_F=block_f,
+    )
+    return out

vllm/models/inkling/nvidia/sconv_swa_attn.py

@@ -0,0 +1,221 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling short-conv state managed as a sliding-window KV cache.
+
+Each decoder layer owns one ``InklingConvState`` (an ``AttentionLayerBase``) that
+emits a single ``SlidingWindowSpec`` for the layer's 4 sconv streams (K, V,
+attn-output, mlp-output), packed head-major into one block:
+
+    H = num_kv_heads (per-rank), N = block_size = sconv_kernel_size,
+    D = head_dim(K) + head_dim(V) + hidden/H(attn) + hidden/H(mlp)
+
+``D`` is TP-invariant; per rank we store ``H/TP`` heads of width ``D``. The conv
+reads/writes this cache out-of-band via a custom backend; the (smaller) conv page
+is padded up to the uniform attention page by ``unify_kv_cache_spec_page_size``.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import ClassVar
+
+import torch
+from torch import nn
+
+from vllm.config import VllmConfig, get_current_vllm_config
+from vllm.distributed import get_tensor_model_parallel_world_size
+from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
+from vllm.v1.attention.backend import (
+    AttentionBackend,
+    AttentionCGSupport,
+    AttentionMetadata,
+    AttentionMetadataBuilder,
+    CommonAttentionMetadata,
+)
+from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheSpec, SlidingWindowSpec
+
+from .ops.sconv import sconv_seq_metadata
+
+# Stream order within the per-head packed D (== contiguous sub-ranges).
+_K, _V, _ATTN, _MLP = 0, 1, 2, 3
+
+
+@dataclass
+class InklingSconvMetadata(AttentionMetadata):
+    block_table: torch.Tensor  # [num_reqs, max_blocks] physical blocks per req
+    slot_mapping: torch.Tensor  # [T] int64 flat slot of each token (-1 => skip)
+    seq_idx: torch.Tensor  # [T] int32 token -> batch request
+    query_start: torch.Tensor  # [T] int32 first x-row of each token's request
+
+
+class InklingSconvMetadataBuilder(AttentionMetadataBuilder[InklingSconvMetadata]):
+    _cudagraph_support: ClassVar[AttentionCGSupport] = (
+        AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
+    )
+
+    def __init__(
+        self,
+        kv_cache_spec: AttentionSpec,
+        layer_names: list[str],
+        vllm_config: VllmConfig,
+        device: torch.device,
+    ) -> None:
+        super().__init__(kv_cache_spec, layer_names, vllm_config, device)
+        assert isinstance(kv_cache_spec, SlidingWindowSpec)
+        # Persistent per-token buffers for CUDA graph capture.
+        max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
+        self.seq_idx_buffer = torch.empty(
+            max_num_tokens, dtype=torch.int32, device=device
+        )
+        self.query_start_buffer = torch.empty(
+            max_num_tokens, dtype=torch.int32, device=device
+        )
+
+    def build(
+        self,
+        common_prefix_len: int,
+        common_attn_metadata: CommonAttentionMetadata,
+        fast_build: bool = False,
+    ) -> InklingSconvMetadata:
+        num_reqs = common_attn_metadata.num_reqs
+        num_actual_tokens = int(common_attn_metadata.query_start_loc_cpu[-1])
+        num_padded_tokens = common_attn_metadata.slot_mapping.shape[0]
+        assert num_padded_tokens >= num_actual_tokens
+
+        # Per-token seq_idx (owning request) and query_start (first x-row of
+        # that request; the fused kernel uses it to tell same-forward taps,
+        # read from x, from pre-forward taps, read from cache) in one launch.
+        sconv_seq_metadata(
+            common_attn_metadata.query_start_loc,
+            num_reqs,
+            num_actual_tokens,
+            self.seq_idx_buffer,
+            self.query_start_buffer,
+            num_padded_tokens,
+        )
+
+        return InklingSconvMetadata(
+            block_table=common_attn_metadata.block_table_tensor,
+            slot_mapping=common_attn_metadata.slot_mapping[:num_padded_tokens],
+            seq_idx=self.seq_idx_buffer[:num_padded_tokens],
+            query_start=self.query_start_buffer[:num_padded_tokens],
+        )
+
+
+class InklingSconvBackend(AttentionBackend):
+    """Custom dummy backend for the sconv sliding-window cache management."""
+
+    @staticmethod
+    def get_name() -> str:
+        return "INKLING_SCONV_SWA"
+
+    @classmethod
+    def indexes_kv_by_block_stride(cls) -> bool:
+        # num_blocks is the outermost dim (HND, see get_kv_cache_shape), so the
+        # padded conv page is read through a strided view.
+        return True
+
+    @staticmethod
+    def get_kv_cache_shape(
+        num_blocks: int,
+        block_size: int,
+        num_kv_heads: int,
+        head_size: int,
+        cache_dtype_str: str = "auto",
+    ) -> tuple[int, ...]:
+        # HND, num-blocks-first, head-major: [num_blocks, H, N, D].
+        return (num_blocks, num_kv_heads, block_size, head_size)
+
+    @staticmethod
+    def get_kv_cache_stride_order(
+        include_num_layers_dimension: bool = False,
+    ) -> tuple[int, ...]:
+        # Identity: physical layout == logical [num_blocks, H, N, D].
+        if include_num_layers_dimension:
+            return (0, 1, 2, 3, 4)
+        return (0, 1, 2, 3)
+
+    @staticmethod
+    def get_impl_cls():
+        raise NotImplementedError(
+            "InklingSconvBackend has no attention impl; the conv runs out-of-band."
+        )
+
+    @staticmethod
+    def get_builder_cls() -> type[InklingSconvMetadataBuilder]:
+        return InklingSconvMetadataBuilder
+
+
+class InklingConvState(nn.Module, AttentionLayerBase):
+    """Per-decoder-layer owner emitting one sliding-window conv-state spec."""
+
+    def __init__(
+        self,
+        *,
+        num_kv_heads: int,
+        head_dim: int,
+        hidden_size: int,
+        kernel_size: int,
+        prefix: str,
+    ) -> None:
+        super().__init__()
+        self.prefix = prefix
+        # Bound to the manager-allocated paged cache by bind_kv_cache; a
+        # placeholder until then. Read out-of-band by InklingShortConv.
+        self.kv_cache = torch.tensor([])
+        tp_size = get_tensor_model_parallel_world_size()
+        # Guardrails for the conv-state layout below; only these are exercised.
+        # tp_size <= num_kv_heads keeps >=1 whole KV head per rank (no
+        # replication/clamping), so the per-head width stays TP-invariant.
+        assert tp_size <= num_kv_heads, (
+            f"sconv SWA cache supports tp_size <= num_kv_heads ({num_kv_heads}), "
+            f"got {tp_size}"
+        )
+        # Per-rank head count; D is TP-invariant (K/V heads and the hidden
+        # chunk both scale 1/TP together). The attn-/mlp-output sconv streams
+        # are hidden-sharded: each rank owns its H/tp chunk (the sublayer
+        # outputs are reduce-scattered / all-gathered around the conv).
+        self.num_kv_heads = num_kv_heads // tp_size
+        hidden_per_head = hidden_size // num_kv_heads
+        # Packed per-head width: K + V + attn-output chunk + mlp-output chunk,
+        # padded to a power of two so every layer's conv page is the same size
+        # and an exact multiple of the attention page (the page unifier then
+        # scales attention block sizes instead of padding).
+        raw_head_size = 2 * head_dim + 2 * hidden_per_head
+        self.head_size = 1 << (raw_head_size - 1).bit_length()
+        self.sliding_window = kernel_size
+        self.block_size = kernel_size
+        # Per-head D-sub-range (offset, width) for each stream. Streams share
+        # the cache; each writes/reads its own width across all H heads.
+        self.stream_ranges: tuple[tuple[int, int], ...] = (
+            (0, head_dim),  # _K
+            (head_dim, head_dim),  # _V
+            (2 * head_dim, hidden_per_head),  # _ATTN
+            (2 * head_dim + hidden_per_head, hidden_per_head),  # _MLP
+        )
+        vllm_config = get_current_vllm_config()
+        self._dtype = vllm_config.model_config.dtype
+        assert self._dtype == torch.bfloat16, (
+            f"sconv SWA cache supports bfloat16 only, got {self._dtype}"
+        )
+        # Register in the forward context so the runner enumerates this owner as
+        # an attention-like layer (get_kv_cache_spec / get_attn_backend).
+        compilation_config = vllm_config.compilation_config
+        if prefix in compilation_config.static_forward_context:
+            raise ValueError(f"Duplicate layer name: {prefix}")
+        compilation_config.static_forward_context[prefix] = self
+
+    def forward(self): ...
+
+    def get_attn_backend(self) -> type[AttentionBackend]:
+        return InklingSconvBackend
+
+    def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:
+        return SlidingWindowSpec(
+            block_size=self.block_size,
+            num_kv_heads=self.num_kv_heads,
+            head_size=self.head_size,
+            head_size_v=0,  # all 4 streams packed into head_size
+            dtype=self._dtype,
+            sliding_window=self.sliding_window,
+        )

vllm/models/inkling/nvidia/short_conv.py

@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling short convolution: depthwise causal conv1d (+ residual) over a paged
+sliding-window conv-state cache.
+
+Each decoder layer owns one ``InklingConvState`` (``sconv_swa_attn.py``) holding the
+manager-allocated paged cache for the layer's 4 sconv streams (K, V, attn-output,
+mlp-output), packed head-major into one block. Each ``InklingShortConv`` is a
+stateless weight + kernel launcher that, per forward (positions-addressed, the
+same path for prefill / decode / mixed), inserts the current tokens' inputs
+into their paged slot and convolves each token against the ``W`` taps ending
+at its absolute position, reading pre-forward window positions out of the
+paged cache via the block table.
+
+Per-forward metadata (``block_table`` / ``slot_mapping`` / ``seq_idx`` /
+``query_start``) is built once by ``InklingSconvMetadataBuilder`` and published under
+the owner's prefix in the forward context; the absolute ``positions`` are
+threaded in from the model. The insert + conv run in a single ``fused_sconv``
+launch (same path for prefill / decode / mixed / spec). All inputs are
+fixed-address persistent buffers and the grid is fixed, so decode can replay
+under a full CUDA graph.
+"""
+
+from __future__ import annotations
+
+import torch
+from torch import nn
+from torch.nn.parameter import Parameter
+
+from vllm.distributed import get_tensor_model_parallel_rank
+from vllm.forward_context import get_forward_context
+from vllm.model_executor.utils import set_weight_attrs
+
+from .ops import fused_sconv
+from .sconv_swa_attn import InklingConvState, InklingSconvMetadata
+
+
+class InklingShortConv(nn.Module):
+    def __init__(
+        self, dim: int, kernel_size: int, owner: InklingConvState, stream_idx: int
+    ) -> None:
+        super().__init__()
+        self.dim = dim
+        self.kernel_size = kernel_size
+        self.owner = owner
+        self.stream_idx = stream_idx
+        self.tp_rank = get_tensor_model_parallel_rank()
+
+        # Depthwise conv weight; checkpoint stores (dim, 1, W).
+        self.weight = Parameter(torch.empty(dim, 1, kernel_size), requires_grad=False)
+        set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
+
+    def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor) -> None:
+        if loaded_weight.shape[0] != param.shape[0]:
+            shard = param.shape[0]
+            loaded_weight = loaded_weight.narrow(0, self.tp_rank * shard, shard)
+        param.data.copy_(loaded_weight)
+
+    def forward(self, x: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
+        # x: (num_tokens, dim); positions: (num_tokens,) absolute positions.
+        attn_metadata = get_forward_context().attn_metadata
+        if not isinstance(attn_metadata, dict):
+            # Memory-profiling / no metadata: identity (residual).
+            return x
+        m = attn_metadata.get(self.owner.prefix)
+        if m is None:
+            return x
+        assert isinstance(m, InklingSconvMetadata)
+        cache = self.owner.kv_cache
+        if cache.numel() == 0:
+            # Cache not yet bound (profiling before KV alloc): identity.
+            return x
+
+        off_s, ws = self.owner.stream_ranges[self.stream_idx]
+        block_size = self.owner.block_size
+        x = x.contiguous()
+        weight = self.weight.squeeze(1)  # (dim, W)
+
+        return fused_sconv(
+            x,
+            weight,
+            cache,
+            positions,
+            m.block_table,
+            m.seq_idx,
+            m.slot_mapping,
+            m.query_start,
+            off_s,
+            ws,
+            block_size,
+            activation=None,
+            use_residual=True,
+        )

vllm/parser/engine/parser_engine.py

@@ -110,6 +110,8 @@ class ParserEngine(Parser):
 
         self._has_reasoning = (
             "THINK_END" in parser_engine_config.token_id_terminals
+            or "THINK_START" in parser_engine_config.terminals
+            or "THINK_END" in parser_engine_config.terminals
             or parser_engine_config.initial_state == ParserState.REASONING
         )
         self._reasoning_ended: bool = not self._has_reasoning

vllm/parser/engine/parser_engine_config.py

@@ -28,6 +28,7 @@ from vllm.parser.engine.events import EventType
 class ParserState(Enum):
     CONTENT = auto()
     REASONING = auto()
+    MESSAGE_HEADER = auto()
     TOOL_PREAMBLE = auto()
     TOOL_NAME = auto()
     TOOL_ARGS = auto()

vllm/parser/engine/registered_adapters.py

@@ -12,6 +12,7 @@ from vllm.parser.deepseek_v32 import DeepSeekV32Parser
 from vllm.parser.engine.adapters import make_adapters
 from vllm.parser.gemma4 import Gemma4Parser
 from vllm.parser.glm47_moe import Glm47MoeParser
+from vllm.parser.inkling import InklingParser
 from vllm.parser.kimi_k2 import KimiK2Parser
 from vllm.parser.minimax_m2 import MinimaxM2Parser
 from vllm.parser.nemotron_v3 import NemotronV3Parser
@@ -62,3 +63,8 @@ from vllm.parser.seed_oss import SeedOssParser
     KimiK2ParserReasoningAdapter,
     KimiK2ParserToolAdapter,
 ) = make_adapters(KimiK2Parser)
+
+(
+    InklingParserReasoningAdapter,
+    InklingParserToolAdapter,
+) = make_adapters(InklingParser)

vllm/parser/engine/streaming_parser_engine.py

@@ -269,6 +269,8 @@ class StreamingParserEngine:
                 SemanticEvent(EventType.REASONING_END, tool_index=self.tool_index)
             )
             self.state = ParserState.CONTENT
+        elif self.state == ParserState.MESSAGE_HEADER:
+            self.state = ParserState.CONTENT
 
         return events
 
@@ -314,6 +316,15 @@ class StreamingParserEngine:
             return self._emit_for_state(value)
 
         if self.skip_tool_parsing and terminal in self._tool_terminals:
+            if self.state == ParserState.MESSAGE_HEADER:
+                self.state = ParserState.CONTENT
+                return [
+                    SemanticEvent(
+                        EventType.TEXT_CHUNK,
+                        value=value,
+                        tool_index=self.tool_index,
+                    )
+                ]
             if EventType.REASONING_END in transition.events:
                 self.state = ParserState.CONTENT
                 return [

vllm/parser/inkling.py

@@ -0,0 +1,411 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling parser: typed content blocks parsed by a single state machine.
+
+Inkling output format (every marker is a dedicated special token)::
+
+    <|message_model|><|content_thinking|>...reasoning...<|end_message|>
+    <|message_model|><|content_text|>...visible text...<|end_message|>
+    <|message_model|><|content_invoke_tool_json|>
+        {"name":"get_weather","args":{"city":"SF"}}<|end_message|>
+
+Blocks are self-describing and may repeat in any order; sampling may
+also end a block with the standalone ``<|content_model_end_sampling|>``
+token. The tool-call payload is a single JSON object whose ``name`` is
+extracted by the engine's name-from-args path and whose ``args`` object
+is carved out of the wrapper by :func:`_inkling_arg_converter`.
+
+Note the terminal *labels*: ``THINK_START``/``THINK_END`` are what the
+engine keys its reasoning plumbing on (``is_reasoning_end``,
+``count_reasoning_tokens``, initial-state seeding), so ``<|end_message|>``
+is labelled ``THINK_END`` here even though it ends every block kind —
+the transition table, not the label, carries the semantics.
+"""
+
+from __future__ import annotations
+
+import functools
+import json
+from collections.abc import Sequence
+from typing import TYPE_CHECKING
+
+from vllm.entrypoints.openai.engine.protocol import (
+    ExtractedToolCallInformation,
+)
+from vllm.parser.engine.events import EventType
+from vllm.parser.engine.parser_engine import ParserEngine
+from vllm.parser.engine.parser_engine_config import (
+    ParserEngineConfig,
+    ParserState,
+    Transition,
+)
+
+if TYPE_CHECKING:
+    from vllm.tokenizers import TokenizerLike
+    from vllm.tool_parsers.abstract_tool_parser import Tool
+
+MESSAGE_MODEL = "<|message_model|>"
+CONTENT_TEXT = "<|content_text|>"
+CONTENT_THINKING = "<|content_thinking|>"
+CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>"
+CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>"
+CONTENT_TOOL_ERROR = "<|content_tool_error|>"
+CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>"
+END_MESSAGE = "<|end_message|>"
+
+INKLING_SPECIAL_TOKENS = (
+    MESSAGE_MODEL,
+    CONTENT_TEXT,
+    CONTENT_THINKING,
+    CONTENT_INVOKE_TOOL_JSON,
+    CONTENT_INVOKE_TOOL_TEXT,
+    CONTENT_TOOL_ERROR,
+    CONTENT_MODEL_END_SAMPLING,
+    END_MESSAGE,
+)
+
+_WS = " \t\r\n"
+
+
+def _scan_json_value(raw: str, start: int) -> int | None:
+    """Return the end index (exclusive) of the JSON object starting at
+    ``raw[start]``, or ``None`` when the object is still unterminated."""
+    depth = 0
+    in_string = False
+    escape = False
+    for i in range(start, len(raw)):
+        ch = raw[i]
+        if escape:
+            escape = False
+            continue
+        if in_string:
+            if ch == "\\":
+                escape = True
+            elif ch == '"':
+                in_string = False
+            continue
+        if ch == '"':
+            in_string = True
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return i + 1
+    return None
+
+
+def _args_value_span(raw: str) -> str | None:
+    """Extract the raw text span of the top-level ``"args"`` value from a
+    (possibly incomplete) ``{"name":...,"args":{...}}`` wrapper.
+
+    Returns the verbatim substring (prefix-stable across growing input,
+    which the engine's argument-delta diffing relies on), possibly an
+    unterminated object prefix; ``None`` when the value has not started.
+    Raises ``ValueError`` when the value is not a JSON object.
+    """
+    depth = 0
+    in_string = False
+    escape = False
+    string_start = -1
+    last_string: str | None = None
+    for i, ch in enumerate(raw):
+        if escape:
+            escape = False
+            continue
+        if in_string:
+            if ch == "\\":
+                escape = True
+            elif ch == '"':
+                in_string = False
+                if depth == 1:
+                    last_string = raw[string_start + 1 : i]
+            continue
+        if ch == '"':
+            in_string = True
+            string_start = i
+        elif ch == ":" and depth == 1 and last_string == "args":
+            value_start = i + 1
+            while value_start < len(raw) and raw[value_start] in _WS:
+                value_start += 1
+            if value_start >= len(raw):
+                return None
+            if raw[value_start] != "{":
+                raise ValueError("Inkling tool call args must be a JSON object")
+            value_end = _scan_json_value(raw, value_start)
+            if value_end is None:
+                return raw[value_start:]
+            return raw[value_start:value_end]
+        elif ch in "{[":
+            depth += 1
+        elif ch in "}]":
+            depth -= 1
+    return None
+
+
+def _inkling_arg_converter(raw_args: str, partial: bool) -> str:
+    """Carve the ``args`` object out of the tool-call JSON wrapper.
+
+    Why a converter at all: the engine's ``tool_args_json`` machinery
+    treats the *entire* TOOL_ARGS text as the tool arguments, but Inkling's
+    payload is the ``{"name":...,"args":{...}}`` wrapper — without a
+    converter, ``_compute_arg_delta`` streams the wrapper verbatim into
+    the OpenAI ``arguments`` field (``converter is None -> raw delta``,
+    unconditionally; ``stream_arg_deltas=False`` does not stop it).
+
+    Why a hand-rolled scanner instead of (partial) ``json.loads`` +
+    ``json.dumps``: the engine diffs successive converter outputs and
+    requires each to extend the previous one (``startswith``); a
+    violation silently drops argument deltas. Re-serialization changes
+    whitespace and closes unterminated structures differently across
+    ticks, so the only prefix-stable output is a verbatim substring of
+    the input. The scanner is also string/escape-aware so an ``"args"``
+    literal inside the name or a string value cannot mislead it, and it
+    still recovers a partial span when EOS truncates the wrapper (where
+    ``json.loads`` would fail).
+    """
+    span = _args_value_span(raw_args)
+    if span is None:
+        # No args value yet (streaming) or none at all (treat as empty).
+        return "" if partial else "{}"
+    return span
+
+
+@functools.cache
+def inkling_config() -> ParserEngineConfig:
+    terminals = {
+        "MSG_MODEL": MESSAGE_MODEL,
+        "TEXT_START": CONTENT_TEXT,
+        "THINK_START": CONTENT_THINKING,
+        "THINK_END": END_MESSAGE,
+        "END_SAMPLING": CONTENT_MODEL_END_SAMPLING,
+        "TOOL_START": CONTENT_INVOKE_TOOL_JSON,
+        "TOOL_TEXT": CONTENT_INVOKE_TOOL_TEXT,
+        "TOOL_ERROR": CONTENT_TOOL_ERROR,
+    }
+    transitions: dict[tuple[ParserState, str], Transition] = {
+        # ── Between blocks / inside a text block ──────────────────────
+        (ParserState.CONTENT, "MSG_MODEL"): Transition(
+            ParserState.MESSAGE_HEADER,
+            (),
+        ),
+        (ParserState.CONTENT, "TEXT_START"): Transition(
+            ParserState.CONTENT,
+            (),
+        ),
+        (ParserState.CONTENT, "THINK_START"): Transition(
+            ParserState.REASONING,
+            (EventType.REASONING_START,),
+        ),
+        (ParserState.CONTENT, "TOOL_START"): Transition(
+            ParserState.TOOL_ARGS,
+            (EventType.TOOL_CALL_START,),
+        ),
+        # Raw / error tool blocks render as visible text.
+        (ParserState.CONTENT, "TOOL_TEXT"): Transition(
+            ParserState.CONTENT,
+            (),
+        ),
+        (ParserState.CONTENT, "TOOL_ERROR"): Transition(
+            ParserState.CONTENT,
+            (),
+        ),
+        # The optional function name between the model-role and content-kind
+        # markers is metadata, not visible assistant content.
+        (ParserState.MESSAGE_HEADER, "MSG_MODEL"): Transition(
+            ParserState.MESSAGE_HEADER,
+            (),
+        ),
+        (ParserState.MESSAGE_HEADER, "TEXT_START"): Transition(
+            ParserState.CONTENT,
+            (),
+        ),
+        (ParserState.MESSAGE_HEADER, "THINK_START"): Transition(
+            ParserState.REASONING,
+            (EventType.REASONING_START,),
+        ),
+        (ParserState.MESSAGE_HEADER, "TOOL_START"): Transition(
+            ParserState.TOOL_ARGS,
+            (EventType.TOOL_CALL_START,),
+        ),
+        (ParserState.MESSAGE_HEADER, "TOOL_TEXT"): Transition(
+            ParserState.CONTENT,
+            (),
+        ),
+        (ParserState.MESSAGE_HEADER, "TOOL_ERROR"): Transition(
+            ParserState.CONTENT,
+            (),
+        ),
+        # ── Inside a thinking block ───────────────────────────────────
+        (ParserState.REASONING, "THINK_START"): Transition(
+            ParserState.REASONING,
+            (),
+        ),
+        # Defensive: tool call opening while a thinking block is still
+        # unclosed.
+        (ParserState.REASONING, "TOOL_START"): Transition(
+            ParserState.TOOL_ARGS,
+            (EventType.REASONING_END, EventType.TOOL_CALL_START),
+        ),
+    }
+    # Block-end terminals behave identically regardless of label. A
+    # closed tool block returns to CONTENT (Inkling has no section wrapper;
+    # blocks of any kind may follow), which also keeps the block-kind
+    # and role tokens out of the engine's tool-terminal set so the
+    # skip_tool_parsing reasoning pass still classifies reasoning.
+    for end in ("THINK_END", "END_SAMPLING"):
+        transitions[(ParserState.CONTENT, end)] = Transition(
+            ParserState.CONTENT,
+            (),
+        )
+        transitions[(ParserState.REASONING, end)] = Transition(
+            ParserState.CONTENT,
+            (EventType.REASONING_END,),
+        )
+        transitions[(ParserState.TOOL_ARGS, end)] = Transition(
+            ParserState.CONTENT,
+            (EventType.TOOL_CALL_END,),
+        )
+        transitions[(ParserState.MESSAGE_HEADER, end)] = Transition(
+            ParserState.CONTENT,
+            (),
+        )
+
+    return ParserEngineConfig(
+        name="inkling",
+        # Normal generation continues after a prompt-prefilled
+        # `<|message_model|>`. Non-streaming parsing receives only the generated
+        # suffix, so begin in the corresponding message-header state as well.
+        initial_state=ParserState.MESSAGE_HEADER,
+        terminals=terminals,
+        # Inkling content-kind markers are the grammar. When the engine is
+        # used through DelegatingParser, the reasoning pass can hand the tool
+        # pass reconstructed text whose token-id slice no longer contains the
+        # content-kind marker that starts the tool block, so keep Inkling on
+        # the text grammar instead of token-id-only terminal matching.
+        token_id_terminals={},
+        transitions=transitions,
+        arg_converter=_inkling_arg_converter,
+        stream_arg_deltas=True,
+        tool_args_json=True,
+        strip_trailing_reasoning_whitespace=True,
+        drop_whitespace_only_content_before_tools=True,
+        strip_content_whitespace_with_tools=False,
+        validate_tool_names=False,
+    )
+
+
+class InklingParser(ParserEngine):
+    CONFIG_NAME = "inkling"
+
+    def __init__(
+        self,
+        tokenizer: TokenizerLike,
+        tools: list[Tool] | None = None,
+        **kwargs,
+    ) -> None:
+        kwargs.setdefault("parser_engine_config", inkling_config())
+        super().__init__(tokenizer, tools, **kwargs)
+
+    def adjust_initial_state_from_prompt(self, prompt_token_ids: Sequence[int]) -> None:
+        """Seed the initial parsing state from the prompt tail.
+
+        Mirrors the Rust parser's ``initialize()``: scanning the prompt
+        backwards, the last relevant special token decides whether generation
+        continues inside a thinking block, a text block, a model message
+        header, or between blocks.
+        """
+        vocab = self.vocab
+        thinking_id = vocab.get(CONTENT_THINKING)
+        text_id = vocab.get(CONTENT_TEXT)
+        model_id = vocab.get(MESSAGE_MODEL)
+        special_ids = {vocab[text] for text in INKLING_SPECIAL_TOKENS if text in vocab}
+        for token_id in reversed(prompt_token_ids):
+            if token_id == thinking_id:
+                self._engine.reset(initial_state=ParserState.REASONING)
+                self._streaming_initialized = True
+                return
+            if token_id == text_id:
+                self._engine.reset(initial_state=ParserState.CONTENT)
+                self._streaming_initialized = True
+                return
+            if token_id == model_id:
+                self._engine.reset(initial_state=ParserState.MESSAGE_HEADER)
+                self._streaming_initialized = True
+                return
+            if token_id in special_ids:
+                break
+
+    def is_reasoning_end(self, input_ids: list[int]) -> bool:
+        vocab = self.vocab
+        thinking_id = vocab.get(CONTENT_THINKING)
+        text_id = vocab.get(CONTENT_TEXT)
+        model_id = vocab.get(MESSAGE_MODEL)
+        end_sampling_id = vocab.get(CONTENT_MODEL_END_SAMPLING)
+        for token_id in reversed(input_ids):
+            if token_id in (thinking_id, model_id):
+                return False
+            if token_id in (text_id, end_sampling_id):
+                return True
+        return False
+
+    def count_reasoning_tokens(self, token_ids: Sequence[int]) -> int:
+        vocab = self.vocab
+        thinking_id = vocab.get(CONTENT_THINKING)
+        end_ids = {
+            token_id
+            for token_id in (
+                vocab.get(END_MESSAGE),
+                vocab.get(CONTENT_MODEL_END_SAMPLING),
+            )
+            if token_id is not None
+        }
+        in_reasoning = False
+        count = 0
+        for token_id in token_ids:
+            if token_id == thinking_id:
+                in_reasoning = True
+                continue
+            if token_id in end_ids:
+                in_reasoning = False
+                continue
+            if in_reasoning:
+                count += 1
+        return count
+
+    def _single_pass_parse(
+        self,
+        text: str,
+        token_ids: Sequence[int],
+        initial_state: ParserState | None = None,
+    ) -> tuple[str | None, str | None, ExtractedToolCallInformation]:
+        reasoning, content, tool_call_info = super()._single_pass_parse(
+            text, token_ids, initial_state=initial_state
+        )
+        # The engine defers content that follows tool-call events within a
+        # single pass; Inkling allows text blocks after tool-call blocks, so
+        # flush the trailing text (matching the Rust unified parser).
+        if self._deferred_content:
+            trailing = self._deferred_content
+            self._deferred_content = ""
+            content = self._strip_content_whitespace(
+                (content or "") + trailing,
+                tool_call_info.tools_called,
+            )
+            tool_call_info = ExtractedToolCallInformation(
+                tools_called=tool_call_info.tools_called,
+                tool_calls=tool_call_info.tool_calls,
+                content=content,
+            )
+        return reasoning, content, tool_call_info
+
+    @staticmethod
+    def _extract_args_value(parsed: dict) -> str | None:
+        # Inkling wraps arguments under "args" rather than "arguments".
+        for key in ("args", "arguments", "parameters"):
+            if key in parsed:
+                val = parsed[key]
+                if isinstance(val, str):
+                    return val
+                return json.dumps(val, ensure_ascii=False)
+        return None

vllm/reasoning/__init__.py

@@ -128,6 +128,10 @@ _REASONING_PARSERS_TO_REGISTER = {
         "step3p5_reasoning_parser",
         "Step3p5ReasoningParser",
     ),
+    "inkling": (
+        "inkling_reasoning_parser",
+        "InklingParserReasoningAdapter",
+    ),
 }
 
 

vllm/reasoning/inkling_reasoning_parser.py

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm.parser.engine.registered_adapters import InklingParserReasoningAdapter
+
+__all__ = ["InklingParserReasoningAdapter"]

vllm/renderers/inkling.py

@@ -0,0 +1,178 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Native Inkling chat renderer for the Python frontend.
+
+Mirrors the Rust frontend's native Inkling renderer
+(``rust/src/chat/src/renderer/inkling/mod.rs``): chat messages are rendered
+directly to token ids — Inkling has no Jinja chat template and no faithful
+text form.
+
+The encoding logic lives in ``inkling_encoding.py`` behind a narrow
+"OpenAI messages + tools -> token ids" call; see the swap-point comment
+in :meth:`InklingRenderer._render` for adopting a standalone Inkling
+input-processing library (mistral-common style) later.
+"""
+
+from vllm.config import VllmConfig
+from vllm.entrypoints.chat_utils import (
+    ChatCompletionMessageParam,
+    ConversationMessage,
+    parse_chat_messages,
+    parse_chat_messages_async,
+)
+from vllm.logger import init_logger
+from vllm.tokenizers.hf import HfTokenizer
+from vllm.utils.async_utils import make_async
+
+from .base import BaseRenderer
+from .inkling_encoding import SPECIAL_TOKEN_SPELLINGS, render_inkling_messages
+from .inputs import DictPrompt
+from .inputs.preprocess import parse_dec_only_prompt
+from .params import ChatParams
+
+logger = init_logger(__name__)
+
+_NAMED_REASONING_EFFORT = {
+    "none": 0.0,
+    "minimal": 0.1,
+    "low": 0.2,
+    "medium": 0.7,
+    "high": 0.9,
+    "xhigh": 0.99,
+    "max": 0.99,
+}
+
+_DEFAULT_REASONING_EFFORT = 0.9
+
+
+def _resolve_reasoning_effort(value: object) -> float | int | None:
+    if value is None:
+        return _DEFAULT_REASONING_EFFORT
+    if isinstance(value, str):
+        return _NAMED_REASONING_EFFORT.get(value)
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    return value
+
+
+class _HfBackedTmlTokenizer:
+    """Adapts an HF tokenizer to the encoding core's tokenizer protocol.
+
+    Special-token ids are resolved from the tokenizer vocab at
+    construction (never hardcoded), trying each known spelling — the Inkling
+    HF vocab exposes some semantic slots as ``<|unused_NNNNNN|>`` tokens.
+    """
+
+    def __init__(self, tokenizer: HfTokenizer) -> None:
+        self._tokenizer = tokenizer
+
+        vocab = tokenizer.get_vocab()
+        special_ids: dict[str, int] = {}
+        missing: list[str] = []
+        for token, spellings in SPECIAL_TOKEN_SPELLINGS.items():
+            for spelling in spellings:
+                token_id = vocab.get(spelling)
+                if token_id is not None:
+                    special_ids[token] = token_id
+                    break
+            else:
+                missing.append(token)
+        if missing:
+            raise ValueError(f"Inkling tokenizer is missing special tokens: {missing}")
+        self._special_ids = special_ids
+
+    def encode_text(self, text: str) -> list[int]:
+        return self._tokenizer.encode(text, add_special_tokens=False)
+
+    def encode_special(self, token: str) -> int:
+        return self._special_ids[token]
+
+
+class InklingRenderer(BaseRenderer[HfTokenizer]):
+    def __init__(
+        self,
+        config: VllmConfig,
+        tokenizer: HfTokenizer | None,
+    ) -> None:
+        super().__init__(config, tokenizer)
+
+        self._inkling_tokenizer = _HfBackedTmlTokenizer(self.get_tokenizer())
+        self._render_async = make_async(self._render, executor=self._executor)
+
+    def _render(
+        self,
+        messages: list[ChatCompletionMessageParam],
+        params: ChatParams,
+    ) -> list[int]:
+        kwargs = params.chat_template_kwargs or {}
+
+        if kwargs.get("continue_final_message"):
+            raise ValueError("Inkling renderer does not support continue_final_message")
+
+        reasoning_effort = _resolve_reasoning_effort(kwargs.get("reasoning_effort"))
+
+        try:
+            # Swap point: to adopt a standalone Inkling input-processing
+            # library, replace this call (and the _HfBackedTmlTokenizer
+            # adapter above) with the library's renderer.
+            return render_inkling_messages(
+                messages,
+                self._inkling_tokenizer,
+                add_generation_prompt=kwargs.get("add_generation_prompt", True),
+                tools=kwargs.get("tools"),
+                reasoning_effort=reasoning_effort,
+            )
+        except ValueError:
+            raise
+        except (TypeError, KeyError) as e:
+            # Malformed request content; surface as a request error.
+            raise ValueError(str(e)) from e
+        except Exception as e:
+            logger.exception("Error while rendering Inkling chat messages")
+            raise ValueError(str(e)) from e
+
+    def render_messages(
+        self,
+        messages: list[ChatCompletionMessageParam],
+        params: ChatParams,
+    ) -> tuple[list[ConversationMessage], DictPrompt]:
+        conversation, mm_data, mm_uuids = parse_chat_messages(
+            messages,
+            self.model_config,
+            content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
+            mm_processor_kwargs=params.mm_processor_kwargs,
+        )
+
+        token_ids = self._render(messages, params)
+
+        prompt = parse_dec_only_prompt(token_ids)
+        if mm_data is not None:
+            prompt["multi_modal_data"] = mm_data
+        if mm_uuids is not None:
+            prompt["multi_modal_uuids"] = mm_uuids
+
+        return conversation, prompt
+
+    async def render_messages_async(
+        self,
+        messages: list[ChatCompletionMessageParam],
+        params: ChatParams,
+    ) -> tuple[list[ConversationMessage], DictPrompt]:
+        conversation, mm_data, mm_uuids = await parse_chat_messages_async(
+            messages,
+            self.model_config,
+            content_format="string",
+            media_io_kwargs=params.media_io_kwargs,
+            mm_processor_kwargs=params.mm_processor_kwargs,
+        )
+
+        token_ids = await self._render_async(messages, params)
+
+        prompt = parse_dec_only_prompt(token_ids)
+        if mm_data is not None:
+            prompt["multi_modal_data"] = mm_data
+        if mm_uuids is not None:
+            prompt["multi_modal_uuids"] = mm_uuids
+
+        return conversation, prompt

vllm/renderers/inkling_encoding.py

@@ -0,0 +1,383 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Inkling chat-encoding core.
+
+Pure implementation of Inkling chat rendering, kept deliberately free of
+vLLM imports: it depends only on the :class:`InklingTextTokenizer` protocol
+and speaks OpenAI-style message dicts. If a standalone Inkling
+input-processing library becomes available, this module is the unit to
+swap out — the swap point is marked in ``vllm/renderers/inkling.py``.
+
+Multimodal parts follow the contract of vLLM's Inkling multimodal processor
+(``InklingMultiModalProcessor`` anchors prompt updates on the bare
+content-kind marker and inserts the per-patch placeholder run itself):
+
+* image parts emit only ``<|content_image|>`` — no seed placeholder id;
+* audio parts emit ``<|content_audio_input|><|audio_end|>`` — no seed
+  placeholder id between them.
+
+``reasoning_effort`` (a float in [0, 0.99], sourced from
+``chat_template_kwargs`` only) renders the ``Thinking effort level:``
+system control block after the tool declarations and initial system messages.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from typing import Any, Protocol
+
+END_OF_TEXT = "<|endoftext|>"
+MESSAGE_USER = "<|message_user|>"
+MESSAGE_MODEL = "<|message_model|>"
+MESSAGE_SYSTEM = "<|message_system|>"
+MESSAGE_TOOL = "<|message_tool|>"
+CONTENT_TEXT = "<|content_text|>"
+CONTENT_IMAGE = "<|content_image|>"
+CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>"
+CONTENT_THINKING = "<|content_thinking|>"
+CONTENT_AUDIO_INPUT = "<|content_audio_input|>"
+CONTENT_TOOL_ERROR = "<|content_tool_error|>"
+CONTENT_XML = "<|content_xml|>"
+CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>"
+CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>"
+END_MESSAGE = "<|end_message|>"
+AUDIO_END = "<|audio_end|>"
+
+ROLE_MESSAGE_TOKENS: dict[str, str] = {
+    "user": MESSAGE_USER,
+    "assistant": MESSAGE_MODEL,
+    "system": MESSAGE_SYSTEM,
+    # The developer role folds into the system role on the Inkling wire.
+    "developer": MESSAGE_SYSTEM,
+    "tool": MESSAGE_TOOL,
+}
+
+# Alternate vocab spellings per semantic token. The Inkling HF tokenizer
+# exposes some semantic slots as ``<|unused_NNNNNN|>`` tokens (notably
+# CONTENT_XML = <|unused_200024|>), so ID resolution must try these
+# spellings in order.
+SPECIAL_TOKEN_SPELLINGS: dict[str, tuple[str, ...]] = {
+    MESSAGE_USER: (MESSAGE_USER,),
+    MESSAGE_MODEL: (MESSAGE_MODEL,),
+    MESSAGE_SYSTEM: (MESSAGE_SYSTEM,),
+    MESSAGE_TOOL: (MESSAGE_TOOL,),
+    CONTENT_TEXT: (CONTENT_TEXT,),
+    CONTENT_IMAGE: (CONTENT_IMAGE,),
+    CONTENT_MODEL_END_SAMPLING: (CONTENT_MODEL_END_SAMPLING,),
+    CONTENT_THINKING: (CONTENT_THINKING,),
+    CONTENT_AUDIO_INPUT: (CONTENT_AUDIO_INPUT,),
+    CONTENT_XML: (CONTENT_XML, "<|unused_200024|>"),
+    CONTENT_INVOKE_TOOL_JSON: (CONTENT_INVOKE_TOOL_JSON,),
+    END_MESSAGE: (END_MESSAGE,),
+    AUDIO_END: (AUDIO_END,),
+}
+
+
+class InklingTextTokenizer(Protocol):
+    """Structural tokenizer contract required by the renderer."""
+
+    def encode_text(self, text: str) -> list[int]: ...
+
+    def encode_special(self, token: str) -> int: ...
+
+
+# OpenAI content-part type spellings that mean image / audio (rendering only
+# needs the kind, not the bytes — the bytes are handled by the MM processor).
+_IMAGE_PART_TYPES = frozenset({"image", "input_image", "image_url"})
+_AUDIO_PART_TYPES = frozenset({"audio", "input_audio", "audio_url"})
+
+_MAX_REASONING_EFFORT = 0.99
+
+
+def render_inkling_messages(
+    messages: Sequence[Mapping[str, Any]],
+    tokenizer: InklingTextTokenizer,
+    *,
+    add_generation_prompt: bool = True,
+    tools: Sequence[Mapping[str, Any]] | None = None,
+    reasoning_effort: float | None = None,
+) -> list[int]:
+    """Render chat messages to Inkling input ids.
+
+    PURE renderer: emits Inkling framing plus bare media markers; media
+    encoding and placeholder expansion happen later in the MM processor.
+    ``add_generation_prompt`` appends the assistant turn opener so the
+    model continues into the response.
+    """
+    input_ids: list[int] = []
+    tool_call_id_to_name: dict[str, str] = {}
+
+    # Request-level tools plus per-developer-message tools (Rust renderer
+    # semantics) are declared in a single leading system block.
+    all_tools = list(tools or [])
+    for message in messages:
+        if message.get("role") == "developer":
+            all_tools.extend(message.get("tools") or [])
+
+    if all_tools:
+        _append_message(
+            input_ids,
+            tokenizer,
+            "system",
+            "xml",
+            _tool_declare_json(all_tools),
+            author_name="tool_declare",
+        )
+
+    for message in messages:
+        role = _expect_role(message)
+        if role not in {"system", "developer"} and reasoning_effort is not None:
+            _append_reasoning_effort(input_ids, tokenizer, reasoning_effort)
+            reasoning_effort = None
+
+        if role == "tool":
+            tool_name = message.get("name") or tool_call_id_to_name.get(
+                str(message.get("tool_call_id") or ""), ""
+            )
+            _append_message(
+                input_ids,
+                tokenizer,
+                "tool",
+                "text",
+                _flatten_text_content(message.get("content")),
+                author_name=str(tool_name),
+            )
+            continue
+
+        if role == "assistant":
+            reasoning_content = message.get("reasoning")
+            if reasoning_content is None:
+                reasoning_content = message.get("reasoning_content")
+            if reasoning_content:
+                if not isinstance(reasoning_content, str):
+                    raise TypeError(
+                        "assistant reasoning_content must be a string for "
+                        "Inkling rendering"
+                    )
+                _append_message(
+                    input_ids,
+                    tokenizer,
+                    "assistant",
+                    "thinking",
+                    reasoning_content,
+                )
+
+        for kind, text in _iter_render_parts(message.get("content")):
+            _append_message(input_ids, tokenizer, role, kind, text)
+
+        if role == "assistant":
+            for tool_call in message.get("tool_calls") or []:
+                name, args = _tool_call_name_and_args(tool_call)
+                tool_call_id = _as_mapping(tool_call).get("id")
+                if tool_call_id:
+                    tool_call_id_to_name[str(tool_call_id)] = name
+                _append_message(
+                    input_ids,
+                    tokenizer,
+                    "assistant",
+                    "invoke_tool_json",
+                    _tool_call_json(name, args),
+                    author_name=name,
+                )
+
+            input_ids.append(tokenizer.encode_special(CONTENT_MODEL_END_SAMPLING))
+
+    if reasoning_effort is not None:
+        _append_reasoning_effort(input_ids, tokenizer, reasoning_effort)
+
+    if add_generation_prompt:
+        input_ids.append(tokenizer.encode_special(MESSAGE_MODEL))
+    return input_ids
+
+
+def _append_reasoning_effort(
+    input_ids: list[int],
+    tokenizer: InklingTextTokenizer,
+    reasoning_effort: float,
+) -> None:
+    _append_message(
+        input_ids,
+        tokenizer,
+        "system",
+        "text",
+        _thinking_effort_text(reasoning_effort),
+    )
+
+
+def _thinking_effort_text(reasoning_effort: Any) -> str:
+    if isinstance(reasoning_effort, bool) or not isinstance(
+        reasoning_effort, (int, float)
+    ):
+        raise TypeError(
+            "Inkling reasoning_effort must be a number in [0.0, 0.99], "
+            f"got {type(reasoning_effort).__name__}"
+        )
+    value = float(reasoning_effort)
+    if not 0.0 <= value <= _MAX_REASONING_EFFORT:
+        raise ValueError(
+            f"Inkling reasoning_effort must be in [0.0, 0.99], got {value}"
+        )
+
+    effort_text = f"{value:.2f}".rstrip("0").rstrip(".")
+    if effort_text in {"0", "-0"}:
+        effort_text = "0.0"
+    return f"Thinking effort level: {effort_text}"
+
+
+def _append_message(
+    input_ids: list[int],
+    tokenizer: InklingTextTokenizer,
+    role: str,
+    kind: str,
+    text: str,
+    *,
+    author_name: str | None = None,
+) -> None:
+    input_ids.append(tokenizer.encode_special(ROLE_MESSAGE_TOKENS[role]))
+    if author_name:
+        input_ids.extend(tokenizer.encode_text(author_name))
+
+    if kind == "text":
+        input_ids.append(tokenizer.encode_special(CONTENT_TEXT))
+        input_ids.extend(tokenizer.encode_text(text))
+    elif kind == "image":
+        # Bare marker only: the MM processor replaces the marker with
+        # `marker + placeholder * num_patches` (see mm_preprocess.py).
+        input_ids.append(tokenizer.encode_special(CONTENT_IMAGE))
+    elif kind == "audio":
+        input_ids.append(tokenizer.encode_special(CONTENT_AUDIO_INPUT))
+        input_ids.append(tokenizer.encode_special(AUDIO_END))
+    elif kind == "thinking":
+        input_ids.append(tokenizer.encode_special(CONTENT_THINKING))
+        input_ids.extend(tokenizer.encode_text(text))
+    elif kind == "xml":
+        input_ids.append(tokenizer.encode_special(CONTENT_XML))
+        input_ids.extend(tokenizer.encode_text(text))
+    elif kind == "invoke_tool_json":
+        input_ids.append(tokenizer.encode_special(CONTENT_INVOKE_TOOL_JSON))
+        input_ids.extend(tokenizer.encode_text(text))
+    else:
+        raise ValueError(f"unsupported Inkling render part kind: {kind!r}")
+
+    input_ids.append(tokenizer.encode_special(END_MESSAGE))
+
+
+def _iter_render_parts(content: Any) -> Iterator[tuple[str, str]]:
+    """Yield (kind, text) per content part: kind in {text, image, audio}."""
+    if content is None:
+        return
+    if isinstance(content, str):
+        if content:
+            yield ("text", content)
+        return
+    if not isinstance(content, Sequence) or isinstance(content, (bytes, bytearray)):
+        raise TypeError("message content must be a string or a sequence of parts")
+    for part in content:
+        if isinstance(part, str):
+            yield ("text", part)
+            continue
+        if not isinstance(part, Mapping):
+            raise TypeError(f"content part must be mapping, got {type(part).__name__}")
+        ptype = part.get("type")
+        if ptype in (None, "text", "input_text"):
+            text = part.get("text", "")
+            yield ("text", text if isinstance(text, str) else "")
+        elif ptype in _IMAGE_PART_TYPES:
+            yield ("image", "")
+        elif ptype in _AUDIO_PART_TYPES:
+            yield ("audio", "")
+        else:
+            raise ValueError(f"unsupported content part type: {ptype!r}")
+
+
+def _flatten_text_content(content: Any) -> str:
+    """Flatten a tool-response content (string or text parts) to text."""
+    if content is None:
+        return ""
+    if isinstance(content, str):
+        return content
+    parts: list[str] = []
+    for kind, text in _iter_render_parts(content):
+        if kind != "text":
+            raise ValueError(
+                "Inkling tool response content must be text, "
+                f"got a part of kind {kind!r}"
+            )
+        parts.append(text)
+    return "".join(parts)
+
+
+def _expect_role(message: Mapping[str, Any]) -> str:
+    role = message.get("role")
+    if role not in ROLE_MESSAGE_TOKENS:
+        raise ValueError(
+            f"unsupported Inkling message role {role!r}; "
+            f"expected one of {sorted(ROLE_MESSAGE_TOKENS)}"
+        )
+    return str(role)
+
+
+def _as_mapping(value: Any) -> Mapping[str, Any]:
+    if isinstance(value, Mapping):
+        return value
+    if hasattr(value, "model_dump"):
+        dumped = value.model_dump()
+        if isinstance(dumped, Mapping):
+            return dumped
+    raise TypeError(f"expected mapping, got {type(value).__name__}")
+
+
+def _canonical_json(value: Any) -> str:
+    return json.dumps(
+        _sort_json(value),
+        ensure_ascii=False,
+        allow_nan=False,
+        separators=(",", ":"),
+    )
+
+
+def _sort_json(value: Any) -> Any:
+    if isinstance(value, Mapping):
+        return {str(key): _sort_json(value[key]) for key in sorted(value)}
+    if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
+        return [_sort_json(item) for item in value]
+    return value
+
+
+def _tool_declare_json(tools: Sequence[Mapping[str, Any]]) -> str:
+    tool_specs = []
+    for tool_value in tools:
+        tool = _as_mapping(tool_value)
+        function = _as_mapping(tool.get("function", {}))
+        tool_specs.append(
+            {
+                "description": function.get("description") or "",
+                "name": function["name"],
+                "parameters": function.get("parameters") or {},
+                "type": tool.get("type", "function"),
+            }
+        )
+    return _canonical_json(tool_specs)
+
+
+def _tool_call_name_and_args(tool_call_value: Any) -> tuple[str, Mapping[str, Any]]:
+    tool_call = _as_mapping(tool_call_value)
+    function = _as_mapping(tool_call.get("function", {}))
+    name = function.get("name")
+    if not isinstance(name, str):
+        raise TypeError("tool call function name must be a string")
+
+    raw_args = function.get("arguments") or {}
+    if isinstance(raw_args, str):
+        args = json.loads(raw_args) if raw_args.strip() else {}
+    else:
+        args = raw_args
+    if not isinstance(args, Mapping):
+        raise TypeError("tool call function arguments must decode to an object")
+    return name, args
+
+
+def _tool_call_json(name: str, args: Mapping[str, Any]) -> str:
+    name_json = json.dumps(name, ensure_ascii=False, allow_nan=False)
+    return f'{{"name":{name_json},"args":{_canonical_json(args)}}}'

vllm/renderers/registry.py

@@ -26,6 +26,7 @@ _VLLM_RENDERERS = {
     "kimi_audio": ("hf", "HfRenderer"),
     "mistral": ("mistral", "MistralRenderer"),
     "terratorch": ("terratorch", "TerratorchRenderer"),
+    "inkling": ("inkling", "InklingRenderer"),
 }
 
 

vllm/tokenizers/registry.py

@@ -44,6 +44,10 @@ _VLLM_TOKENIZERS = {
     "hf": ("hf", "CachedHfTokenizer"),
     "kimi_audio": ("kimi_audio", "KimiAudioTokenizer"),
     "mistral": ("mistral", "MistralTokenizer"),
+    # Inkling uses the plain HF tokenizer for token operations; the "inkling"
+    # mode exists to select the InklingRenderer, which renders chat to
+    # token ids natively (Inkling has no Jinja chat template).
+    "inkling": ("hf", "CachedHfTokenizer"),
 }
 
 

vllm/tool_parsers/__init__.py

@@ -174,6 +174,10 @@ _TOOL_PARSERS_TO_REGISTER = {
         "step3p5_tool_parser",
         "Step3p5ToolParser",
     ),
+    "inkling": (
+        "inkling_tool_parser",
+        "InklingEngineToolParser",
+    ),
     "xlam": (
         "xlam_tool_parser",
         "xLAMToolParser",

vllm/tool_parsers/inkling_tool_parser.py

@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm.parser.engine.registered_adapters import InklingParserToolAdapter
+
+
+class InklingEngineToolParser(InklingParserToolAdapter):  # type: ignore[valid-type, misc]
+    # No Inkling structural-tag grammar is wired up yet; fall back to auto
+    # parsing for named/required tool choice.
+    structural_tag_model = None
+    supports_required_and_named = False

vllm/transformers_utils/config.py

@@ -127,6 +127,8 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
     laguna="LagunaConfig",
     lfm2_moe="Lfm2MoeConfig",
     **{"unlimited-ocr": "UnlimitedOCRConfig"},
+    inkling_mm_model="InklingMMConfig",
+    inkling_model="InklingModelConfig",
 )
 
 _SPECULATIVE_DECODING_CONFIGS: set[str] = {"eagle", "speculators", "medusa"}

vllm/transformers_utils/configs/__init__.py

@@ -94,6 +94,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
     "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
     "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
     "Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
+    "InklingModelConfig": "vllm.models.inkling.configs",
+    "InklingAudioConfig": "vllm.models.inkling.configs",
+    "InklingVisionConfig": "vllm.models.inkling.configs",
+    "InklingMMConfig": "vllm.models.inkling.configs",
     # Special case: DeepseekV3Config is from HuggingFace Transformers
     "DeepseekV3Config": "transformers",
 }
@@ -174,6 +178,10 @@ __all__ = [
     "Qwen3_5TextConfig",
     "Qwen3_5MoeConfig",
     "Qwen3_5MoeTextConfig",
+    "InklingModelConfig",
+    "InklingAudioConfig",
+    "InklingVisionConfig",
+    "InklingMMConfig",
 ]
 
 

vllm/transformers_utils/processors/__init__.py

@@ -44,6 +44,9 @@ __all__ = [
     "Ovis2_5Processor",
     "Qwen3ASRProcessor",
     "Step3VLProcessor",
+    "InklingProcessor",
+    "InklingImageProcessor",
+    "InklingAudioFeatureExtractor",
 ]
 
 _CLASS_TO_MODULE: dict[str, str] = {
@@ -80,6 +83,9 @@ _CLASS_TO_MODULE: dict[str, str] = {
     "Ovis2_5Processor": "vllm.transformers_utils.processors.ovis2_5",
     "Qwen3ASRProcessor": "vllm.transformers_utils.processors.qwen3_asr",
     "Step3VLProcessor": "vllm.transformers_utils.processors.step3_vl",
+    "InklingProcessor": "vllm.transformers_utils.processors.inkling",
+    "InklingImageProcessor": "vllm.transformers_utils.processors.inkling",
+    "InklingAudioFeatureExtractor": "vllm.transformers_utils.processors.inkling",
 }
 
 

vllm/transformers_utils/processors/inkling.py

@@ -0,0 +1,504 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Vendored HuggingFace-convention processors for the Inkling Titan model.
+
+Implements the image processor (numba patchifier), the audio feature extractor
+(STFT/dMel path), the composite processor, and the MM token-id constants.
+
+Besides raw bytes / file paths, the extractors also accept the dummy inputs
+vLLM generates during profiling (PIL images / numpy audio arrays).
+"""
+
+from __future__ import annotations
+
+import io
+import math
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from numba import njit
+from transformers.feature_extraction_utils import (
+    BatchFeature,
+    FeatureExtractionMixin,
+)
+from transformers.image_processing_utils import BaseImageProcessor
+from transformers.image_utils import ImageInput
+
+# ---------------------------------------------------------------------------
+# MM token-id constants
+# ---------------------------------------------------------------------------
+
+# Block-start marker token ids. These are real tokens the model was trained on
+# (``<|content_image|>`` / ``<|content_audio_input|>``) that mark the start of an
+# image/audio embedding block; they are kept verbatim in ``input_ids``.
+IMAGE_MARKER_ID = 200005  # <|content_image|>
+AUDIO_MARKER_ID = 200020  # <|content_audio_input|>
+
+# Per-patch / per-frame placeholder ids marking where tower embeddings are
+# scattered in. These are unused slots in the padded vocabulary and the
+# corresponding positions are always overwritten by tower embeddings.
+IMAGE_TOKEN_ID = 200054  # <|unused_200054|>
+AUDIO_TOKEN_ID = 200053  # <|unused_200053|>
+
+# ---------------------------------------------------------------------------
+# Image processing
+# ---------------------------------------------------------------------------
+
+IMAGE_MEAN = np.array([0.48145466, 0.4578275, 0.40821073], dtype=np.float32)
+IMAGE_STD = np.array([0.26862954, 0.2613026, 0.2757771], dtype=np.float32)
+PAD_RAW_VALUE = np.float32(-1.0 / 255.0)
+PAD_NORM = (np.full((3,), PAD_RAW_VALUE, dtype=np.float32) - IMAGE_MEAN) / IMAGE_STD
+
+
+def _validate_image_rescale(
+    rescale_image_frac: float | None,
+    rescale_image_max_upscaled_long_edge: int | None,
+) -> None:
+    if rescale_image_frac is not None and (
+        not math.isfinite(rescale_image_frac) or rescale_image_frac <= 0
+    ):
+        raise ValueError(
+            "rescale_image_frac must be positive and finite or None, "
+            f"got {rescale_image_frac}"
+        )
+    if rescale_image_max_upscaled_long_edge is None:
+        return
+    if rescale_image_max_upscaled_long_edge <= 0:
+        raise ValueError(
+            "rescale_image_max_upscaled_long_edge must be positive or None, "
+            f"got {rescale_image_max_upscaled_long_edge}"
+        )
+    if rescale_image_frac is None or rescale_image_frac <= 1.0:
+        raise ValueError(
+            "rescale_image_max_upscaled_long_edge requires rescale_image_frac > 1, "
+            f"got {rescale_image_frac}"
+        )
+
+
+def _scaled_image_dimensions(
+    width: int,
+    height: int,
+    rescale_image_frac: float | None,
+    rescale_image_max_upscaled_long_edge: int | None,
+) -> tuple[int, int]:
+    """Return the long-edge-scaled ``(width, height)``."""
+    if rescale_image_frac is None:
+        return width, height
+
+    long_edge = max(width, height)
+    if long_edge == 0:
+        return width, height
+
+    target_long_edge = float(long_edge) * rescale_image_frac
+    if rescale_image_max_upscaled_long_edge is not None:
+        effective_cap = max(rescale_image_max_upscaled_long_edge, long_edge)
+        target_long_edge = min(target_long_edge, float(effective_cap))
+
+    ratio = target_long_edge / float(long_edge)
+    if ratio == 1.0:
+        return width, height
+
+    def scale(value: int) -> int:
+        return max(1, math.floor(float(value) * ratio + 0.5))
+
+    return scale(width), scale(height)
+
+
+def _load_image_bytes(image) -> bytes:
+    """Encode a PIL image as raw PNG bytes for preprocessing.
+
+    The HF processor is always handed ``PIL.Image`` instances by vLLM, so no
+    other input types need to be supported here.
+    """
+    if image.mode != "RGB":
+        image = image.convert("RGB")
+    buf = io.BytesIO()
+    image.save(buf, format="PNG")
+    return buf.getvalue()
+
+
+@njit(cache=True)
+def _fill_patches_numba(
+    arr: np.ndarray,
+    patch_size: int,
+    patches: np.ndarray,
+    mean: np.ndarray,
+    std: np.ndarray,
+    pad_norm: np.ndarray,
+) -> None:
+    h = arr.shape[0]
+    w = arr.shape[1]
+    nph = (h + patch_size - 1) // patch_size
+    npw = w // patch_size + 1
+    inv255 = np.float32(1.0 / 255.0)
+
+    for k in range(nph * npw):
+        i = k // npw
+        j = k - i * npw
+        y_base = i * patch_size
+        x_base = j * patch_size
+
+        for y in range(patch_size):
+            iy = y_base + y
+            for x in range(patch_size):
+                ix = x_base + x
+                if iy < h and ix < w:
+                    for c in range(3):
+                        raw = np.float32(arr[iy, ix, c]) * inv255
+                        patches[k, y, x, c] = (raw - mean[c]) / std[c]
+                else:
+                    for c in range(3):
+                        patches[k, y, x, c] = pad_norm[c]
+
+
+def _encode_image_bytes(
+    image_bytes: bytes,
+    *,
+    patch_size: int,
+    rescale_image_frac: float | None,
+    rescale_image_max_upscaled_long_edge: int | None,
+) -> torch.Tensor:
+    if patch_size <= 0:
+        raise ValueError("patch_size must be greater than zero")
+    _validate_image_rescale(
+        rescale_image_frac,
+        rescale_image_max_upscaled_long_edge,
+    )
+
+    from PIL import Image
+
+    image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
+    scaled_size = _scaled_image_dimensions(
+        image.width,
+        image.height,
+        rescale_image_frac=rescale_image_frac,
+        rescale_image_max_upscaled_long_edge=rescale_image_max_upscaled_long_edge,
+    )
+    if scaled_size != image.size:
+        image = image.resize(scaled_size, resample=Image.Resampling.LANCZOS)
+    arr = np.array(image, dtype=np.uint8, copy=True)
+    height, width, _ = arr.shape
+
+    nph = (height + patch_size - 1) // patch_size
+    npw = width // patch_size + 1
+    num_patches = nph * npw
+
+    patches = np.empty((num_patches, patch_size, patch_size, 3), dtype=np.float32)
+    _fill_patches_numba(arr, patch_size, patches, IMAGE_MEAN, IMAGE_STD, PAD_NORM)
+
+    return (
+        torch.from_numpy(patches)
+        .to(torch.bfloat16)
+        .view(num_patches, 1, patch_size, patch_size, 3)
+        .expand(num_patches, 2, patch_size, patch_size, 3)
+    )
+
+
+class InklingImageProcessor(BaseImageProcessor):
+    r"""Turn raw images into ``vision_patches_bthwc`` for Inkling hMLP.
+
+    ``rescale_image_frac`` scales the long edge while preserving aspect ratio.
+    ``rescale_image_max_upscaled_long_edge`` optionally caps only upscaling and
+    therefore requires a scale factor greater than one. The defaults, ``2.0`` and
+    ``2048``, grow images toward a 2048-pixel long edge by at most 2x, while leaving
+    images already at or above 2048 unchanged.
+    """
+
+    model_input_names = ["vision_patches_bthwc"]
+
+    def __init__(
+        self,
+        patch_size: int = 40,
+        rescale_image_frac: float | None = 2.0,
+        rescale_image_max_upscaled_long_edge: int | None = 2048,
+        **kwargs,
+    ):
+        if patch_size <= 0:
+            raise ValueError("patch_size must be greater than zero")
+        _validate_image_rescale(
+            rescale_image_frac,
+            rescale_image_max_upscaled_long_edge,
+        )
+        super().__init__(**kwargs)
+        self.patch_size = patch_size
+        self.rescale_image_frac = rescale_image_frac
+        self.rescale_image_max_upscaled_long_edge = rescale_image_max_upscaled_long_edge
+
+    def _encode_one(self, image) -> torch.Tensor:
+        return _encode_image_bytes(
+            _load_image_bytes(image),
+            patch_size=self.patch_size,
+            rescale_image_frac=self.rescale_image_frac,
+            rescale_image_max_upscaled_long_edge=self.rescale_image_max_upscaled_long_edge,
+        )
+
+    def preprocess(
+        self,
+        images: ImageInput | list,
+        return_tensors: str | None = "pt",
+        **kwargs,
+    ) -> BatchFeature:
+        del return_tensors, kwargs
+        if not isinstance(images, (list, tuple)):
+            images = [images]
+
+        per_image_patches: list[torch.Tensor] = []
+        num_patches: list[int] = []
+        num_tokens: list[int] = []
+        for img in images:
+            vp = self._encode_one(img)
+            n_patches = int(vp.shape[0])
+            per_image_patches.append(vp)
+            num_patches.append(n_patches)
+            num_tokens.append(n_patches)
+
+        if len(per_image_patches) == 1:
+            vision_patches_bthwc = per_image_patches[0]
+        elif per_image_patches:
+            vision_patches_bthwc = torch.cat(per_image_patches, dim=0)
+        else:
+            vision_patches_bthwc = torch.empty(0)
+
+        data = {
+            "vision_patches_bthwc": vision_patches_bthwc,
+            "num_patches": num_patches,
+            "num_tokens": num_tokens,
+        }
+        return BatchFeature(data=data, tensor_type=None)
+
+
+# ---------------------------------------------------------------------------
+# Audio feature extraction
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class InklingAudioEncoderParams:
+    """Audio preprocessing parameters used to convert raw audio into dMel bins."""
+
+    sample_rate: int = 16_000
+    window_size_multiplier: float = 2.0
+    n_fft: int | None = None
+    n_mels: int = 80
+    num_dmel_bins: int = 16
+    dmel_min_value: float = -7.0
+    dmel_max_value: float = 2.0
+    audio_token_duration_s: float = 0.05
+
+
+def _to_exact_int(value: float, name: str, tolerance: float = 1e-6) -> int:
+    rounded = round(value)
+    if abs(value - rounded) > tolerance:
+        raise ValueError(f"{name} must resolve to an integer sample count, got {value}")
+    return int(rounded)
+
+
+def _hz_to_mel(frequencies: np.ndarray) -> np.ndarray:
+    """Slaney mel scale, matching the librosa/torchaudio convention."""
+    frequencies = np.asarray(frequencies, dtype=np.float64)
+    f_sp = 200.0 / 3.0
+    min_log_hz = 1000.0
+    min_log_mel = min_log_hz / f_sp
+    logstep = np.log(6.4) / 27.0
+    linear = frequencies / f_sp
+    log = (
+        min_log_mel + np.log(np.maximum(frequencies, min_log_hz) / min_log_hz) / logstep
+    )
+    return np.where(frequencies >= min_log_hz, log, linear)
+
+
+def _mel_to_hz(mels: np.ndarray) -> np.ndarray:
+    mels = np.asarray(mels, dtype=np.float64)
+    f_sp = 200.0 / 3.0
+    min_log_hz = 1000.0
+    min_log_mel = min_log_hz / f_sp
+    logstep = np.log(6.4) / 27.0
+    linear = mels * f_sp
+    log = min_log_hz * np.exp(logstep * (mels - min_log_mel))
+    return np.where(mels >= min_log_mel, log, linear)
+
+
+_MEL_BASIS_CACHE: dict[tuple[int, int, int], torch.Tensor] = {}
+
+
+def _mel_basis(sample_rate: int, n_fft: int, n_mels: int) -> torch.Tensor:
+    key = (sample_rate, n_fft, n_mels)
+    cached = _MEL_BASIS_CACHE.get(key)
+    if cached is not None:
+        return cached
+
+    fft_bins = n_fft // 2 + 1
+    fft_freqs = np.arange(fft_bins, dtype=np.float64) * sample_rate / n_fft
+    mel_edges = _mel_to_hz(
+        np.linspace(
+            _hz_to_mel(np.array([0.0]))[0],
+            _hz_to_mel(np.array([sample_rate / 2.0]))[0],
+            n_mels + 2,
+            dtype=np.float64,
+        )
+    )
+    mel_widths = np.diff(mel_edges)
+    lower = (fft_freqs[None, :] - mel_edges[:-2, None]) / mel_widths[:-1, None]
+    upper = (mel_edges[2:, None] - fft_freqs[None, :]) / mel_widths[1:, None]
+    weights = np.maximum(0.0, np.minimum(lower, upper))
+
+    # Slaney area normalization.
+    weights *= (2.0 / (mel_edges[2:] - mel_edges[:-2]))[:, None]
+    basis = torch.from_numpy(weights.astype(np.float32, copy=False)).contiguous()
+    _MEL_BASIS_CACHE[key] = basis
+    return basis
+
+
+def _dmel_bins(audio: torch.Tensor, params: InklingAudioEncoderParams) -> torch.Tensor:
+    hop_length = _to_exact_int(
+        params.audio_token_duration_s * params.sample_rate,
+        "audio_token_duration_s * sample_rate",
+    )
+    window_size = _to_exact_int(
+        params.audio_token_duration_s
+        * params.window_size_multiplier
+        * params.sample_rate,
+        "audio_token_duration_s * window_size_multiplier * sample_rate",
+    )
+    n_fft = params.n_fft or window_size
+    if hop_length <= 0 or window_size <= 0 or n_fft <= 0:
+        raise ValueError("audio hop length, window size, and n_fft must be positive")
+    if audio.numel() == 0:
+        return torch.empty((0, params.n_mels), dtype=torch.int32)
+
+    right_pad = math.ceil(audio.numel() / hop_length) * hop_length - audio.numel()
+    left_pad = max(n_fft - hop_length, 0)
+    audio = F.pad(audio, (left_pad, right_pad))
+
+    window = torch.hann_window(window_size, periodic=True, dtype=torch.float32)
+    spec = torch.stft(
+        audio.unsqueeze(0),
+        n_fft=n_fft,
+        hop_length=hop_length,
+        win_length=window_size,
+        window=window,
+        center=False,
+        normalized=False,
+        onesided=True,
+        return_complex=True,
+    )
+    spec_ri = torch.view_as_real(spec)
+    magnitude = (
+        (spec_ri[..., 0].square() + spec_ri[..., 1].square())
+        .clamp_min(1e-10)
+        .sqrt()
+        .squeeze(0)
+    )
+
+    mel = (
+        _mel_basis(params.sample_rate, n_fft, params.n_mels)
+        .matmul(magnitude)
+        .clamp_min(1e-10)
+        .log10()
+    )
+    mel = mel.to(torch.float64).clamp(
+        min=params.dmel_min_value, max=params.dmel_max_value
+    )
+    bin_centers = torch.linspace(
+        params.dmel_min_value,
+        params.dmel_max_value,
+        params.num_dmel_bins,
+        dtype=torch.float64,
+    )
+    dmel_bins = (mel.unsqueeze(-1) - bin_centers).abs().argmin(dim=-1)
+    return dmel_bins.to(torch.int32).T.contiguous()
+
+
+class InklingAudioFeatureExtractor(FeatureExtractionMixin):
+    """Convert raw audio into Inkling dMel bins in the HF feature-extractor API."""
+
+    model_input_names = ["dmel_bins"]
+
+    def __init__(self, params: dict | None = None, **kwargs):
+        super().__init__(**kwargs)
+        merged = InklingAudioEncoderParams()
+        if params:
+            for k, v in params.items():
+                if hasattr(merged, k):
+                    setattr(merged, k, v)
+        # also accept flat kwargs (HF config style)
+        for k in list(kwargs.keys()):
+            if hasattr(merged, k):
+                setattr(merged, k, kwargs[k])
+        self.params = merged
+
+    def _decode_one(self, audio) -> torch.Tensor:
+        # vLLM hands the feature extractor numpy arrays (the dummy-input builder
+        # during profiling, and MultiModalDataParser after resampling to the
+        # target sample rate), so no other input types need to be supported.
+        return torch.from_numpy(
+            np.ascontiguousarray(audio.astype(np.float32, copy=False))
+        ).flatten()
+
+    def _encode_one(self, audio) -> torch.Tensor:
+        return _dmel_bins(self._decode_one(audio), self.params)
+
+    def __call__(
+        self,
+        audios: Sequence | None,
+        return_tensors: str | None = None,
+        **kwargs,
+    ) -> BatchFeature:
+        del return_tensors, kwargs
+        if audios is None:
+            audios = []
+        if not isinstance(audios, (list, tuple)):
+            audios = [audios]
+
+        dmel_bins = [self._encode_one(a) for a in audios]
+        data = {
+            # per-clip feature: dmel bins as float32 [T, n_mels]
+            "dmel_bins": [bins.to(torch.float32) for bins in dmel_bins],
+            "num_audio_tokens": [int(bins.shape[0]) for bins in dmel_bins],
+        }
+        # return_tensors intentionally ignored: per-clip features have ragged T.
+        return BatchFeature(data=data, tensor_type=None)
+
+
+# ---------------------------------------------------------------------------
+# Composite processor
+# ---------------------------------------------------------------------------
+
+
+class InklingProcessor:
+    """Bundle Inkling image + audio preprocessing with the MM token ids."""
+
+    def __init__(
+        self,
+        image_processor: InklingImageProcessor | None = None,
+        audio_feature_extractor: InklingAudioFeatureExtractor | None = None,
+        tokenizer=None,
+    ):
+        self.image_processor = image_processor or InklingImageProcessor()
+        self.audio_feature_extractor = (
+            audio_feature_extractor or InklingAudioFeatureExtractor()
+        )
+        self.tokenizer = tokenizer
+
+    def process_images(self, images: list):
+        """Raw images -> BatchFeature(vision_patches_bthwc, num_patches, num_tokens)."""
+        return self.image_processor.preprocess(images, return_tensors="pt")
+
+    def process_audios(self, audios: list):
+        """Raw audios -> BatchFeature(dmel_bins, num_audio_tokens)."""
+        return self.audio_feature_extractor(audios)
+
+
+__all__ = [
+    "InklingImageProcessor",
+    "InklingAudioFeatureExtractor",
+    "InklingAudioEncoderParams",
+    "InklingProcessor",
+    "IMAGE_MARKER_ID",
+    "AUDIO_MARKER_ID",
+    "IMAGE_TOKEN_ID",
+    "AUDIO_TOKEN_ID",
+]