From ee1d25f199ee76079e761b34e865e13b40ffdbe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eldar=20Kurti=C4=87?= <8884008+eldarkurtic@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:55:41 +0100 Subject: [PATCH 01/74] [Llama4,Quantization] Simplify and generalize logic for Q/K permutations in quantized self-attn layers (#34471) Signed-off-by: Your Name Co-authored-by: Your Name Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- vllm/model_executor/models/llama4.py | 97 +++++++++------------------- 1 file changed, 29 insertions(+), 68 deletions(-) diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index 4050bf0453e..b84b4e2ae51 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -44,9 +44,6 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig -from vllm.model_executor.layers.quantization.compressed_tensors import ( - compressed_tensors as ct, -) from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, @@ -831,74 +828,38 @@ class Llama4ForCausalLM(LlamaForCausalLM, MixtureOfExperts): name: str, loaded_weight: torch.Tensor, ) -> tuple[str, torch.Tensor]: - # Helper function to permute the weight's channels - def permute( - w: torch.Tensor, - n_heads: int, - is_nvfp4_weight_scale: bool, - is_ct_int8_or_fp8_weight_scale: bool, - ): - # Calculate the expected shape of the weight. - # Do not rely on w's shape, as it may be in another layout. - attn_in = self.config.head_dim * n_heads - attn_out = ( - self.config.hidden_size - if not is_ct_int8_or_fp8_weight_scale - else w.shape[-1] - ) - - # If the weight is FP4 packed as uint8, we need to divide attn_out - # by 2. - if w.dtype == torch.uint8 and w.shape[1] * 2 == attn_out: - attn_out = attn_out // 2 - - # If the weight is a weight scale, we need to divide attn_out by - # block size, which is currently 16. - elif ( - w.dtype == torch.float8_e4m3fn - and is_nvfp4_weight_scale - and w.shape[1] * 16 == attn_out - ): - attn_out = attn_out // 16 - - return ( - w.view(n_heads, attn_in // n_heads // 2, 2, attn_out) - .transpose(1, 2) - .reshape(attn_in, attn_out) - ) - modules = name.split(".") - - # Permute Q/K weights and weight block scales for rotary embedding - is_weight = modules[-1] == "weight" - is_nvfp4_weight_scale = ( - modules[-1] == "weight_scale" and loaded_weight.dtype == torch.float8_e4m3fn + # Permute Q/K weights and corresponding scales for rotary embedding. + # This pathway is validated against modelopt and compressed-tensors ckpts, + # and for per-tensor, per-group (e.g. GPTQ), and per-channel quant schemes. + # Note: permutations are not feasible only for per-block (e.g. DeepSeek 128x128) + # For per-block quantization, consider not quantizing q/k_proj. + is_weight = modules[-1] in ("weight", "weight_packed") + is_weight_scale = ( + modules[-1] == "weight_scale" + and loaded_weight.numel() > 1 # no need to permute per-tensor scales ) - is_ct_int8_or_fp8_weight_scale = False - if modules[-1] == "weight_scale" and isinstance( - self.model.quant_config, ct.CompressedTensorsConfig - ): - from compressed_tensors import CompressionFormat + is_k_proj = "wk" in modules or "k_proj" in modules + is_q_proj = "wq" in modules or "q_proj" in modules - is_ct_int8_or_fp8_weight_scale = self.model.quant_config.quant_format in [ - CompressionFormat.int_quantized.value, - CompressionFormat.float_quantized.value, - ] and loaded_weight.dtype in [torch.float16, torch.bfloat16, torch.float32] + if (is_weight or is_weight_scale) and (is_k_proj or is_q_proj): + original_ndim = loaded_weight.ndim + if original_ndim == 1: + loaded_weight = loaded_weight.unsqueeze(-1) - if is_weight or is_nvfp4_weight_scale or is_ct_int8_or_fp8_weight_scale: - if "wk" in modules or "k_proj" in modules: - loaded_weight = permute( - loaded_weight, - self.config.num_key_value_heads, - is_nvfp4_weight_scale, - is_ct_int8_or_fp8_weight_scale, - ) - elif "wq" in modules or "q_proj" in modules: - loaded_weight = permute( - loaded_weight, - self.config.num_attention_heads, - is_nvfp4_weight_scale, - is_ct_int8_or_fp8_weight_scale, - ) + f_out, f_in = loaded_weight.shape + n_heads = ( + self.config.num_key_value_heads + if is_k_proj + else self.config.num_attention_heads + ) + loaded_weight = ( + loaded_weight.view(n_heads, f_out // n_heads // 2, 2, f_in) + .transpose(1, 2) + .reshape(f_out, f_in) + ) + + if original_ndim == 1: + loaded_weight = loaded_weight.squeeze(-1) return name, loaded_weight From 4685a630a293cd7c928092efd0f8c2606a770877 Mon Sep 17 00:00:00 2001 From: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Date: Thu, 19 Feb 2026 10:56:14 -0500 Subject: [PATCH 02/74] [Model Bash][DeepSeekR1] Remove Shared Expert Clone (#34344) Signed-off-by: Robert Shaw Co-authored-by: Robert Shaw --- .../fused_moe/runner/default_moe_runner.py | 22 +++++++++---------- vllm/model_executor/models/minicpm.py | 2 +- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py index c0d23964c47..e92f068f08c 100644 --- a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py @@ -240,24 +240,22 @@ class DefaultMoERunner(MoERunner): ) ) - hidden_states_clone: torch.Tensor | None = None + shared_experts_input: torch.Tensor | None = None if use_shared_experts_stream: assert self.shared_experts_stream is not None + assert self.moe_config.disable_inplace shared_experts_input = ( shared_input if shared_input is not None else hidden_states ) - # Clone BEFORE switching streams to avoid race condition - # where routed_expert kernel may mutate hidden_states. - hidden_states_clone = shared_experts_input.clone() - - # Record that the clone will be used by shared_experts_stream - # to avoid gc issue from deallocation of hidden_states_clone - # For more details: https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501 + # Record that the shared_experts_input will be used in the + # shared_experts_stream to to avoid gc issue from + # deallocation. For more details: + # https://docs.pytorch.org/docs/stable/generated/torch.Tensor.record_stream.html # noqa: E501 # NOTE: We don't need shared_output.record_stream(current_stream()) # because we synch the streams before using shared_output. - hidden_states_clone.record_stream(self.shared_experts_stream) + shared_experts_input.record_stream(self.shared_experts_stream) # Mark sync start point for the separate shared experts # stream here since we want to run in parallel with the @@ -265,7 +263,7 @@ class DefaultMoERunner(MoERunner): assert self.shared_experts_stream is not None self.shared_experts_stream.wait_stream(current_stream()) - return use_shared_experts_stream, hidden_states_clone + return use_shared_experts_stream, shared_experts_input def ensure_dp_chunking_init(self): if not self.use_dp_chunking or self.batched_hidden_states is not None: @@ -584,7 +582,7 @@ class DefaultMoERunner(MoERunner): use_chunked_impl = self.use_dp_chunking - use_shared_experts_stream, hidden_states_clone = ( + use_shared_experts_stream, shared_experts_input = ( self._maybe_setup_shared_experts_stream( hidden_states, shared_input, @@ -726,7 +724,7 @@ class DefaultMoERunner(MoERunner): with torch.cuda.stream(self.shared_experts_stream): # Note that hidden_states clone() is necessary here to avoid # conflict with the main stream - shared_output = self.shared_experts(hidden_states_clone) + shared_output = self.shared_experts(shared_experts_input) current_stream().wait_stream(self.shared_experts_stream) final_hidden_states = ( diff --git a/vllm/model_executor/models/minicpm.py b/vllm/model_executor/models/minicpm.py index 4217d119a18..4492b57630f 100644 --- a/vllm/model_executor/models/minicpm.py +++ b/vllm/model_executor/models/minicpm.py @@ -175,7 +175,7 @@ class MiniCPMMoE(nn.Module): ) final_hidden_states = fused_experts( - hidden_states, self.ws, self.w2s, topk_weights, topk_ids, inplace=True + hidden_states, self.ws, self.w2s, topk_weights, topk_ids, inplace=False ) if self.tp_size > 1: From 3eff45d793daa976a21d0df5954cf6cc6723335f Mon Sep 17 00:00:00 2001 From: roikoren755 <26850796+roikoren755@users.noreply.github.com> Date: Thu, 19 Feb 2026 19:47:05 +0200 Subject: [PATCH 03/74] Revert "[NemotronH] Do not force router to run in fp32 (#34582)" (#34808) Signed-off-by: Roi Koren Co-authored-by: Michael Goin --- vllm/model_executor/models/nemotron_h.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/models/nemotron_h.py b/vllm/model_executor/models/nemotron_h.py index d51becac7e2..06141013c46 100644 --- a/vllm/model_executor/models/nemotron_h.py +++ b/vllm/model_executor/models/nemotron_h.py @@ -148,10 +148,12 @@ class NemotronHMoE(nn.Module): self.is_sequence_parallel = parallel_config.use_sequence_parallel_moe + router_logits_dtype = torch.float32 self.gate = ReplicatedLinear( config.hidden_size, config.n_routed_experts, bias=False, + params_dtype=router_logits_dtype, quant_config=None, prefix=f"{prefix}.gate", ) @@ -230,6 +232,7 @@ class NemotronHMoE(nn.Module): enable_eplb=self.enable_eplb, num_redundant_experts=self.n_redundant_experts, is_sequence_parallel=self.is_sequence_parallel, + router_logits_dtype=router_logits_dtype, routed_input_transform=self.fc1_latent_proj, ) @@ -241,7 +244,7 @@ class NemotronHMoE(nn.Module): hidden_states = sequence_parallel_chunk(hidden_states) # router_logits: (num_tokens, n_experts) - router_logits, _ = self.gate(hidden_states) + router_logits, _ = self.gate(hidden_states.to(dtype=torch.float32)) # SharedFusedMoE handles: # - shared experts (with original hidden_states) From c683d11c94655655cd7bf95a27aef7e245325102 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Thu, 19 Feb 2026 13:23:49 -0500 Subject: [PATCH 04/74] [Refactor] Deprecate `head_first` for `chunk_gated_delta_rule` (#34263) Signed-off-by: yewentao256 --- vllm/model_executor/layers/fla/ops/chunk.py | 37 ++++--------------- vllm/model_executor/models/llava_onevision.py | 1 - vllm/model_executor/models/qwen3_next.py | 6 --- 3 files changed, 8 insertions(+), 36 deletions(-) diff --git a/vllm/model_executor/layers/fla/ops/chunk.py b/vllm/model_executor/layers/fla/ops/chunk.py index 958464b6941..40f8c3c2a16 100644 --- a/vllm/model_executor/layers/fla/ops/chunk.py +++ b/vllm/model_executor/layers/fla/ops/chunk.py @@ -10,7 +10,6 @@ import warnings import torch -from einops import rearrange from .chunk_delta_h import chunk_gated_delta_rule_fwd_h from .chunk_o import chunk_fwd_o @@ -119,21 +118,20 @@ def chunk_gated_delta_rule( initial_state: torch.Tensor = None, output_final_state: bool = False, cu_seqlens: torch.LongTensor | None = None, - head_first: bool = False, use_qk_l2norm_in_kernel: bool = False, ): r""" Args: q (torch.Tensor): - queries of shape `[B, T, H, K]` if `head_first=False` else `[B, H, T, K]`. + Queries of shape `[B, T, H, K]`. k (torch.Tensor): - keys of shape `[B, T, H, K]` if `head_first=False` else `[B, H, T, K]`. + Keys of shape `[B, T, H, K]`. v (torch.Tensor): - values of shape `[B, T, H, V]` if `head_first=False` else `[B, H, T, V]`. + Values of shape `[B, T, H, V]`. g (torch.Tensor): - (forget) gating tensor (in log space!) of shape `[B, T, H]` if `head_first=False` else `[B, H, T]`. + (forget) Gating tensor (in log space!) of shape `[B, T, H]`. beta (torch.Tensor): - betas of shape `[B, T, H]` if `head_first=False` else `[B, H, T]`. + Betas of shape `[B, T, H]`. scale (Optional[int]): Scale factor for the RetNet attention scores. If not provided, it will default to `1 / sqrt(K)`. Default: `None`. @@ -146,13 +144,9 @@ def chunk_gated_delta_rule( cu_seqlens (torch.LongTensor): Cumulative sequence lengths of shape `[N+1]` used for variable-length training, consistent with the FlashAttention API. - head_first (Optional[bool]): - Whether the inputs are in the head-first format, which is not supported for variable-length inputs. - Default: `False`. - Returns: o (torch.Tensor): - Outputs of shape `[B, T, H, V]` if `head_first=False` else `[B, H, T, V]`. + Outputs of shape `[B, T, H, V]`. final_state (torch.Tensor): Final state of shape `[N, H, V, K]` if `output_final_state=True` else `None`. @@ -189,24 +183,11 @@ def chunk_gated_delta_rule( assert q.dtype != torch.float32, ( "ChunkGatedDeltaRuleFunction does not support float32. Please use bfloat16." ) - assert len(beta.shape) == 3, ( - "beta must be of shape [B, T, H] if head_first=False, or [B, H, T] otherwise." - ) - - if head_first: - raise DeprecationWarning( - "head_first is deprecated and will be removed in a future version. " - "Please use head_first=False for now instead.", - stacklevel=2, - ) - q, k, v, beta, g = map( - lambda x: rearrange(x, "b h t ... -> b t h ..."), (q, k, v, beta, g) - ) - if not head_first and q.shape[1] < q.shape[2]: + assert len(beta.shape) == 3, "beta must be of shape [B, T, H]." + if q.shape[1] < q.shape[2]: warnings.warn( f"Input tensor shape suggests potential format mismatch: seq_len ({q.shape[1]}) < num_heads ({q.shape[2]}). " "This may indicate the inputs were passed in head-first format [B, H, T, ...] " - "when head_first=False was specified. " "Please verify your input tensor format matches the expected shape [B, T, H, ...].", stacklevel=2, ) @@ -235,6 +216,4 @@ def chunk_gated_delta_rule( cu_seqlens, use_qk_l2norm_in_kernel, ) - if head_first: - o = rearrange(o, "b t h ... -> b h t ...") return o, final_state diff --git a/vllm/model_executor/models/llava_onevision.py b/vllm/model_executor/models/llava_onevision.py index 39633eaf979..290ace8bff6 100644 --- a/vllm/model_executor/models/llava_onevision.py +++ b/vllm/model_executor/models/llava_onevision.py @@ -867,7 +867,6 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs) if not mm_input_by_modality: return [] - return None # The result multimodal_embeddings is tuple of tensors, with each # tensor corresponding to a multimodal data item (image or video). diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 6f8aea79d1d..16116c67ab1 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -115,7 +115,6 @@ def fi_chunk_gated_delta_rule( initial_state: torch.Tensor, output_final_state: bool, cu_seqlens: torch.LongTensor | None = None, - head_first: bool = False, use_qk_l2norm_in_kernel: bool = True, ): from flashinfer.gdn_prefill import ( @@ -172,7 +171,6 @@ class ChunkGatedDeltaRule(CustomOp): initial_state: torch.Tensor, output_final_state: bool, cu_seqlens: torch.LongTensor | None = None, - head_first: bool = False, use_qk_l2norm_in_kernel: bool = True, ): return fi_chunk_gated_delta_rule( @@ -184,7 +182,6 @@ class ChunkGatedDeltaRule(CustomOp): initial_state=initial_state, output_final_state=output_final_state, cu_seqlens=cu_seqlens, - head_first=head_first, use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, ) @@ -198,7 +195,6 @@ class ChunkGatedDeltaRule(CustomOp): initial_state: torch.Tensor, output_final_state: bool, cu_seqlens: torch.LongTensor | None = None, - head_first: bool = False, use_qk_l2norm_in_kernel: bool = True, ): return fla_chunk_gated_delta_rule( @@ -210,7 +206,6 @@ class ChunkGatedDeltaRule(CustomOp): initial_state=initial_state, output_final_state=output_final_state, cu_seqlens=cu_seqlens, - head_first=head_first, use_qk_l2norm_in_kernel=use_qk_l2norm_in_kernel, ) @@ -790,7 +785,6 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase): initial_state=initial_state, output_final_state=True, cu_seqlens=non_spec_query_start_loc, - head_first=False, use_qk_l2norm_in_kernel=True, ) # Init cache From 304319c4edcc1a50317c22715ad6c0111459025d Mon Sep 17 00:00:00 2001 From: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com> Date: Thu, 19 Feb 2026 15:26:53 -0600 Subject: [PATCH 05/74] Change targets for AMD build in the "CI" pipeline (#34918) Signed-off-by: Alexei V. Ivanov --- .buildkite/hardware_tests/amd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/hardware_tests/amd.yaml b/.buildkite/hardware_tests/amd.yaml index 0fd8d348595..2831bbc9d68 100644 --- a/.buildkite/hardware_tests/amd.yaml +++ b/.buildkite/hardware_tests/amd.yaml @@ -10,7 +10,7 @@ steps: docker build --build-arg max_jobs=16 --build-arg REMOTE_VLLM=1 - --build-arg ARG_PYTORCH_ROCM_ARCH='gfx90a;gfx942' + --build-arg ARG_PYTORCH_ROCM_ARCH='gfx942;gfx950' --build-arg VLLM_BRANCH=$BUILDKITE_COMMIT --tag "rocm/vllm-ci:${BUILDKITE_COMMIT}" -f docker/Dockerfile.rocm From 4fb8beefaa8b2c4bd2cd3b336b01ff006dc98bdc Mon Sep 17 00:00:00 2001 From: Roger Wang Date: Thu, 19 Feb 2026 13:34:55 -0800 Subject: [PATCH 06/74] [Bugfix] Fix cutlass fp8 kernel on hopper for Qwen3.5 (#34914) Signed-off-by: Roger Wang --- .../layers/quantization/utils/flashinfer_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py index 42fae9ee932..3d7d8e68fdc 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_utils.py @@ -455,4 +455,15 @@ def prepare_fp8_moe_layer_for_fi( w2_input_scale=w2_input_scale, ) + # Clamp block scales to avoid NaN from the FlashInfer CUTLASS kernel. + # Some FP8 models have near-zero block scales (~1e-23) for dead/unused + # experts. The CUTLASS kernel doesn't handle these correctly on Hopper + # (SM 9.0), producing NaN instead of near-zero output. Clamping to a + # small minimum prevents this without affecting model accuracy since + # these experts' effective weights are already zero. + if block_quant: + _FI_CUTLASS_MIN_BLOCK_SCALE = 1e-10 + w13_scale.clamp_(min=_FI_CUTLASS_MIN_BLOCK_SCALE) + w2_scale.clamp_(min=_FI_CUTLASS_MIN_BLOCK_SCALE) + return w13, w2, w13_scale From 662205d34eb1bb42228768d7a69a1ac4abf38c89 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Thu, 19 Feb 2026 17:49:07 -0500 Subject: [PATCH 07/74] [Bugfix] Fix Basic Models Test (#34818) Signed-off-by: Matthew Bonanni Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> --- .../processing/test_tensor_schema.py | 5 +- tests/models/utils.py | 9 +- tests/v1/spec_decode/test_eagle.py | 2 +- vllm/config/cache.py | 4 +- vllm/config/vllm.py | 97 ++++---- .../attention/chunked_local_attention.py | 11 +- .../layers/attention/mla_attention.py | 17 +- vllm/platforms/cuda.py | 219 +++++------------- vllm/platforms/interface.py | 7 + vllm/v1/engine/core.py | 9 +- vllm/v1/executor/multiproc_executor.py | 4 + vllm/v1/executor/ray_executor.py | 5 + vllm/v1/executor/uniproc_executor.py | 2 + vllm/v1/worker/gpu_model_runner.py | 5 +- 14 files changed, 175 insertions(+), 221 deletions(-) diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index 8f79936478d..c81a8fe09d3 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -13,6 +13,7 @@ import torch.nn as nn from PIL import Image from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config +from vllm.config.cache import CacheConfig from vllm.config.multimodal import ( AudioDummyOptions, BaseDummyOptions, @@ -131,7 +132,9 @@ def initialize_dummy_model( ): temp_file = tempfile.mkstemp()[1] current_device = torch.get_default_device() - vllm_config = VllmConfig(model_config=model_config) + vllm_config = VllmConfig( + model_config=model_config, cache_config=CacheConfig(block_size=16) + ) with set_current_vllm_config(vllm_config=vllm_config): init_distributed_environment( world_size=1, diff --git a/tests/models/utils.py b/tests/models/utils.py index 4830f18dccf..8c1fb63d67a 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -457,6 +457,9 @@ def dummy_hf_overrides( # Kimi uses `num_expert_group` instead of `n_group`. if n_group is None: n_group = getattr(text_config, "num_expert_group", None) + # InternS1Pro uses `router_n_groups` instead of `n_group`. + if n_group is None: + n_group = getattr(text_config, "router_n_groups", None) num_experts = n_group * 2 if n_group is not None else 2 # we use three layers for Gemma-3n to check @@ -486,12 +489,14 @@ def dummy_hf_overrides( # Only set MoE related config when the model has MoE layers. # Otherwise all models detected as MoE by _get_transformers_backend_cls. if model_arch_config.num_experts > 0: + orig_topk = getattr(text_config, "num_experts_per_tok", 2) + topk = min(orig_topk, 2) update_dict.update( { "num_experts": num_experts, - "num_experts_per_tok": 2, + "num_experts_per_tok": topk, # Kimi uses `num_experts_per_token`. - "num_experts_per_token": 2, + "num_experts_per_token": topk, "num_local_experts": num_experts, # Otherwise there will not be any expert layers "first_k_dense_replace": 0, diff --git a/tests/v1/spec_decode/test_eagle.py b/tests/v1/spec_decode/test_eagle.py index 8b180168dff..65e97b7ad5b 100644 --- a/tests/v1/spec_decode/test_eagle.py +++ b/tests/v1/spec_decode/test_eagle.py @@ -78,7 +78,7 @@ def _create_proposer( device = current_platform.device_type vllm_config = VllmConfig( model_config=model_config, - cache_config=CacheConfig(), + cache_config=CacheConfig(block_size=16), speculative_config=speculative_config, device_config=DeviceConfig(device=device), parallel_config=ParallelConfig(), diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 0823b00a351..313a4577b50 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -41,8 +41,8 @@ class CacheConfig: block_size: SkipValidation[int] = None # type: ignore[assignment] """Size of a contiguous cache block in number of tokens. - This is None until `Platform.check_and_update_config()` sets it based on - the current platform. Always an int by the time the engine starts.""" + This is None until the platform sets it. Always an int by the time + the engine starts.""" gpu_memory_utilization: float = Field(default=0.9, gt=0, le=1) """The fraction of GPU memory to be used for the model executor, which can range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index e951e6f2c8a..fffe769e7a4 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -915,32 +915,6 @@ class VllmConfig: ) current_platform.check_and_update_config(self) - # If DCP, ensure the block size is right. - if self.parallel_config.decode_context_parallel_size > 1: - if self.parallel_config.dcp_kv_cache_interleave_size > 1 and ( - self.parallel_config.cp_kv_cache_interleave_size - != self.parallel_config.dcp_kv_cache_interleave_size - ): - self.parallel_config.cp_kv_cache_interleave_size = ( - self.parallel_config.dcp_kv_cache_interleave_size - ) - logger.warning_once( - "cp_kv_cache_interleave_size is overridden by dcp_kv_cache" - "_interleave_size. And dcp-kv-cache-interleave-size will be " - "deprecated when PCP is fully supported." - ) - assert ( - self.parallel_config.cp_kv_cache_interleave_size - <= self.cache_config.block_size - and self.cache_config.block_size - % self.parallel_config.cp_kv_cache_interleave_size - == 0 - ), ( - f"Block_size({self.cache_config.block_size}) should be greater " - "than or equal to and divisible by cp_kv_cache_interleave_size " - f"({self.parallel_config.cp_kv_cache_interleave_size})." - ) - # Do this after all the updates to compilation_config.mode effective_dp_size = ( self.parallel_config.data_parallel_size @@ -1108,26 +1082,6 @@ class VllmConfig: # Default to enable HMA if not explicitly disabled by user or logic above. self.scheduler_config.disable_hybrid_kv_cache_manager = False - if self.cache_config.mamba_cache_mode == "align": - assert ( - self.cache_config.block_size - <= self.scheduler_config.max_num_batched_tokens - ), ( - "In Mamba cache align mode, block_size " - f"({self.cache_config.block_size}) must be <= " - "max_num_batched_tokens " - f"({self.scheduler_config.max_num_batched_tokens})." - ) - if self.scheduler_config.long_prefill_token_threshold > 0: - assert ( - self.scheduler_config.long_prefill_token_threshold - >= self.cache_config.block_size - ) - assert not self.scheduler_config.disable_chunked_mm_input, ( - "Chunked MM input is required because we need the flexibility to " - "schedule a multiple of block_size tokens even if they are in the " - "middle of a mm input" - ) if self.compilation_config.debug_dump_path: self.compilation_config.debug_dump_path = ( self.compilation_config.debug_dump_path.absolute().expanduser() @@ -1488,6 +1442,57 @@ class VllmConfig: f"compilation_config={self.compilation_config!r}" ) + def validate_block_size(self) -> None: + """Validate block_size against DCP and mamba constraints. + + Called after Platform.update_block_size_for_backend() has + finalised block_size, so that the checks see the real value + rather than the initial None sentinel. + """ + block_size = self.cache_config.block_size + assert block_size is not None, ( + "validate_block_size called before block_size was set" + ) + + # DCP interleave-size compatibility + if self.parallel_config.decode_context_parallel_size > 1: + if self.parallel_config.dcp_kv_cache_interleave_size > 1 and ( + self.parallel_config.cp_kv_cache_interleave_size + != self.parallel_config.dcp_kv_cache_interleave_size + ): + self.parallel_config.cp_kv_cache_interleave_size = ( + self.parallel_config.dcp_kv_cache_interleave_size + ) + logger.warning_once( + "cp_kv_cache_interleave_size is overridden by dcp_kv_cache" + "_interleave_size. And dcp-kv-cache-interleave-size will be " + "deprecated when PCP is fully supported." + ) + assert ( + self.parallel_config.cp_kv_cache_interleave_size <= block_size + and block_size % self.parallel_config.cp_kv_cache_interleave_size == 0 + ), ( + f"Block_size({block_size}) should be greater " + "than or equal to and divisible by cp_kv_cache_interleave_size " + f"({self.parallel_config.cp_kv_cache_interleave_size})." + ) + + # Mamba cache align-mode constraints + if self.cache_config.mamba_cache_mode == "align": + assert block_size <= self.scheduler_config.max_num_batched_tokens, ( + "In Mamba cache align mode, block_size " + f"({block_size}) must be <= " + "max_num_batched_tokens " + f"({self.scheduler_config.max_num_batched_tokens})." + ) + if self.scheduler_config.long_prefill_token_threshold > 0: + assert self.scheduler_config.long_prefill_token_threshold >= block_size + assert not self.scheduler_config.disable_chunked_mm_input, ( + "Chunked MM input is required because we need the flexibility " + "to schedule a multiple of block_size tokens even if they are " + "in the middle of a mm input" + ) + @model_validator(mode="after") def validate_mamba_block_size(self) -> "VllmConfig": if self.model_config is None: diff --git a/vllm/model_executor/layers/attention/chunked_local_attention.py b/vllm/model_executor/layers/attention/chunked_local_attention.py index e33733c0cc1..522981820d6 100644 --- a/vllm/model_executor/layers/attention/chunked_local_attention.py +++ b/vllm/model_executor/layers/attention/chunked_local_attention.py @@ -30,9 +30,8 @@ from vllm.v1.kv_cache_interface import ( def create_chunked_local_attention_backend( underlying_attn_backend: AttentionBackend, attention_chunk_size: int, - block_size: int, ) -> type[AttentionBackend]: - prefix = f"ChunkedLocalAttention_{attention_chunk_size}_{block_size}_" + prefix = f"ChunkedLocalAttention_{attention_chunk_size}_" underlying_builder = underlying_attn_backend.get_builder_cls() assert issubclass(underlying_builder, AttentionMetadataBuilder) @@ -55,7 +54,9 @@ def create_chunked_local_attention_backend( fast_build: bool = False, ): cm, make_virtual_batches_block_table = make_local_attention_virtual_batches( - attention_chunk_size, common_attn_metadata, block_size + attention_chunk_size, + common_attn_metadata, + self.kv_cache_spec.block_size, ) metadata = super().build(common_prefix_len, cm, fast_build) metadata.make_virtual_batches_block_table = make_virtual_batches_block_table @@ -97,13 +98,13 @@ class ChunkedLocalAttention(Attention): block_size = cache_config.block_size else: kv_cache_dtype = "auto" - block_size = 16 + block_size = None underlying_attn_backend = get_attn_backend( head_size, dtype, kv_cache_dtype, block_size ) attn_backend = create_chunked_local_attention_backend( - underlying_attn_backend, attention_chunk_size, block_size + underlying_attn_backend, attention_chunk_size ) super().__init__( diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 98ff02e9d4a..4fe25b02779 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -407,17 +407,24 @@ class MLAAttention(nn.Module, AttentionLayerBase): ) # Attributes for forward_impl method - self.chunked_prefill_workspace_size = ( - MLACommonMetadataBuilder.determine_chunked_prefill_workspace_size( - get_current_vllm_config() - ) - ) + self._vllm_config = get_current_vllm_config() + self._chunked_prefill_workspace_size: int | None = None self._decode_concat_quant_fp8_op = _DecodeConcatQuantFP8( static=True, group_shape=GroupShape.PER_TENSOR, compile_native=True, ) + @property + def chunked_prefill_workspace_size(self) -> int: + if self._chunked_prefill_workspace_size is None: + self._chunked_prefill_workspace_size = ( + MLACommonMetadataBuilder.determine_chunked_prefill_workspace_size( + self._vllm_config + ) + ) + return self._chunked_prefill_workspace_size + def forward( self, q: torch.Tensor, diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 2314d0a8b67..921054f73e6 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -169,21 +169,6 @@ class CudaPlatformBase(Platform): if parallel_config.worker_cls == "auto": parallel_config.worker_cls = "vllm.v1.worker.gpu_worker.Worker" - cache_config = vllm_config.cache_config - user_specified_block_size = cache_config.block_size is not None - if not user_specified_block_size: - cache_config.block_size = 16 - - # Ensure block_size is compatible with the attention backend. - # Note: model_config may be None during testing. - # Skip hybrid (attention+mamba) models — their block_size is - # managed by HybridAttentionMambaModelConfig - if model_config is not None and not model_config.is_hybrid: - cls._update_block_size_for_backend( - vllm_config, - user_specified_block_size, - ) - scheduler_config = vllm_config.scheduler_config # Note: model_config may be None during testing if ( @@ -199,148 +184,47 @@ class CudaPlatformBase(Platform): scheduler_config.disable_chunked_mm_input = True @classmethod - def _update_block_size_for_backend( - cls, - vllm_config: "VllmConfig", - user_specified_block_size: bool, - ) -> None: - """Ensure block_size is compatible with the attention backend. - - If the user specified --block-size, the selector validates/filters - backends by that block size (raising on incompatibility). Otherwise, - the backend is selected unconstrained and block_size is set to the - backend's preferred value. - """ - from vllm.config.vllm import set_current_vllm_config - from vllm.v1.attention.selector import AttentionSelectorConfig - - model_config = vllm_config.model_config + def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: cache_config = vllm_config.cache_config - - device_capability = cls.get_device_capability() - if device_capability is None: + if cache_config.block_size is not None: + # User specified --block-size; keep it. return - use_mla = model_config.use_mla - attn_selector_config = AttentionSelectorConfig( - head_size=model_config.get_head_size(), - dtype=model_config.dtype, # type: ignore[arg-type] - kv_cache_dtype=cache_config.cache_dtype, - block_size=cache_config.block_size if user_specified_block_size else None, - use_mla=use_mla, - has_sink=False, - use_sparse=use_mla and hasattr(model_config.hf_config, "index_topk"), - use_mm_prefix=model_config.is_mm_prefix_lm, + model_config = vllm_config.model_config + # model_config may be None during testing. + # Skip hybrid models — their block_size is managed by + # HybridAttentionMambaModelConfig. + if model_config is None or model_config.is_hybrid: + cache_config.block_size = 16 + return + + from vllm.config.vllm import ( + get_layers_from_vllm_config, + set_current_vllm_config, + ) + from vllm.model_executor.layers.attention_layer_base import ( + AttentionLayerBase, ) - user_specified_backend = vllm_config.attention_config.backend - num_heads = model_config.get_num_attention_heads( - vllm_config.parallel_config, + attn_layers = get_layers_from_vllm_config( + vllm_config, + AttentionLayerBase, ) + if not attn_layers: + cache_config.block_size = 16 + return + + first_layer = next(iter(attn_layers.values())) + backend_cls = first_layer.get_attn_backend() with set_current_vllm_config(vllm_config): - chosen_backend = cls.select_attention_backend( - selected_backend=user_specified_backend, - attn_selector_config=attn_selector_config, - device_capability=device_capability, - # Don't raise here — we produce better errors below. - raise_on_invalid=False, - num_heads=num_heads, + preferred = backend_cls.get_preferred_block_size(16) + if preferred != 16: + logger.info( + "Setting kv cache block size to %d for %s backend.", + preferred, + backend_cls.get_name(), ) - - # If the user's --block-size forced a non-optimal backend, - # warn them. Only relevant when the user didn't also specify - # --attention-backend (in which case the choice is explicit). - if ( - chosen_backend is not None - and user_specified_block_size - and user_specified_backend is None - ): - optimal = cls.select_attention_backend( - selected_backend=None, - attn_selector_config=attn_selector_config._replace( - block_size=None, - ), - device_capability=device_capability, - raise_on_invalid=False, - num_heads=num_heads, - ) - if optimal is not None and optimal != chosen_backend: - logger.warning( - "--block-size %d is not supported by the preferred " - "%s backend. Using %s instead, which may result " - "in reduced performance. Consider removing " - "--block-size to auto-select the optimal " - "block size.", - cache_config.block_size, - optimal.name, - chosen_backend.name, - ) - - if chosen_backend is not None: - if user_specified_block_size: - # User's block_size is compatible with the chosen - # backend. - return - # User didn't specify --block-size, so auto-select the - # preferred block size for the chosen backend. - try: - backend_class = chosen_backend.get_class() - except ImportError: - return # Will fail later with a better error - preferred = backend_class.get_preferred_block_size( - cache_config.block_size, - ) - if cache_config.block_size != preferred: - logger.info( - "Setting kv cache block size to %d for %s backend.", - preferred, - chosen_backend.name, - ) - cache_config.block_size = preferred - return - - # No valid backend found. If the user didn't constrain the - # selection, defer the error to get_attn_backend_cls where - # the full config (including per-layer settings) is - # available. - if not user_specified_block_size: - return - - if user_specified_backend is not None: - # User specified --block-size and --attention-backend - # and they are incompatible. - try: - backend_class = user_specified_backend.get_class() - supported = backend_class.get_supported_kernel_block_sizes() - except ImportError: - supported = None - raise ValueError( - f"User-specified --block-size " - f"{cache_config.block_size} is incompatible with " - f"the specified --attention-backend " - f"{user_specified_backend.name} (supported kernel " - f"block sizes: {supported}). Either remove " - f"--block-size to auto-select, or choose a " - f"compatible value." - ) - else: - # User specified --block-size but no backend supports - # it. - _, invalid_reasons = cls.get_valid_backends( - device_capability=device_capability, - attn_selector_config=attn_selector_config, - num_heads=num_heads, - ) - reasons_str = ", ".join( - f"{b.name}: [{', '.join(r)}]" for b, r in invalid_reasons.items() - ) - raise ValueError( - f"No valid attention backend found for " - f"--block-size {cache_config.block_size}. " - f"Reasons: {{{reasons_str}}}. Either remove " - f"--block-size to auto-select, or choose a " - f"compatible value." - ) + cache_config.block_size = preferred @classmethod def get_current_memory_usage( @@ -358,10 +242,10 @@ class CudaPlatformBase(Platform): num_heads: int | None = None, ) -> tuple[ list[tuple["AttentionBackendEnum", int]], - dict["AttentionBackendEnum", list[str]], + dict["AttentionBackendEnum", tuple[int, list[str]]], ]: valid_backends_priorities = [] - invalid_reasons = {} + invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {} backend_priorities = _get_backend_priorities( attn_selector_config.use_mla, @@ -378,7 +262,7 @@ class CudaPlatformBase(Platform): except ImportError: invalid_reasons_i = ["ImportError"] if invalid_reasons_i: - invalid_reasons[backend] = invalid_reasons_i + invalid_reasons[backend] = (priority, invalid_reasons_i) else: valid_backends_priorities.append((backend, priority)) @@ -439,7 +323,7 @@ class CudaPlatformBase(Platform): "{" + ", ".join( f"{backend.name}: [{', '.join(reasons)}]" - for backend, reasons in invalid_reasons.items() + for backend, (_, reasons) in invalid_reasons.items() ) + "}" ) @@ -452,7 +336,30 @@ class CudaPlatformBase(Platform): # Select the one with the highest priority (lowest index). sorted_backends = sorted(valid_backends_priorities, key=lambda x: x[1]) - return sorted_backends[0][0] + chosen_backend, chosen_priority = sorted_backends[0] + + # If the user specified --block-size (but not --attention-backend), + # check whether that constraint precluded any higher-priority backends. + if attn_selector_config.block_size is not None: + excluded = [ + backend + for backend, (priority, reasons) in invalid_reasons.items() + if priority < chosen_priority + and reasons == ["block_size not supported"] + ] + if excluded: + names = ", ".join(b.name for b in excluded) + logger.warning( + "--block-size %d excluded higher-priority backend(s) " + "%s. Using %s instead, which may result in reduced " + "performance. Consider removing --block-size to " + "auto-select the optimal block size.", + attn_selector_config.block_size, + names, + chosen_backend.name, + ) + + return chosen_backend @classmethod def get_attn_backend_cls( @@ -487,7 +394,7 @@ class CudaPlatformBase(Platform): "{" + ", ".join( f"{backend.name}: [{', '.join(reasons)}]" - for backend, reasons in invalid_reasons.items() + for backend, (_, reasons) in invalid_reasons.items() ) + "}" ) @@ -499,7 +406,7 @@ class CudaPlatformBase(Platform): logger.info_once( "Using %s attention backend out of potential backends: %s", chosen_backend.name, - tuple(b[0].name for b in valid_backends_priorities), + tuple(backend.name for backend, _ in valid_backends_priorities), scope="local", ) diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index 6794c05f5e5..ba44fa6d967 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -406,6 +406,13 @@ class Platform: """ pass + @classmethod + def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: + """ + Ensure block_size is compatible with the attention backend. + """ + pass + @classmethod def verify_model_arch(cls, model_arch: str) -> None: """ diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index 573a31027e7..d7a52b090c0 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -114,7 +114,14 @@ class EngineCore: num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( vllm_config ) - + if kv_cache_config.kv_cache_groups: + vllm_config.cache_config.block_size = min( + g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups + ) + elif vllm_config.cache_config.block_size is None: + # Attention-free models (encoder-only, SSM) — use default. + vllm_config.cache_config.block_size = 16 + vllm_config.validate_block_size() vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks self.collective_rpc("initialize_cache", args=(num_gpu_blocks, num_cpu_blocks)) diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index b63cbd6586f..9cc7dc63ad8 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -41,6 +41,7 @@ from vllm.distributed.parallel_state import ( ) from vllm.envs import enable_envs_cache from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.tracing import instrument, maybe_init_worker_tracer from vllm.utils.network_utils import ( get_distributed_init_method, @@ -579,6 +580,9 @@ class WorkerProc: self._init_message_queues(input_shm_handle, vllm_config) self.worker.load_model() + # Set block size based on the attention backends + current_platform.update_block_size_for_backend(vllm_config) + # Enable environment variable cache (e.g. assume no more # environment variable overrides after this point) enable_envs_cache() diff --git a/vllm/v1/executor/ray_executor.py b/vllm/v1/executor/ray_executor.py index ad51526ae94..6c939a59387 100644 --- a/vllm/v1/executor/ray_executor.py +++ b/vllm/v1/executor/ray_executor.py @@ -385,6 +385,11 @@ class RayDistributedExecutor(Executor): self.collective_rpc("init_device") self.collective_rpc("load_model") + def _update_block_size(worker): + current_platform.update_block_size_for_backend(worker.vllm_config) + + self.collective_rpc(_update_block_size) + for pp_rank in range(self.parallel_config.pipeline_parallel_size): self.pp_tp_workers.append([]) for tp_rank in range(self.parallel_config.tensor_parallel_size): diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index b9c7b550170..290c4dc8bbc 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -12,6 +12,7 @@ import torch.distributed as dist import vllm.envs as envs from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType @@ -46,6 +47,7 @@ class UniProcExecutor(Executor): self.driver_worker.init_worker(all_kwargs=[kwargs]) self.driver_worker.init_device() self.driver_worker.load_model() + current_platform.update_block_size_for_backend(self.vllm_config) def _distributed_args(self) -> tuple[str, int, int]: """Return (distributed_init_method, rank, local_rank).""" diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 41ec062305b..ba1428c42ee 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -513,6 +513,7 @@ class GPUModelRunner( custom_logitsprocs: Sequence[str | type[LogitsProcessor]] = ( tuple(logits_processors) if logits_processors is not None else () ) + placeholder_block_size = self.cache_config.block_size or 16 self.input_batch = InputBatch( max_num_reqs=self.max_num_reqs, # We need to use the encoder length for encoder-decoer @@ -522,8 +523,8 @@ class GPUModelRunner( device=self.device, pin_memory=self.pin_memory, vocab_size=self.model_config.get_vocab_size(), - block_sizes=[self.cache_config.block_size], - kernel_block_sizes=[self.cache_config.block_size], + block_sizes=[placeholder_block_size], + kernel_block_sizes=[placeholder_block_size], is_spec_decode=bool(self.vllm_config.speculative_config), logitsprocs=build_logitsprocs( self.vllm_config, From f72061a19ae7fbb7f193c31f0abea355fab41892 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 19 Feb 2026 18:20:52 -0500 Subject: [PATCH 08/74] [UX] More descriptive reasons in is_supported_config for MoE (#34908) Signed-off-by: mgoin --- .../layers/fused_moe/flashinfer_trtllm_moe.py | 23 +++++++++++-------- .../layers/fused_moe/modular_kernel.py | 11 ++++++--- .../quantization/utils/flashinfer_fp4_moe.py | 14 ++++++----- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/flashinfer_trtllm_moe.py b/vllm/model_executor/layers/fused_moe/flashinfer_trtllm_moe.py index 910c8387726..732ab8e929c 100644 --- a/vllm/model_executor/layers/fused_moe/flashinfer_trtllm_moe.py +++ b/vllm/model_executor/layers/fused_moe/flashinfer_trtllm_moe.py @@ -129,25 +129,28 @@ def is_supported_config_trtllm_fp8( return f"kernel does not support {reason}" if not _supports_current_device(): - return False, _make_reason("current device") + return False, _make_reason(f"current device {current_platform.device_name}") elif not (moe_config.is_act_and_mul or _supports_no_act_and_mul()): return False, _make_reason("no act_and_mul MLP layer") elif not _supports_activation(moe_config.activation): return False, _make_reason(f"{moe_config.activation} activation") elif not _supports_quant_scheme(weight_key, activation_key): - return False, _make_reason("quantization scheme") + return False, _make_reason(f"quantization scheme {weight_key}x{activation_key}") elif not _supports_parallel_config(moe_config.moe_parallel_config): - return False, _make_reason("parallel config") + return False, _make_reason(f"parallel config {moe_config.moe_parallel_config}") elif not _supports_routing_method( weight_key, activation_key, moe_config.routing_method ): - return False, _make_reason("routing method") + return False, _make_reason(f"routing method {moe_config.routing_method}") elif activation_format != mk.FusedMoEActivationFormat.Standard: - return False, _make_reason("activation format") + return False, _make_reason(f"activation format {activation_format}") elif not _supports_router_logits_dtype( moe_config.router_logits_dtype, moe_config.routing_method ): - return False, _make_reason("float32 router_logits with non-DeepSeekV3 routing") + return False, _make_reason( + "float32 router_logits with non-DeepSeekV3 routing " + f"{moe_config.router_logits_dtype}x{moe_config.routing_method}" + ) return True, None @@ -165,17 +168,17 @@ def is_supported_config_trtllm_bf16( return f"kernel does not support {reason}" if not _supports_current_device(): - return False, _make_reason("current device") + return False, _make_reason(f"current device {current_platform.device_name}") elif not (moe_config.is_act_and_mul or _supports_no_act_and_mul()): return False, _make_reason("no act_and_mul MLP layer") elif not _supports_activation(moe_config.activation): return False, _make_reason(f"{moe_config.activation} activation") elif not _supports_parallel_config(moe_config.moe_parallel_config): - return False, _make_reason("parallel config") + return False, _make_reason(f"parallel config {moe_config.moe_parallel_config}") elif not _supports_routing_method_bf16(moe_config.routing_method): - return False, _make_reason("routing method") + return False, _make_reason(f"routing method {moe_config.routing_method}") elif activation_format != mk.FusedMoEActivationFormat.Standard: - return False, _make_reason("activation format") + return False, _make_reason(f"activation format {activation_format}") return True, None diff --git a/vllm/model_executor/layers/fused_moe/modular_kernel.py b/vllm/model_executor/layers/fused_moe/modular_kernel.py index 7e6855778fd..b4ceaa379f0 100644 --- a/vllm/model_executor/layers/fused_moe/modular_kernel.py +++ b/vllm/model_executor/layers/fused_moe/modular_kernel.py @@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.utils import ( from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, ) +from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.v1.worker.ubatching import ( dbo_enabled, @@ -498,15 +499,19 @@ class FusedMoEPermuteExpertsUnpermute(ABC): return f"kernel does not support {reason}" if not cls._supports_current_device(): - return False, _make_reason("current device") + return False, _make_reason(f"current device {current_platform.device_name}") elif not (moe_config.is_act_and_mul or cls._supports_no_act_and_mul()): return False, _make_reason("no act_and_mul MLP layer") elif not cls._supports_activation(moe_config.activation): return False, _make_reason(f"{moe_config.activation} activation") elif not cls._supports_quant_scheme(weight_key, activation_key): - return False, _make_reason("quantization scheme") + return False, _make_reason( + f"quantization scheme {weight_key}x{activation_key}" + ) elif not cls._supports_parallel_config(moe_config.moe_parallel_config): - return False, _make_reason("parallel config") + return False, _make_reason( + f"parallel config {moe_config.moe_parallel_config}" + ) elif activation_format != cls.activation_format(): return False, _make_reason(f"{activation_format.value} activation format") return True, None diff --git a/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py b/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py index d61303923ea..840663703ce 100644 --- a/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py +++ b/vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py @@ -109,21 +109,23 @@ def is_supported_config_trtllm( return f"kernel does not support {reason}" if not _supports_current_device(): - return False, _make_reason("current device") + return False, _make_reason(f"current device {current_platform.device_name}") elif not (moe_config.is_act_and_mul or _supports_no_act_and_mul()): return False, _make_reason("no act_and_mul MLP layer") elif not _supports_activation(moe_config.activation): return False, _make_reason(f"{moe_config.activation} activation") elif not _supports_quant_scheme(weight_key, activation_key): - return False, _make_reason("quantization scheme") + return False, _make_reason(f"quantization scheme {weight_key}x{activation_key}") elif not _supports_parallel_config(moe_config.moe_parallel_config): - return False, _make_reason("parallel config") + return False, _make_reason(f"parallel config {moe_config.moe_parallel_config}") elif not _supports_routing_method(moe_config.routing_method): - return False, _make_reason("routing method") + return False, _make_reason(f"routing method {moe_config.routing_method}") elif activation_format != mk.FusedMoEActivationFormat.Standard: - return False, _make_reason("activation format") + return False, _make_reason(f"activation format {activation_format}") elif moe_config.hidden_dim % 512 != 0: - return False, _make_reason("hidden_dim must be divisible by 512") + return False, _make_reason( + f"hidden_dim must be divisible by 512, found {moe_config.hidden_dim}" + ) return True, None From 648951a9c3ab7d8ade25b80edb55eb4018acfd58 Mon Sep 17 00:00:00 2001 From: Mayank Ketkar Date: Thu, 19 Feb 2026 16:01:00 -0800 Subject: [PATCH 09/74] [Bugfix] Fix benchmark_fused_collective crash on CustomOp init (#34665) Signed-off-by: Mayank Ketkar Signed-off-by: Mayank Ketkar Co-authored-by: Mayank Ketkar Co-authored-by: Claude Opus 4.6 (1M context) --- .../kernels/benchmark_fused_collective.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/benchmarks/kernels/benchmark_fused_collective.py b/benchmarks/kernels/benchmark_fused_collective.py index 3cd52160dfb..633529edf16 100644 --- a/benchmarks/kernels/benchmark_fused_collective.py +++ b/benchmarks/kernels/benchmark_fused_collective.py @@ -408,18 +408,18 @@ def run_benchmarks( rms_eps = 1e-6 results = {} - vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) use_oneshot_options = [False] if no_oneshot else [True, False] - # Create RMSNorm and QuantFP8 layers once for native benchmarks - if "none" in quant_modes: # Standard AllReduce + RMSNorm + # Re-create VllmFusedAllreduce per config so CustomOp binds the + # correct forward method (native vs custom kernel). for custom_op in ["-rms_norm", "+rms_norm"]: with set_current_vllm_config( VllmConfig(compilation_config=CompilationConfig(custom_ops=[custom_op])) ): try: + vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) suffix = ( "_custom_rms_norm" if "+" in custom_op else "_native_rms_norm" ) @@ -438,6 +438,7 @@ def run_benchmarks( VllmConfig(compilation_config=CompilationConfig(custom_ops=["-rms_norm"])) ): try: + vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) standard_allreduce_rmsnorm_native_compiled = torch.compile( vllm_fused_allreduce.allreduce_rmsnorm, fullgraph=True, @@ -482,7 +483,7 @@ def run_benchmarks( "_custom_rms_norm" if "+" in rms_norm_custom_op else "_native_rms_norm" ) for quant_fp8_custom_op in ["-quant_fp8", "+quant_fp8"]: - suffix += ( + op_suffix = suffix + ( "_custom_quant_fp8" if "+" in quant_fp8_custom_op else "_native_quant_fp8" @@ -495,16 +496,17 @@ def run_benchmarks( ) ): try: + vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) time_ms = benchmark_operation( vllm_fused_allreduce.allreduce_rmsnorm_fp8_quant, input_tensor, residual=residual, scale_factor=scale_fp8, ) - results[f"standard_allreduce{suffix}"] = time_ms + results[f"standard_allreduce{op_suffix}"] = time_ms except Exception as e: logger.error("Standard AllReduce+RMSNorm+FP8 failed: %s", e) - results[f"standard_allreduce{suffix}"] = float("inf") + results[f"standard_allreduce{op_suffix}"] = float("inf") # Standard AllReduce + RMSNorm + FP8 Quant Native Compiled with set_current_vllm_config( @@ -515,6 +517,7 @@ def run_benchmarks( ) ): try: + vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) standard_allreduce_rmsnorm_fp8_quant_native_compiled = torch.compile( vllm_fused_allreduce.allreduce_rmsnorm_fp8_quant, fullgraph=True, @@ -580,6 +583,7 @@ def run_benchmarks( ) ): try: + vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) time_ms = benchmark_operation( vllm_fused_allreduce.allreduce_rmsnorm_fp4_quant, input_tensor, @@ -598,6 +602,7 @@ def run_benchmarks( VllmConfig(compilation_config=CompilationConfig(custom_ops=["-rms_norm"])) ): try: + vllm_fused_allreduce = VllmFusedAllreduce(hidden_dim, dtype) standard_allreduce_rmsnorm_fp4_quant_native_compiled = torch.compile( vllm_fused_allreduce.allreduce_rmsnorm_fp4_quant, fullgraph=True, From 40b2f1c3d9c1dbcec185e8b6911fd273524f5b88 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 19 Feb 2026 16:05:37 -0800 Subject: [PATCH 10/74] [Model Runner V2] Minor CPU optimizations (#34856) Signed-off-by: Nick Hill --- .../device_communicators/shm_broadcast.py | 4 ++-- vllm/v1/worker/gpu/async_utils.py | 19 ++++++++++++++++--- vllm/v1/worker/gpu/buffer_utils.py | 8 +++----- vllm/v1/worker/gpu/model_runner.py | 7 +++++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/vllm/distributed/device_communicators/shm_broadcast.py b/vllm/distributed/device_communicators/shm_broadcast.py index ef5f74c1e4c..ac46a566737 100644 --- a/vllm/distributed/device_communicators/shm_broadcast.py +++ b/vllm/distributed/device_communicators/shm_broadcast.py @@ -513,8 +513,8 @@ class MessageQueue: assert self._is_local_reader, "Only readers can acquire read" start_time = time.monotonic() n_warning = 1 - while True: - with self.buffer.get_metadata(self.current_idx) as metadata_buffer: + with self.buffer.get_metadata(self.current_idx) as metadata_buffer: + while True: # Memory fence ensures we see the latest writes from the writer. # Without this, we may read stale flags from our CPU cache # and spin indefinitely even though writer has updated them. diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index afcfa8dfb4d..e628e38bd02 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextlib import numpy as np import torch @@ -14,6 +15,7 @@ class AsyncOutput(AsyncModelRunnerOutput): model_runner_output: ModelRunnerOutput, sampler_output: SamplerOutput, num_sampled_tokens: torch.Tensor, + main_stream: torch.cuda.Stream, copy_stream: torch.cuda.Stream, copy_event: torch.cuda.Event, ): @@ -25,9 +27,8 @@ class AsyncOutput(AsyncModelRunnerOutput): self.num_sampled_tokens = num_sampled_tokens self.copy_event = copy_event - default_stream = torch.cuda.current_stream() - with torch.cuda.stream(copy_stream): - copy_stream.wait_stream(default_stream) + with stream(copy_stream, main_stream): + copy_stream.wait_stream(main_stream) self.sampled_token_ids = async_copy_to_np(sampler_output.sampled_token_ids) self.logprobs_tensors: LogprobsTensors | None = None @@ -71,3 +72,15 @@ class AsyncOutput(AsyncModelRunnerOutput): def async_copy_to_np(x: torch.Tensor) -> np.ndarray: return x.to("cpu", non_blocking=True).numpy() + + +@contextlib.contextmanager +def stream(to_stream: torch.cuda.Stream, from_stream: torch.cuda.Stream): + """Lightweight version of torch.cuda.stream() context manager which + avoids current_stream and device lookups. + """ + try: + torch.cuda.set_stream(to_stream) + yield + finally: + torch.cuda.set_stream(from_stream) diff --git a/vllm/v1/worker/gpu/buffer_utils.py b/vllm/v1/worker/gpu/buffer_utils.py index d2cb20186ff..ad910933aa2 100644 --- a/vllm/v1/worker/gpu/buffer_utils.py +++ b/vllm/v1/worker/gpu/buffer_utils.py @@ -22,7 +22,6 @@ def async_copy_to_gpu( if isinstance(x, np.ndarray): x = torch.from_numpy(x) assert x.is_cpu - assert not x.is_pinned() if out is None: assert device is not None @@ -30,6 +29,8 @@ def async_copy_to_gpu( # CPU-to-CPU copy tmp = x.pin_memory() + assert tmp is not x + # CPU-to-GPU copy return out.copy_(tmp, non_blocking=True) @@ -75,11 +76,8 @@ class UvaBufferPool: out: torch.Tensor | None = None, ) -> torch.Tensor: uva = self.copy_to_uva(x) - if out is None: - # CPU-to-GPU copy - return uva.clone() # CPU-to-GPU copy - return out.copy_(uva, non_blocking=True) + return uva.clone() if out is None else out.copy_(uva, non_blocking=True) class UvaBackedTensor: diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index cbae001c29c..57d25822913 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools import gc import time from copy import deepcopy @@ -239,6 +240,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): def get_model(self) -> nn.Module: return self.model + @functools.cached_property + def main_stream(self) -> torch.cuda.Stream: + # Cache the default CUDA stream to avoid lookup overhead. + return torch.cuda.current_stream(self.device) + def get_kv_cache_spec(self): return get_kv_cache_spec(self.vllm_config) @@ -1065,6 +1071,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): model_runner_output=model_runner_output, sampler_output=sampler_output, num_sampled_tokens=num_sampled, + main_stream=self.main_stream, copy_stream=self.output_copy_stream, copy_event=self.output_copy_event, ) From 16f24e87975ef4cd2c12879425062913ef62f6fd Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Thu, 19 Feb 2026 20:14:54 -0500 Subject: [PATCH 11/74] [CI] Add GPT-OSS Eval job for H100 (#34359) Signed-off-by: Michael Goin --- .buildkite/test_areas/misc.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f58aa204b55..c6b43b97aec 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -147,6 +147,19 @@ steps: - pytest -v -s transformers_utils - pytest -v -s config +- label: GPT-OSS Eval (H100) + timeout_in_minutes: 60 + working_dir: "/vllm-workspace/" + device: h100 + optional: true + source_file_dependencies: + - tests/evals/gpt_oss + - vllm/model_executor/models/gpt_oss.py + - vllm/model_executor/layers/quantization/mxfp4.py + commands: + - uv pip install --system 'gpt-oss[eval]==0.0.5' + - pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58 + - label: GPT-OSS Eval (B200) timeout_in_minutes: 60 working_dir: "/vllm-workspace/" From 76df6072ff4829980ad71764191fc970a873275a Mon Sep 17 00:00:00 2001 From: Mark McLoughlin Date: Fri, 20 Feb 2026 01:21:46 +0000 Subject: [PATCH 12/74] [Core] Fix state names in pause_scheduler() (#34840) Signed-off-by: Mark McLoughlin --- vllm/v1/engine/core.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index d7a52b090c0..b805abe8a66 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -1370,15 +1370,15 @@ class EngineCoreProc(EngineCore): ) -> Future | None: """Pause generation; behavior depends on mode. - All pause states queue new adds. PAUSE_ABORT and PAUSE_KEEP skip step(); - PAUSE_WAIT allows step() so in-flight requests can drain. + All pause modes queue new adds -- "abort" and "keep" skip step(); + "wait" allows step() so in-flight requests can drain. - - ``abort``: Set PAUSE_ABORT, abort all requests, wait for abort - outputs to be sent (when running with output_queue), clear caches, - then complete the returned Future. - - ``wait``: Set PAUSE_WAIT (queue adds, keep stepping); when drained, - set PAUSE_KEEP, clear caches, complete the returned Future. - - ``keep``: Set PAUSE_KEEP; return a Future that completes when the + - ``abort``: Set PAUSED_NEW, abort all requests, wait for abort + outputs to be sent (when running with output_queue), optionally + clear caches, then complete the returned Future. + - ``wait``: Set PAUSED_NEW (queue adds, keep stepping); when drained, + optionally clear caches, then complete the returned Future. + - ``keep``: Set PAUSED_ALL; return a Future that completes when the output queue is empty. """ if mode not in ("keep", "abort", "wait"): From ac900c89bba77f69ed42e8a19a5006bd215eeb80 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 20 Feb 2026 11:57:55 +0800 Subject: [PATCH 13/74] [Refactor] Implement output type check in LLM (#34794) Signed-off-by: DarkLight1337 --- vllm/entrypoints/llm.py | 92 +++++++++++++++++++++++------------- vllm/v1/engine/llm_engine.py | 4 -- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 9d1e2912c4b..f1b32c750b0 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -10,7 +10,7 @@ import cloudpickle import torch.nn as nn from pydantic import ValidationError from tqdm.auto import tqdm -from typing_extensions import TypeVar +from typing_extensions import TypeVar, overload from vllm.beam_search import ( BeamSearchInstance, @@ -94,6 +94,11 @@ if TYPE_CHECKING: logger = init_logger(__name__) +_O = TypeVar( + "_O", + bound=RequestOutput | PoolingRequestOutput, + default=RequestOutput | PoolingRequestOutput, +) _P = TypeVar("_P", bound=SamplingParams | PoolingParams | None) _R = TypeVar("_R", default=Any) @@ -447,17 +452,16 @@ class LLM: if sampling_params is None: sampling_params = self.get_default_sampling_params() - outputs = self._run_completion( + return self._run_completion( prompts=prompts, params=sampling_params, + output_type=RequestOutput, use_tqdm=use_tqdm, lora_request=lora_request, tokenization_kwargs=tokenization_kwargs, priority=priority, ) - return self.engine_class.validate_outputs(outputs, RequestOutput) - def enqueue( self, prompts: PromptType | Sequence[PromptType], @@ -524,23 +528,43 @@ class LLM: return request_ids + @overload def wait_for_completion( self, + *, use_tqdm: bool | Callable[..., tqdm] = True, - ) -> list[RequestOutput]: + ) -> list[RequestOutput | PoolingRequestOutput]: ... + + @overload + def wait_for_completion( + self, + output_type: type[_O] | tuple[type[_O], ...], + *, + use_tqdm: bool | Callable[..., tqdm] = True, + ) -> list[_O]: ... + + def wait_for_completion( + self, + output_type: type[Any] | tuple[type[Any], ...] | None = None, + *, + use_tqdm: bool | Callable[..., tqdm] = True, + ) -> list[Any]: """Wait for all enqueued requests to complete and return results. This method processes all requests currently in the engine queue and returns their outputs. Use after enqueue() to get results. Args: + output_type: The expected output type, defaults to RequestOutput. use_tqdm: If True, shows a tqdm progress bar. Returns: - A list of RequestOutput objects for all completed requests. + A list of output objects for all completed requests. """ - outputs = self._run_engine(use_tqdm=use_tqdm) - return self.engine_class.validate_outputs(outputs, RequestOutput) + if output_type is None: + output_type = (RequestOutput, PoolingRequestOutput) + + return self._run_engine(output_type, use_tqdm=use_tqdm) def _resolve_mm_lora( self, @@ -744,13 +768,13 @@ class LLM: # only runs for one step # we don't need to use tqdm here - raw_output = self._render_and_run_requests( + output = self._render_and_run_requests( prompts=(beam.get_prompt() for beam in all_beams), params=self._params_to_seq(sampling_params, len(all_beams)), + output_type=RequestOutput, lora_requests=[beam.lora_request for beam in all_beams], use_tqdm=False, ) - output = self.engine_class.validate_outputs(raw_output, RequestOutput) for (start, end), instance in zip( instance_start_and_end, instances_batch @@ -987,9 +1011,10 @@ class LLM: if sampling_params is None: sampling_params = self.get_default_sampling_params() - outputs = self._run_chat( + return self._run_chat( messages=messages, params=sampling_params, + output_type=RequestOutput, use_tqdm=use_tqdm, lora_request=lora_request, chat_template=chat_template, @@ -1002,8 +1027,6 @@ class LLM: mm_processor_kwargs=mm_processor_kwargs, ) - return self.engine_class.validate_outputs(outputs, RequestOutput) - def encode( self, prompts: PromptType | Sequence[PromptType] | DataPrompt, @@ -1135,19 +1158,16 @@ class LLM: outputs = self._run_completion( prompts=prompts_seq, params=params_seq, + output_type=PoolingRequestOutput, use_tqdm=use_tqdm, lora_request=lora_request, tokenization_kwargs=tokenization_kwargs, ) - model_outputs = self.engine_class.validate_outputs( - outputs, PoolingRequestOutput - ) - if use_io_processor: # get the post-processed model outputs assert self.io_processor is not None - processed_outputs = self.io_processor.post_process(model_outputs) + processed_outputs = self.io_processor.post_process(outputs) return [ PoolingRequestOutput[Any]( @@ -1160,8 +1180,8 @@ class LLM: finished=True, ) ] - else: - return model_outputs + + return outputs def embed( self, @@ -1353,8 +1373,7 @@ class LLM: embed_2=encoded_output_2, ) - items = self.engine_class.validate_outputs(scores, PoolingRequestOutput) - return [ScoringRequestOutput.from_base(item) for item in items] + return [ScoringRequestOutput.from_base(item) for item in scores] def _late_interaction_score( self, @@ -1393,7 +1412,7 @@ class LLM: ) text_2.append(text) - encoded_output: list[PoolingRequestOutput] = self.encode( + encoded_output = self.encode( text_1 + text_2, use_tqdm=use_tqdm, lora_request=lora_request, @@ -1402,8 +1421,8 @@ class LLM: tokenization_kwargs=tokenization_kwargs, ) - encoded_output_1: list[PoolingRequestOutput] = encoded_output[0 : len(text_1)] - encoded_output_2: list[PoolingRequestOutput] = encoded_output[len(text_1) :] + encoded_output_1 = encoded_output[0 : len(text_1)] + encoded_output_2 = encoded_output[len(text_1) :] if len(encoded_output_1) == 1: encoded_output_1 = encoded_output_1 * len(encoded_output_2) @@ -1434,8 +1453,7 @@ class LLM: ) ) - items = self.engine_class.validate_outputs(scores, PoolingRequestOutput) - return [ScoringRequestOutput.from_base(item) for item in items] + return [ScoringRequestOutput.from_base(item) for item in scores] def _cross_encoding_score( self, @@ -1491,13 +1509,12 @@ class LLM: outputs = self._run_completion( prompts=prompts, params=pooling_params_list, + output_type=PoolingRequestOutput, use_tqdm=use_tqdm, lora_request=lora_request, ) - items = self.engine_class.validate_outputs(outputs, PoolingRequestOutput) - - return [ScoringRequestOutput.from_base(item) for item in items] + return [ScoringRequestOutput.from_base(item) for item in outputs] def score( self, @@ -1759,6 +1776,7 @@ class LLM: params: SamplingParams | PoolingParams | Sequence[SamplingParams | PoolingParams], + output_type: type[_O], *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, @@ -1790,6 +1808,7 @@ class LLM: ) ), params=seq_params, + output_type=output_type, use_tqdm=use_tqdm, lora_requests=seq_lora_requests, priorities=seq_priority, @@ -1802,6 +1821,7 @@ class LLM: params: SamplingParams | PoolingParams | Sequence[SamplingParams | PoolingParams], + output_type: type[_O], *, use_tqdm: bool | Callable[..., tqdm] = True, lora_request: Sequence[LoRARequest] | LoRARequest | None = None, @@ -1848,6 +1868,7 @@ class LLM: ) ), params=seq_params, + output_type=output_type, lora_requests=seq_lora_requests, use_tqdm=use_tqdm, ) @@ -1856,6 +1877,7 @@ class LLM: self, prompts: Iterable[ProcessorInputs], params: Sequence[SamplingParams | PoolingParams], + output_type: type[_O], *, lora_requests: Sequence[LoRARequest | None] | None = None, priorities: Sequence[int] | None = None, @@ -1878,7 +1900,7 @@ class LLM: priorities=priorities, ) - return self._run_engine(use_tqdm=use_tqdm) + return self._run_engine(output_type, use_tqdm=use_tqdm) def _render_and_add_requests( self, @@ -1932,9 +1954,10 @@ class LLM: def _run_engine( self, + output_type: type[_O] | tuple[type[_O], ...], *, use_tqdm: bool | Callable[..., tqdm] = True, - ) -> list[RequestOutput | PoolingRequestOutput]: + ) -> list[_O]: # Initialize tqdm. if use_tqdm: num_requests = self.llm_engine.get_num_unfinished_requests() @@ -1947,14 +1970,15 @@ class LLM: ) # Run the engine. - outputs: list[RequestOutput | PoolingRequestOutput] = [] + outputs: list[_O] = [] total_in_toks = 0 total_out_toks = 0 while self.llm_engine.has_unfinished_requests(): step_outputs = self.llm_engine.step() for output in step_outputs: + assert isinstance(output, output_type) if output.finished: - outputs.append(output) + outputs.append(output) # type: ignore[arg-type] if use_tqdm: if isinstance(output, RequestOutput): # Calculate tokens only for RequestOutput diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index c4f0442f3e4..6a8df0dc734 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -199,10 +199,6 @@ class LLMEngine: self.should_execute_dummy_batch = True return aggregated_has_unfinished - @classmethod - def validate_outputs(cls, outputs, output_type): - return outputs - def get_supported_tasks(self) -> tuple[SupportedTask, ...]: if not hasattr(self, "_supported_tasks"): # Cache the result From a1a2d79442ed00284e70b829e07cadbb887bdf73 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 19 Feb 2026 19:59:15 -0800 Subject: [PATCH 14/74] [ci] Use the right tag for CPU arm64 image (#34915) Signed-off-by: Kevin H. Luu --- .buildkite/image_build/image_build_cpu_arm64.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.buildkite/image_build/image_build_cpu_arm64.sh b/.buildkite/image_build/image_build_cpu_arm64.sh index 3f25fbaec7a..ff3d11c8d59 100755 --- a/.buildkite/image_build/image_build_cpu_arm64.sh +++ b/.buildkite/image_build/image_build_cpu_arm64.sh @@ -14,7 +14,7 @@ BUILDKITE_COMMIT=$3 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" # skip build if image already exists -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then +if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then echo "Image not found, proceeding with build..." else echo "Image found" @@ -25,9 +25,9 @@ fi docker build --file docker/Dockerfile.cpu \ --build-arg max_jobs=16 \ --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \ + --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu \ --target vllm-test \ --progress plain . # push -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu +docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu From d9e62c03eb98e3adcf82a2177f4a8b8f851406e4 Mon Sep 17 00:00:00 2001 From: Bowen Bao Date: Thu, 19 Feb 2026 21:27:14 -0800 Subject: [PATCH 15/74] [Quark] Fix MoE fp8 activation scale handling on mi300 (#34386) Signed-off-by: Bowen Bao --- vllm/model_executor/layers/quantization/quark/quark_moe.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 66db0950599..8394857cf9c 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -858,7 +858,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): layer.w2_input_scale = None def process_weights_after_loading(self, layer): - if self.static_input_scales: + if self.static_input_scales and self.input_dtype == "fp8": # firstly, process activations if fp8 static input if layer.w13_input_scale is None or layer.w2_input_scale is None: raise ValueError( @@ -883,14 +883,14 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): if current_platform.is_fp8_fnuz(): # Normalize the weights and scales _, _, w13_input_scale = normalize_e4m3fn_to_e4m3fnuz( - torch.empty_like(layer.w13_weight, dtype=torch.float8_e4m3fnuz), + torch.empty_like(layer.w13_weight, dtype=torch.float8_e4m3fn), torch.empty_like( layer.w13_weight_scale, dtype=layer.w13_weight_scale.dtype ), layer.w13_input_scale, ) _, _, w2_input_scale = normalize_e4m3fn_to_e4m3fnuz( - torch.empty_like(layer.w2_weight, dtype=torch.float8_e4m3fnuz), + torch.empty_like(layer.w2_weight, dtype=torch.float8_e4m3fn), torch.empty_like( layer.w2_weight_scale, dtype=layer.w13_weight_scale.dtype ), From 4e2c7caf2d11444ce6c1e4895bc921c93610bd7c Mon Sep 17 00:00:00 2001 From: Matthias Gehre Date: Fri, 20 Feb 2026 06:27:26 +0100 Subject: [PATCH 16/74] [Bugfix] Add regression test for MoE quant_config under torch.compile (#34335) Signed-off-by: Matthias Gehre --- tests/quantization/test_compressed_tensors.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/quantization/test_compressed_tensors.py b/tests/quantization/test_compressed_tensors.py index 795591ec35e..e5a047a7c34 100644 --- a/tests/quantization/test_compressed_tensors.py +++ b/tests/quantization/test_compressed_tensors.py @@ -816,3 +816,26 @@ def test_compressed_tensors_moe_ignore_with_model(vllm_runner): # Verify the model can generate output output = llm.generate_greedy("Hello, my name is", max_tokens=4) assert output + + +def test_w4a16_moe_torch_compile(vllm_runner): + """Regression test: MoE quant_config must be initialized inside the + moe_forward custom op, not just in forward_native which is compiled by + Dynamo (attribute mutations are not replayed at runtime). + + Without the fix in _moe_forward/_moe_forward_shared, this hits: + AssertionError: Hidden size mismatch 2048 != 1024 + because use_int4_w4a16 is False (moe_quant_config stays None). + """ + model_path = "nm-testing/tinysmokeqwen3moe-W4A16-first-only-CTstable" + + with vllm_runner( + model_path, + enforce_eager=False, + max_model_len=256, + compilation_config={ + "cudagraph_mode": "NONE", + }, + ) as llm: + output = llm.generate_greedy("Hi", max_tokens=1) + assert output From 81bfc21a6ad0cb498dbe5466ccf2987624efbba5 Mon Sep 17 00:00:00 2001 From: Elizabeth Thomas Date: Thu, 19 Feb 2026 23:29:08 -0600 Subject: [PATCH 17/74] [Model Bash]: Improve FP8 Oracle for Config Specific Kernel Selection (#34260) Signed-off-by: Elizabeth Thomas Signed-off-by: Robert Shaw Signed-off-by: Robert Shaw Co-authored-by: Robert Shaw Co-authored-by: Robert Shaw --- .../layers/fused_moe/oracle/fp8.py | 59 +++++++++++++++---- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 50b89eb35ed..2432209899e 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -34,6 +34,8 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import ( QuantKey, + kFp8Dynamic128Sym, + kFp8Static128BlockSym, ) from vllm.platforms import current_platform @@ -55,6 +57,49 @@ class Fp8MoeBackend(Enum): XPU = "XPU" +def _get_priority_backends( + moe_config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, +) -> list[Fp8MoeBackend]: + """ + Get available backends in priority order based on platform and config. + + This function can be extended to become more complex as needed. + """ + + _AVAILABLE_BACKENDS = [ + Fp8MoeBackend.AITER, + Fp8MoeBackend.FLASHINFER_TRTLLM, + Fp8MoeBackend.FLASHINFER_CUTLASS, + Fp8MoeBackend.DEEPGEMM, + Fp8MoeBackend.VLLM_CUTLASS, + Fp8MoeBackend.TRITON, + Fp8MoeBackend.MARLIN, + Fp8MoeBackend.BATCHED_DEEPGEMM, + Fp8MoeBackend.BATCHED_VLLM_CUTLASS, + Fp8MoeBackend.BATCHED_TRITON, + Fp8MoeBackend.XPU, + ] + + def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None: + backends.insert(0, backends.pop(backends.index(backend))) + + # On Hopper for Block Fp8, prefer Triton for TP and FI CUTLASS for EP. + if ( + current_platform.is_cuda() + and current_platform.is_device_capability(90) + and activation_key == kFp8Dynamic128Sym + and weight_key == kFp8Static128BlockSym + ): + if moe_config.moe_parallel_config.ep_size > 1: + _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.FLASHINFER_CUTLASS) + else: + _move_to_front(_AVAILABLE_BACKENDS, Fp8MoeBackend.TRITON) + + return _AVAILABLE_BACKENDS + + def backend_to_kernel_cls( backend: Fp8MoeBackend, ) -> type[mk.FusedMoEPermuteExpertsUnpermute]: @@ -151,19 +196,7 @@ def select_fp8_moe_backend( return Fp8MoeBackend.TRITON, backend_to_kernel_cls(Fp8MoeBackend.TRITON) # NOTE: the kernels are selected in the following order. - AVAILABLE_BACKENDS = [ - Fp8MoeBackend.AITER, - Fp8MoeBackend.FLASHINFER_TRTLLM, - Fp8MoeBackend.FLASHINFER_CUTLASS, - Fp8MoeBackend.DEEPGEMM, - Fp8MoeBackend.BATCHED_DEEPGEMM, - Fp8MoeBackend.VLLM_CUTLASS, - Fp8MoeBackend.BATCHED_VLLM_CUTLASS, - Fp8MoeBackend.TRITON, - Fp8MoeBackend.BATCHED_TRITON, - Fp8MoeBackend.MARLIN, - Fp8MoeBackend.XPU, - ] + AVAILABLE_BACKENDS = _get_priority_backends(config, weight_key, activation_key) # NOTE(rob): We need to peak into the P/F selection to determine # if we are using the batched or standard expert format, which From 676f82ae8140a512dae73bcae6c6d23907f55e0e Mon Sep 17 00:00:00 2001 From: Varun Chawla <34209028+veeceey@users.noreply.github.com> Date: Thu, 19 Feb 2026 21:30:33 -0800 Subject: [PATCH 18/74] Add validation to reject non-text content in system messages (#34072) Signed-off-by: Varun Chawla --- tests/entrypoints/openai/test_chat_error.py | 139 +++++++++++++++++- .../openai/chat_completion/protocol.py | 49 ++++++ 2 files changed, 187 insertions(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_chat_error.py b/tests/entrypoints/openai/test_chat_error.py index 41b8b52c42c..7d84be21844 100644 --- a/tests/entrypoints/openai/test_chat_error.py +++ b/tests/entrypoints/openai/test_chat_error.py @@ -4,7 +4,7 @@ from dataclasses import dataclass, field from http import HTTPStatus from typing import Any -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -233,3 +233,140 @@ async def test_chat_error_stream(): f"Expected error message in chunks: {chunks}" ) assert chunks[-1] == "data: [DONE]\n\n" + + +@pytest.mark.parametrize( + "image_content", + [ + [{"type": "image_url", "image_url": {"url": "https://example.com/image.jpg"}}], + [{"image_url": {"url": "https://example.com/image.jpg"}}], + ], +) +def test_system_message_warns_on_image(image_content): + """Test that system messages with image content trigger a warning.""" + with patch( + "vllm.entrypoints.openai.chat_completion.protocol.logger" + ) as mock_logger: + ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + { + "role": "system", + "content": image_content, + } + ], + ) + + mock_logger.warning_once.assert_called() + call_args = str(mock_logger.warning_once.call_args) + assert "System messages should only contain text" in call_args + assert "image_url" in call_args + + +def test_system_message_accepts_text(): + """Test that system messages can contain text content.""" + # Should not raise an exception + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + {"role": "system", "content": "You are a helpful assistant."}, + ], + ) + assert request.messages[0]["role"] == "system" + + +def test_system_message_accepts_text_array(): + """Test that system messages can contain an array with text content.""" + # Should not raise an exception + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + { + "role": "system", + "content": [{"type": "text", "text": "You are a helpful assistant."}], + }, + ], + ) + assert request.messages[0]["role"] == "system" + + +def test_user_message_accepts_image(): + """Test that user messages can still contain image content.""" + # Should not raise an exception + request = ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.jpg"}, + }, + ], + }, + ], + ) + assert request.messages[0]["role"] == "user" + + +@pytest.mark.parametrize( + "audio_content", + [ + [ + { + "type": "input_audio", + "input_audio": {"data": "base64data", "format": "wav"}, + } + ], + [{"input_audio": {"data": "base64data", "format": "wav"}}], + ], +) +def test_system_message_warns_on_audio(audio_content): + """Test that system messages with audio content trigger a warning.""" + with patch( + "vllm.entrypoints.openai.chat_completion.protocol.logger" + ) as mock_logger: + ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + { + "role": "system", + "content": audio_content, + } + ], + ) + + mock_logger.warning_once.assert_called() + call_args = str(mock_logger.warning_once.call_args) + assert "System messages should only contain text" in call_args + assert "input_audio" in call_args + + +@pytest.mark.parametrize( + "video_content", + [ + [{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}], + [{"video_url": {"url": "https://example.com/video.mp4"}}], + ], +) +def test_system_message_warns_on_video(video_content): + """Test that system messages with video content trigger a warning.""" + with patch( + "vllm.entrypoints.openai.chat_completion.protocol.logger" + ) as mock_logger: + ChatCompletionRequest( + model=MODEL_NAME, + messages=[ + { + "role": "system", + "content": video_content, + } + ], + ) + + mock_logger.warning_once.assert_called() + call_args = str(mock_logger.warning_once.call_args) + assert "System messages should only contain text" in call_args + assert "video_url" in call_args diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 71e59152a85..14feb49767b 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -674,3 +674,52 @@ class ChatCompletionRequest(OpenAIBaseModel): "Parameter 'cache_salt' must be a non-empty string if provided." ) return data + + @model_validator(mode="before") + @classmethod + def check_system_message_content_type(cls, data): + """Warn if system messages contain non-text content. + + According to OpenAI API spec, system messages can only be of type + 'text'. We log a warning instead of rejecting to avoid breaking + users who intentionally send multimodal system messages. + See: https://platform.openai.com/docs/api-reference/chat/create#chat_create-messages-system_message + """ + if not isinstance(data, dict): + return data + messages = data.get("messages", []) + for msg in messages: + # Check if this is a system message + if isinstance(msg, dict) and msg.get("role") == "system": + content = msg.get("content") + + # If content is a list (multimodal format) + if isinstance(content, list): + for part in content: + if isinstance(part, dict): + part_type = part.get("type") + # Infer type when 'type' field is not explicit + if part_type is None: + if "image_url" in part or "image_pil" in part: + part_type = "image_url" + elif "image_embeds" in part: + part_type = "image_embeds" + elif "audio_url" in part: + part_type = "audio_url" + elif "input_audio" in part: + part_type = "input_audio" + elif "audio_embeds" in part: + part_type = "audio_embeds" + elif "video_url" in part: + part_type = "video_url" + + # Warn about non-text content in system messages + if part_type and part_type != "text": + logger.warning_once( + "System messages should only contain text " + "content according to the OpenAI API spec. " + "Found content type: '%s'.", + part_type, + ) + + return data From 0c1dc42748760fc75aef68e973c9ff7a47501337 Mon Sep 17 00:00:00 2001 From: rasmith Date: Thu, 19 Feb 2026 23:32:40 -0600 Subject: [PATCH 19/74] [CI][AMD][BugFix][P/D] Add default_vllm_config to test_moriio_connector.py so tests pass (#33739) Signed-off-by: Randall Smith --- tests/v1/kv_connector/unit/test_moriio_connector.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_moriio_connector.py b/tests/v1/kv_connector/unit/test_moriio_connector.py index 1cc6988635d..1eca4964fd6 100644 --- a/tests/v1/kv_connector/unit/test_moriio_connector.py +++ b/tests/v1/kv_connector/unit/test_moriio_connector.py @@ -17,6 +17,7 @@ from vllm.config import ( ModelConfig, SchedulerConfig, VllmConfig, + set_current_vllm_config, ) from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import ( MoRIIOAgentMetadata, @@ -433,10 +434,11 @@ def test_register_kv_caches(mock_parallel_groups): } ) - connector = MoRIIOConnector(vllm_config, KVConnectorRole.WORKER) - connector.connector_worker = FakeMorIIOConnectorWorker( - vllm_config, connector.engine_id, hand_shake_latency=0 - ) + with set_current_vllm_config(vllm_config): + connector = MoRIIOConnector(vllm_config, KVConnectorRole.WORKER) + connector.connector_worker = FakeMorIIOConnectorWorker( + vllm_config, connector.engine_id, hand_shake_latency=0 + ) from mori.io import ( MemoryDesc, @@ -523,7 +525,8 @@ def test_moriio_handshake_returns_metadata(mock_parallel_groups): "handshake_port": handshake_port, } ) - connector = MoRIIOConnector(vllm_config, KVConnectorRole.WORKER) + with set_current_vllm_config(vllm_config): + connector = MoRIIOConnector(vllm_config, KVConnectorRole.WORKER) # Execute register_kv_caches connector.register_kv_caches(kv_caches) From 07cab212f0dcc51cfe4e4f93b58935e8079f26b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=A8=E6=9C=B1=20=C2=B7=20Kiki?= Date: Fri, 20 Feb 2026 13:33:25 +0800 Subject: [PATCH 20/74] [Misc] Add deprecated environment variable utilities (#33677) Signed-off-by: carlory Co-authored-by: Claude Opus 4.5 Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- vllm/config/utils.py | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/vllm/config/utils.py b/vllm/config/utils.py index d17637338fe..c6fca2f9386 100644 --- a/vllm/config/utils.py +++ b/vllm/config/utils.py @@ -7,6 +7,7 @@ import enum import hashlib import inspect import json +import os import pathlib import textwrap from collections.abc import Callable, Mapping, Sequence, Set @@ -21,6 +22,7 @@ from pydantic.fields import Field as PydanticField from pydantic.fields import FieldInfo from typing_extensions import dataclass_transform, runtime_checkable +import vllm.envs as envs from vllm.logger import init_logger logger = init_logger(__name__) @@ -380,3 +382,66 @@ def handle_deprecated( for new_name in new_names: setattr(config, new_name, old_val) + + +def get_from_deprecated_env_if_set( + env_name: str, + removal_version: str, + field_name: str | None = None, +) -> str | None: + """ + Get value from deprecated environment variable with warning. + + Args: + env_name: Name of the deprecated environment variable + removal_version: Version when it will be removed + field_name: Name of the field to suggest as alternative + + Returns: + The environment variable value if set, None otherwise + """ + if envs.is_set(env_name): + value = os.environ.get(env_name) + alt_msg = f" Please use {field_name} instead." if field_name else "" + logger.warning_once( + "Using %s environment variable is deprecated and will be removed in %s.%s", + env_name, + removal_version, + alt_msg, + ) + return value + return None + + +def set_from_deprecated_env_if_set( + config: ConfigT, + env_name: str, + removal_version: str, + field_name: str, + to_bool: bool = False, + to_int: bool = False, +) -> None: + """ + Set object field from deprecated environment variable with warning. + + Args: + config: Config object to set the field on + env_name: Name of the deprecated environment variable + removal_version: Version when the env var will be removed + field_name: Name of the field to set + to_bool: Whether to convert the environment variable value to boolean + to_int: Whether to convert the environment variable value to integer + Returns: + None + """ + if to_bool and to_int: + raise ValueError("Cannot convert to both boolean and integer.") + + env_value = get_from_deprecated_env_if_set(env_name, removal_version, field_name) + if env_value is not None: + field_value: str | bool | int = env_value + if to_bool: + field_value = env_value.lower() in ("1", "true") + elif to_int: + field_value = int(env_value) + setattr(config, field_name, field_value) From f5432e35a3a4f0bd6e7d49c51a35a0a01bc32452 Mon Sep 17 00:00:00 2001 From: Micah Williamson Date: Thu, 19 Feb 2026 23:37:49 -0600 Subject: [PATCH 21/74] [ROCm][CI] Loosen RemoteOpenAIServer Startup Timeout (#34922) Signed-off-by: Micah Williamson --- tests/entrypoints/openai/test_serving_chat.py | 2 +- tests/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 1d96b05ac71..33c69578ce9 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -126,7 +126,7 @@ def gptoss_speculative_server(default_server_args: list[str]): if is_aiter_found_and_supported(): env_dict = {"VLLM_ROCM_USE_AITER": "1"} with RemoteOpenAIServer( - GPT_OSS_MODEL_NAME, server_args, env_dict=env_dict + GPT_OSS_MODEL_NAME, server_args, env_dict=env_dict, max_wait_seconds=480 ) as remote_server: yield remote_server diff --git a/tests/utils.py b/tests/utils.py index 5252115f291..9ab6df9e231 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -190,7 +190,7 @@ class RemoteOpenAIServer: model_loader.download_model(model_config) self._start_server(model, vllm_serve_args, env_dict) - max_wait_seconds = max_wait_seconds or 240 + max_wait_seconds = max_wait_seconds or 360 self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds) def __enter__(self): From ea37530b474fa738a99a53a8975af4e389b968c7 Mon Sep 17 00:00:00 2001 From: tianshu-Michael-yu <101950379+tianshu-Michael-yu@users.noreply.github.com> Date: Thu, 19 Feb 2026 22:07:23 -0800 Subject: [PATCH 22/74] [Models] LFM2: Support LoRA (#34921) Co-authored-by: Piotr Mazurek Co-authored-by: Cyrus Leung --- vllm/model_executor/models/lfm2.py | 26 ++++++++++++++++++-------- vllm/model_executor/models/lfm2_moe.py | 26 ++++++++++++++++++-------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/vllm/model_executor/models/lfm2.py b/vllm/model_executor/models/lfm2.py index fa611ad504b..453173fc817 100644 --- a/vllm/model_executor/models/lfm2.py +++ b/vllm/model_executor/models/lfm2.py @@ -39,6 +39,7 @@ from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP, Suppo from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, @@ -66,12 +67,12 @@ class Lfm2MLP(nn.Module): ff_dim = int(ffn_dim_multiplier * ff_dim) ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of) - self.w1 = MergedColumnParallelLinear( + self.w13 = MergedColumnParallelLinear( input_size=dim, output_sizes=[ff_dim] * 2, bias=False, quant_config=quant_config, - prefix=f"{prefix}.w1", + prefix=f"{prefix}.w13", ) self.w2 = RowParallelLinear( input_size=ff_dim, @@ -83,7 +84,7 @@ class Lfm2MLP(nn.Module): self.act_fn = SiluAndMul() def forward(self, x: torch.Tensor) -> torch.Tensor: - gate_up, _ = self.w1(x) + gate_up, _ = self.w13(x) x = self.act_fn(gate_up) x, _ = self.w2(x) return x @@ -376,8 +377,8 @@ class Lfm2Model(nn.Module): (".qkv_proj", ".q_proj", "q"), (".qkv_proj", ".k_proj", "k"), (".qkv_proj", ".v_proj", "v"), - (".w1", ".w1", 0), - (".w1", ".w3", 1), + (".w13", ".w1", 0), + (".w13", ".w3", 1), ] params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -386,9 +387,11 @@ class Lfm2Model(nn.Module): name = name.replace(".conv.", ".short_conv.", 1) for param_name, weight_name, shard_id in stacked_params_mapping: - if weight_name not in name: + # Use segment-boundary matching (trailing dot) to prevent + # e.g. ".w1" from matching inside ".w13" in pre-fused keys. + if weight_name + "." not in name: continue - name = name.replace(weight_name, param_name) + name = name.replace(weight_name + ".", param_name + ".") if is_pp_missing_parameter(name, self): continue @@ -415,13 +418,20 @@ class Lfm2ForCausalLM( "k_proj", "v_proj", ], - "w1": [ + "w13": [ "w1", "w3", ], "in_proj": ["in_proj"], } + # HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision + # with the inner .conv.conv child (ShortConv has a child self.conv, so + # naming the container .conv too makes _match_target_modules match both) + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={".conv.": ".short_conv."}, + ) + # LoRA specific attributes embedding_modules = { "embed_tokens": "input_embeddings", diff --git a/vllm/model_executor/models/lfm2_moe.py b/vllm/model_executor/models/lfm2_moe.py index 22bd554bdda..b7ca710eadc 100644 --- a/vllm/model_executor/models/lfm2_moe.py +++ b/vllm/model_executor/models/lfm2_moe.py @@ -52,6 +52,7 @@ from .interfaces import ( from .utils import ( AutoWeightsLoader, PPMissingLayer, + WeightsMapper, extract_layer_index, is_pp_missing_parameter, make_empty_intermediate_tensors_factory, @@ -69,12 +70,12 @@ class Lfm2MoeMlp(nn.Module): prefix: str = "", ): super().__init__() - self.w1 = MergedColumnParallelLinear( + self.w13 = MergedColumnParallelLinear( input_size=dim, output_sizes=[ff_dim] * 2, bias=False, quant_config=quant_config, - prefix=f"{prefix}.w1", + prefix=f"{prefix}.w13", ) self.w2 = RowParallelLinear( input_size=ff_dim, @@ -86,7 +87,7 @@ class Lfm2MoeMlp(nn.Module): self.act_fn = SiluAndMul() def forward(self, x: torch.Tensor) -> torch.Tensor: - gate_up, _ = self.w1(x) + gate_up, _ = self.w13(x) x = self.act_fn(gate_up) x, _ = self.w2(x) return x @@ -501,8 +502,8 @@ class Lfm2MoeModel(nn.Module): (".qkv_proj", ".q_proj", "q"), (".qkv_proj", ".k_proj", "k"), (".qkv_proj", ".v_proj", "v"), - (".w1", ".w1", 0), - (".w1", ".w3", 1), + (".w13", ".w1", 0), + (".w13", ".w3", 1), ] params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() @@ -516,12 +517,14 @@ class Lfm2MoeModel(nn.Module): for param_name, weight_name, shard_id in stacked_params_mapping: # Skip non-stacked layers and experts (experts handled below). - if weight_name not in name: + # Use segment-boundary matching (trailing dot) to prevent + # e.g. ".w1" from matching inside ".w13" in pre-fused keys. + if weight_name + "." not in name: continue if ("feed_forward.experts." in name) and name not in params_dict: continue - name = name.replace(weight_name, param_name) + name = name.replace(weight_name + ".", param_name + ".") # Skip loading extra bias for GPTQ models. if ( name.endswith(".bias") or name.endswith("_bias") @@ -596,13 +599,20 @@ class Lfm2MoeForCausalLM( "k_proj", "v_proj", ], - "w1": [ + "w13": [ "w1", "w3", ], "in_proj": ["in_proj"], } + # HF uses .conv. but vLLM uses .short_conv. to avoid LoRA regex collision + # with the inner .conv.conv child (ShortConv has a child self.conv, so + # naming the container .conv too makes _match_target_modules match both) + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_substr={".conv.": ".short_conv."}, + ) + # LoRA specific attributes embedding_modules = { "embed_tokens": "input_embeddings", From 059779231f158b8b570e71aaa5c66f49b41b2fb1 Mon Sep 17 00:00:00 2001 From: Frank Wang <41319051+frankwang28@users.noreply.github.com> Date: Thu, 19 Feb 2026 22:07:57 -0800 Subject: [PATCH 23/74] [Minor] Add logging when using MXFP4 MXFP8 TRTLLM backend (#34916) Signed-off-by: frankwang28 Signed-off-by: Frank Wang <41319051+frankwang28@users.noreply.github.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> --- vllm/model_executor/layers/quantization/mxfp4.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/model_executor/layers/quantization/mxfp4.py b/vllm/model_executor/layers/quantization/mxfp4.py index 4b24885b431..492963855e8 100644 --- a/vllm/model_executor/layers/quantization/mxfp4.py +++ b/vllm/model_executor/layers/quantization/mxfp4.py @@ -128,6 +128,9 @@ def get_mxfp4_backend(with_lora_support: bool) -> Mxfp4Backend: and has_flashinfer() and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8 ): + logger.info_once( + "Using FlashInfer MXFP4 MXFP8 TRTLLM backend for SM100", scope="local" + ) return Mxfp4Backend.SM100_FI_MXFP4_MXFP8_TRTLLM elif current_platform.is_device_capability_family(100) and has_flashinfer(): logger.info_once( From 8de7c636cc02a8306441af868b9c1d0e6d64799f Mon Sep 17 00:00:00 2001 From: Kevin McKay Date: Fri, 20 Feb 2026 00:25:46 -0600 Subject: [PATCH 24/74] [Bugfix][Hardware][AMD] Fix ROCM_AITER_FA speculative decoding support (#32877) Signed-off-by: c0de128 Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com> --- vllm/v1/attention/backends/rocm_aiter_fa.py | 37 +++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 5ff45082971..141d57d908e 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -1076,10 +1076,43 @@ class AiterFlashAttentionImpl(AttentionImpl): # calculate for decodes if num_decodes > 0: assert attn_metadata.decode_metadata is not None - if self.sliding_window[0] != -1: + decode_max_query_len = attn_metadata.decode_metadata.max_query_len + + # Use unified_attention for speculative decoding (multi-token) + # or when sliding window is enabled + if self.sliding_window[0] != -1 or decode_max_query_len > 1: assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), ( - "Sliding window with shuffle layout is not supported yet." + "Shuffle KV cache layout is not supported with sliding " + "window or speculative decoding (multi-token decode)." ) + from aiter.ops.triton.unified_attention import ( + unified_attention, + ) + + descale_shape = ( + attn_metadata.query_start_loc[:num_decodes].shape[0] - 1, + key_cache.shape[2], + ) + unified_attention( + q=query[:num_decode_tokens], + k=key_cache, + v=value_cache, + out=output[:num_decode_tokens], + cu_seqlens_q=attn_metadata.query_start_loc[:num_decodes], + max_seqlen_q=decode_max_query_len, + seqused_k=attn_metadata.seq_lens[:num_decodes], + max_seqlen_k=attn_metadata.max_seq_len, + softmax_scale=self.scale, + causal=True, + alibi_slopes=self.alibi_slopes, + window_size=self.sliding_window, + block_table=attn_metadata.block_table[:num_decodes], + softcap=self.logits_soft_cap, + q_descale=None, + k_descale=layer._k_scale.expand(descale_shape), + v_descale=layer._v_scale.expand(descale_shape), + ) + return if rocm_aiter_ops.is_shuffle_kv_cache_enabled(): num_blocks, block_size, num_kv_heads, head_size = key_cache.shape From b1c4f0b26548d36fca304b298957e4791eafa09b Mon Sep 17 00:00:00 2001 From: Xin Yang <105740670+xyang16@users.noreply.github.com> Date: Fri, 20 Feb 2026 01:34:45 -0800 Subject: [PATCH 25/74] [Kernel] Optimize grouped topk kernel (#34206) Signed-off-by: Xin Yang --- csrc/moe/grouped_topk_kernels.cu | 459 ++++++++++++++++++++----- csrc/moe/moeTopKFuncs.cuh | 257 ++++++++++++++ tests/kernels/moe/test_grouped_topk.py | 25 +- 3 files changed, 642 insertions(+), 99 deletions(-) create mode 100644 csrc/moe/moeTopKFuncs.cuh diff --git a/csrc/moe/grouped_topk_kernels.cu b/csrc/moe/grouped_topk_kernels.cu index eaebf4e353e..6a4dad3be7c 100644 --- a/csrc/moe/grouped_topk_kernels.cu +++ b/csrc/moe/grouped_topk_kernels.cu @@ -1,6 +1,6 @@ /* * Adapted from - * https://github.com/NVIDIA/TensorRT-LLM/blob/v0.21.0/cpp/tensorrt_llm/kernels/noAuxTcKernels.cu + * https://github.com/NVIDIA/TensorRT-LLM/blob/v1.3.0rc2/cpp/tensorrt_llm/kernels/noAuxTcKernels.cu * Copyright (c) 2025, The vLLM team. * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & * AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 @@ -17,8 +17,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include "moeTopKFuncs.cuh" #include #include +#include #include #include #include @@ -30,7 +32,17 @@ namespace vllm { namespace moe { constexpr unsigned FULL_WARP_MASK = 0xffffffff; -constexpr int32_t WARP_SIZE = 32; +static constexpr int WARP_SIZE = 32; +static constexpr int NumNemotronExperts = 512; +static constexpr int NumKimiK2Experts = 384; +static constexpr int NumDeepseekExperts = 256; +static constexpr int MaxSupportedExpertCount = + std::max({NumNemotronExperts, NumKimiK2Experts, NumDeepseekExperts}); +static constexpr int MaxNumExpertsUnit = 128; +static constexpr int NumTopGroupScores = 2; +static constexpr int DefaultMaxNumTopExperts = 8; +static constexpr int MaxSupportedTopExperts = 22; +static constexpr int MaxNumTopGroups = 4; namespace warp_topk { @@ -657,76 +669,335 @@ __global__ void grouped_topk_fused_kernel( #endif } -template +template +__global__ void grouped_topk_fused_small_expert_count_kernel( + T* scores, float* topkValues, IdxT* topkIndices, BiasT const* routingBias, + int64_t const numTokens, int64_t const numGroup, int64_t const topkGroup, + int64_t const topk, int64_t const numExperts, + int64_t const numExpertsPerGroup, bool const renormalize, + double const routedScalingFactor) { +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaGridDependencySynchronize(); +#endif + // declare shared memory structure + // number of experts is bounded by number of threads + __shared__ float __attribute((aligned(128))) smemScoreSigmoid[MaxNumExperts]; + __shared__ float __attribute((aligned(128))) smemScoreBias[MaxNumExperts]; + // number of expert groups is bounded by number of warps + int constexpr NumWarps = MaxNumExperts / WARP_SIZE; + __shared__ float __attribute((aligned(128))) smemGroupScores[NumWarps]; + + // needed for warp reduce + auto block = cg::this_thread_block(); + auto warp = cg::tiled_partition(block); + + // for the final reduction of weight norm, only some lanes need to participate + int32_t laneIdx = threadIdx.x % WARP_SIZE; + int32_t warpIdx = __shfl_sync(0xffffffff, threadIdx.x / WARP_SIZE, 0); + + if constexpr (UseGroups) { + if (warpIdx >= numGroup) { + return; + } + } + // note that for invalid scores, we simply use a negative value: + // they work well even with the compacted format used in topK, and + // sigmoid / bias activated scores cannot be negative + const float invalidScoreFloat = float{-INFINITY}; + + // load bias already; each warp represents one expert group + auto threadExpert = threadIdx.x; + bool expertSelected = threadExpert < numExperts; + if constexpr (UseGroups) { + threadExpert = warpIdx * numExpertsPerGroup + laneIdx; + expertSelected = laneIdx < numExpertsPerGroup; + } + + auto scoreIdx = int64_t{blockIdx.x} * int64_t{numExperts} + threadExpert; + auto biasVal = expertSelected ? static_cast(routingBias[threadExpert]) + : invalidScoreFloat; + topkValues += blockIdx.x * topk; + topkIndices += blockIdx.x * topk; + + // get our assigned thread score; each warp represents one expert group + float score = + expertSelected ? static_cast(scores[scoreIdx]) : invalidScoreFloat; + auto scoreSigmoid = apply_scoring(score); + // write the sigmoid score to shared for later use + if (expertSelected) { + smemScoreSigmoid[threadExpert] = scoreSigmoid; + } + + // get the score with bias + // note that with invalid values, because sigmoid is < 1 and bias is -1, + // we must get a negative value, which is smaller than any valid value + auto scoreBias = float{scoreSigmoid + float{biasVal}}; + + if (expertSelected) { + smemScoreBias[threadExpert] = scoreBias; + } + + // registers for top group score reduction + float topExpGroupScores[NumTopGroupScores]; + [[maybe_unused]] int32_t topExpGroupIdx[NumTopGroupScores]; + float topGroups[MaxNumTopGroups]; // bound of numGroup + int32_t topGroupIdx[MaxNumTopGroups]; + float expertScoreGroup[MaxNumTopGroups]; + int32_t expertIdxGroup[MaxNumTopGroups]; + float topScores[MaxNumTopExperts]; // bound of topk + int32_t topExperts[MaxNumTopExperts]; + + if constexpr (UseGroups) { + reduce_topk::reduceTopK(warp, topExpGroupScores, topExpGroupIdx, scoreBias, + threadExpert, + /* minValue */ invalidScoreFloat); + + // get the final group score and write it to shared + if (warp.thread_rank() == 0) { + auto groupScore = topExpGroupScores[0] + topExpGroupScores[1]; + smemGroupScores[warpIdx] = groupScore; + } + } + + // make group scores available to all warps + __syncthreads(); + + if constexpr (UseGroups) { + if (warpIdx == 0) { + // a single warp performs the selection of top groups, and goes on to + // select the final experts + float groupScore = + laneIdx < numGroup ? smemGroupScores[laneIdx] : invalidScoreFloat; + + reduce_topk::reduceTopK(warp, topGroups, topGroupIdx, groupScore, laneIdx, + /* minValue */ invalidScoreFloat); + // final expert selection: get relevant indexes and scores from shared +#pragma unroll + for (int ii = 0; ii < MaxNumTopGroups; ++ii) { // bound of numGroup + auto groupIdx = topGroupIdx[ii]; + expertIdxGroup[ii] = groupIdx * numExpertsPerGroup + laneIdx; + + expertScoreGroup[ii] = (ii < topkGroup) && expertSelected + ? smemScoreBias[expertIdxGroup[ii]] + : invalidScoreFloat; + } + + reduce_topk::reduceTopK(warp, topScores, topExperts, expertScoreGroup, + expertIdxGroup, /* minValue */ invalidScoreFloat, + topk); + } + } else if constexpr (MaxNumExperts > MaxNumExpertsUnit) { + // without groups, and the expert number is larger than MaxNumExpertsUnit, + // we need to use multiple warps to calculate the intermediate topk results + + int constexpr NumExpertWarps = (MaxNumExperts - 1) / MaxNumExpertsUnit + 1; + int constexpr NumInterTopK = NumExpertWarps * MaxNumTopExperts; + __shared__ float + __attribute((aligned(128))) smemInterTopScores[NumInterTopK]; + __shared__ int32_t + __attribute((aligned(128))) smemInterTopExperts[NumInterTopK]; + if (warpIdx < NumExpertWarps) { + int offset = warpIdx * WARP_SIZE * MaxNumTopGroups; +#pragma unroll + for (int ii = 0; ii < MaxNumTopGroups; ++ii) { + auto expertIdx = ii * WARP_SIZE + laneIdx; + expertIdxGroup[ii] = offset + expertIdx; + expertScoreGroup[ii] = offset + expertIdx < numExperts + ? smemScoreBias[offset + expertIdx] + : invalidScoreFloat; + } + reduce_topk::reduceTopK(warp, topScores, topExperts, expertScoreGroup, + expertIdxGroup, + /* minValue */ invalidScoreFloat, topk); + + if (laneIdx < topk) { + smemInterTopScores[warpIdx * MaxNumTopExperts + laneIdx] = + topScores[laneIdx]; + smemInterTopExperts[warpIdx * MaxNumTopExperts + laneIdx] = + topExperts[laneIdx]; + } else if (laneIdx >= topk && laneIdx < MaxNumTopExperts) { + smemInterTopScores[warpIdx * MaxNumTopExperts + laneIdx] = + invalidScoreFloat; + smemInterTopExperts[warpIdx * MaxNumTopExperts + laneIdx] = + MaxNumExperts - 1; + } + } + __syncthreads(); + if (warpIdx == 0) { + int constexpr NumInterTopKPerThread = (NumInterTopK - 1) / WARP_SIZE + 1; + float intermediateScore[NumInterTopKPerThread]; + int32_t intermediateExpert[NumInterTopKPerThread]; + for (int i = laneIdx; i < NumInterTopKPerThread * WARP_SIZE; + i += WARP_SIZE) { + int ii = i / WARP_SIZE; + if (i < NumInterTopK) { + intermediateScore[ii] = smemInterTopScores[i]; + intermediateExpert[ii] = smemInterTopExperts[i]; + } else { + intermediateScore[ii] = invalidScoreFloat; + intermediateExpert[ii] = MaxNumExperts - 1; + } + } + reduce_topk::reduceTopK(warp, topScores, topExperts, intermediateScore, + intermediateExpert, + /* minValue */ invalidScoreFloat, topk); + } + } else { + // without groups, and the expert number is smaller than MaxNumExpertsUnit + // each thread just takes `MaxNumTopGroups` experts + if (warpIdx == 0) { +#pragma unroll + for (int ii = 0; ii < MaxNumTopGroups; ++ii) { + auto expertIdx = ii * WARP_SIZE + laneIdx; + expertIdxGroup[ii] = expertIdx; + expertScoreGroup[ii] = expertIdx < numExperts ? smemScoreBias[expertIdx] + : invalidScoreFloat; + } + reduce_topk::reduceTopK(warp, topScores, topExperts, expertScoreGroup, + expertIdxGroup, + /* minValue */ invalidScoreFloat, topk); + } + } + + if (warpIdx == 0) { + // determine our lane's expert index and write to output + int32_t expertIdx = + laneIdx < topk ? topExperts[laneIdx] : MaxNumExperts - 1; + float scoreNorm = laneIdx < topk ? smemScoreSigmoid[expertIdx] : 0.F; + float finalScore = static_cast(scoreNorm * routedScalingFactor); + // norm the value + if (renormalize) { + auto redNorm = cg::reduce(warp, scoreNorm, cg::plus{}); + finalScore /= (redNorm + 1e-20); + } + // store the topk scores and experts to output + if (laneIdx < topk) { + topkValues[laneIdx] = finalScore; + topkIndices[laneIdx] = expertIdx; + } + } + +#if (defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)) + cudaTriggerProgrammaticLaunchCompletion(); +#endif +} + +template void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices, BiasT const* bias, int64_t const num_tokens, int64_t const num_experts, int64_t const n_group, int64_t const topk_group, int64_t const topk, bool const renormalize, double const routed_scaling_factor, - int const scoring_func, bool enable_pdl = false, - cudaStream_t const stream = 0) { + bool enable_pdl = false, cudaStream_t const stream = 0) { cudaLaunchConfig_t config; - // One block per token; one warp per group. - config.gridDim = static_cast(num_tokens); - config.blockDim = static_cast(n_group) * WARP_SIZE; - // Dynamic shared memory: WarpSelect staging + per-group topk buffers. - int32_t const num_warps = static_cast(n_group); - size_t const val_bytes = - static_cast(num_warps) * WARP_SIZE * sizeof(T); - size_t const val_bytes_aligned = - warp_topk::round_up_to_multiple_of<256>(val_bytes); - size_t const idx_bytes = - static_cast(num_warps) * WARP_SIZE * sizeof(int32_t); - size_t const internal_bytes = val_bytes_aligned + idx_bytes; - size_t const extra_bytes = 16 + static_cast(n_group) * sizeof(T); - config.dynamicSmemBytes = internal_bytes + extra_bytes; config.stream = stream; cudaLaunchAttribute attrs[1]; attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl; config.numAttrs = 1; config.attrs = attrs; - auto const sf = static_cast(scoring_func); - switch (sf) { - case SCORING_NONE: { - auto* kernel_instance = - &grouped_topk_fused_kernel; - cudaLaunchKernelEx(&config, kernel_instance, scores, topk_values, - topk_indices, bias, num_tokens, num_experts, n_group, - topk_group, topk, renormalize, routed_scaling_factor); - return; + + // Check if we can use the optimized + // grouped_topk_fused_small_expert_count_kernel + bool const is_single_group = + (n_group == 1) && (topk_group == 1) && + (num_experts <= MaxSupportedExpertCount) && + (topk <= DefaultMaxNumTopExperts || topk == MaxSupportedTopExperts); + + int64_t const experts_per_group = num_experts / n_group; + bool const is_multi_group = + (n_group > 1) && (num_experts <= NumDeepseekExperts) && + (experts_per_group <= WARP_SIZE) && + (experts_per_group * topk_group <= MaxNumExpertsUnit) && + (topk <= DefaultMaxNumTopExperts) && (topk_group <= MaxNumTopGroups); + + if (is_single_group || is_multi_group) { + auto* kernel_instance = + &grouped_topk_fused_small_expert_count_kernel; + int num_threads = NumDeepseekExperts; + if (is_single_group) { + // Special case for Nemotron, which selects top 22 from 512 experts, and 1 + // group only. + if (num_experts == NumNemotronExperts && n_group == 1 && + topk == MaxSupportedTopExperts) { + kernel_instance = &grouped_topk_fused_small_expert_count_kernel< + T, BiasT, IdxT, SF, NumNemotronExperts, false, + MaxSupportedTopExperts>; + num_threads = NumNemotronExperts; + } else if (num_experts > NumKimiK2Experts && + num_experts <= MaxSupportedExpertCount) { + kernel_instance = &grouped_topk_fused_small_expert_count_kernel< + T, BiasT, IdxT, SF, MaxSupportedExpertCount, false>; + num_threads = MaxSupportedExpertCount; + } else if (num_experts > MaxNumExpertsUnit && + num_experts <= NumKimiK2Experts) { + kernel_instance = &grouped_topk_fused_small_expert_count_kernel< + T, BiasT, IdxT, SF, NumKimiK2Experts, false>; + num_threads = NumKimiK2Experts; + } else { + kernel_instance = &grouped_topk_fused_small_expert_count_kernel< + T, BiasT, IdxT, SF, MaxNumExpertsUnit, false>; + num_threads = MaxNumExpertsUnit; + } } - case SCORING_SIGMOID: { - auto* kernel_instance = - &grouped_topk_fused_kernel; - cudaLaunchKernelEx(&config, kernel_instance, scores, topk_values, - topk_indices, bias, num_tokens, num_experts, n_group, - topk_group, topk, renormalize, routed_scaling_factor); - return; - } - default: - // should be guarded by higher level checks. - TORCH_CHECK(false, "Unsupported scoring_func in invokeNoAuxTc"); + config.gridDim = num_tokens; + config.blockDim = num_threads; + config.dynamicSmemBytes = 0; + cudaLaunchKernelEx(&config, kernel_instance, scores, topk_values, + topk_indices, bias, num_tokens, n_group, topk_group, + topk, num_experts, num_experts / n_group, renormalize, + routed_scaling_factor); + } else { + auto* kernel_instance = &grouped_topk_fused_kernel; + // One block per token; one warp per group. + config.gridDim = static_cast(num_tokens); + config.blockDim = static_cast(n_group) * WARP_SIZE; + // Dynamic shared memory: WarpSelect staging + per-group topk buffers. + int32_t const num_warps = static_cast(n_group); + size_t const val_bytes = + static_cast(num_warps) * WARP_SIZE * sizeof(T); + size_t const val_bytes_aligned = + warp_topk::round_up_to_multiple_of<256>(val_bytes); + size_t const idx_bytes = + static_cast(num_warps) * WARP_SIZE * sizeof(int32_t); + size_t const internal_bytes = val_bytes_aligned + idx_bytes; + size_t const extra_bytes = 16 + static_cast(n_group) * sizeof(T); + config.dynamicSmemBytes = internal_bytes + extra_bytes; + cudaLaunchKernelEx(&config, kernel_instance, scores, topk_values, + topk_indices, bias, num_tokens, num_experts, n_group, + topk_group, topk, renormalize, routed_scaling_factor); } } -#define INSTANTIATE_NOAUX_TC(T, BiasT, IdxT) \ - template void invokeNoAuxTc( \ +#define INSTANTIATE_NOAUX_TC(T, BiasT, IdxT, SF) \ + template void invokeNoAuxTc( \ T * scores, float* topk_values, IdxT* topk_indices, BiasT const* bias, \ int64_t const num_tokens, int64_t const num_experts, \ int64_t const n_group, int64_t const topk_group, int64_t const topk, \ bool const renormalize, double const routed_scaling_factor, \ - int const scoring_func, bool enable_pdl, cudaStream_t const stream); + bool enable_pdl, cudaStream_t const stream); -INSTANTIATE_NOAUX_TC(float, float, int32_t); -INSTANTIATE_NOAUX_TC(float, half, int32_t); -INSTANTIATE_NOAUX_TC(float, __nv_bfloat16, int32_t); -INSTANTIATE_NOAUX_TC(half, float, int32_t); -INSTANTIATE_NOAUX_TC(half, half, int32_t); -INSTANTIATE_NOAUX_TC(half, __nv_bfloat16, int32_t); -INSTANTIATE_NOAUX_TC(__nv_bfloat16, float, int32_t); -INSTANTIATE_NOAUX_TC(__nv_bfloat16, half, int32_t); -INSTANTIATE_NOAUX_TC(__nv_bfloat16, __nv_bfloat16, int32_t); +INSTANTIATE_NOAUX_TC(float, float, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(float, half, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(float, __nv_bfloat16, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(half, float, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(half, half, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(half, __nv_bfloat16, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(__nv_bfloat16, float, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(__nv_bfloat16, half, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(__nv_bfloat16, __nv_bfloat16, int32_t, SCORING_SIGMOID); +INSTANTIATE_NOAUX_TC(float, float, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(float, half, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(float, __nv_bfloat16, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(half, float, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(half, half, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(half, __nv_bfloat16, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(__nv_bfloat16, float, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(__nv_bfloat16, half, int32_t, SCORING_NONE); +INSTANTIATE_NOAUX_TC(__nv_bfloat16, __nv_bfloat16, int32_t, SCORING_NONE); } // end namespace moe } // namespace vllm @@ -762,46 +1033,53 @@ std::tuple grouped_topk( {num_tokens, topk}, torch::dtype(torch::kInt32).device(torch::kCUDA)); auto stream = c10::cuda::getCurrentCUDAStream(scores.get_device()); + auto const sf = static_cast(scoring_func); -#define LAUNCH_KERNEL(T, IdxT) \ - do { \ - switch (bias_type) { \ - case torch::kFloat16: \ - vllm::moe::invokeNoAuxTc( \ - reinterpret_cast(scores.mutable_data_ptr()), \ - reinterpret_cast(topk_values.mutable_data_ptr()), \ - reinterpret_cast(topk_indices.mutable_data_ptr()), \ - reinterpret_cast(bias.data_ptr()), num_tokens, \ - num_experts, n_group, topk_group, topk, renormalize, \ - routed_scaling_factor, static_cast(scoring_func), false, \ - stream); \ - break; \ - case torch::kFloat32: \ - vllm::moe::invokeNoAuxTc( \ - reinterpret_cast(scores.mutable_data_ptr()), \ - reinterpret_cast(topk_values.mutable_data_ptr()), \ - reinterpret_cast(topk_indices.mutable_data_ptr()), \ - reinterpret_cast(bias.data_ptr()), num_tokens, \ - num_experts, n_group, topk_group, topk, renormalize, \ - routed_scaling_factor, static_cast(scoring_func), false, \ - stream); \ - break; \ - case torch::kBFloat16: \ - vllm::moe::invokeNoAuxTc( \ - reinterpret_cast(scores.mutable_data_ptr()), \ - reinterpret_cast(topk_values.mutable_data_ptr()), \ - reinterpret_cast(topk_indices.mutable_data_ptr()), \ - reinterpret_cast<__nv_bfloat16 const*>(bias.data_ptr()), \ - num_tokens, num_experts, n_group, topk_group, topk, renormalize, \ - routed_scaling_factor, static_cast(scoring_func), false, \ - stream); \ - break; \ - default: \ - throw std::invalid_argument( \ - "Invalid bias dtype, only supports float16, float32, and " \ - "bfloat16"); \ - break; \ - } \ +#define LAUNCH_KERNEL_SF(T, BiasT, IdxT) \ + do { \ + switch (sf) { \ + case vllm::moe::SCORING_NONE: \ + vllm::moe::invokeNoAuxTc( \ + reinterpret_cast(scores.mutable_data_ptr()), \ + reinterpret_cast(topk_values.mutable_data_ptr()), \ + reinterpret_cast(topk_indices.mutable_data_ptr()), \ + reinterpret_cast(bias.data_ptr()), num_tokens, \ + num_experts, n_group, topk_group, topk, renormalize, \ + routed_scaling_factor, false, stream); \ + break; \ + case vllm::moe::SCORING_SIGMOID: \ + vllm::moe::invokeNoAuxTc( \ + reinterpret_cast(scores.mutable_data_ptr()), \ + reinterpret_cast(topk_values.mutable_data_ptr()), \ + reinterpret_cast(topk_indices.mutable_data_ptr()), \ + reinterpret_cast(bias.data_ptr()), num_tokens, \ + num_experts, n_group, topk_group, topk, renormalize, \ + routed_scaling_factor, false, stream); \ + break; \ + default: \ + throw std::invalid_argument("Unsupported scoring_func"); \ + break; \ + } \ + } while (0) + +#define LAUNCH_KERNEL(T, IdxT) \ + do { \ + switch (bias_type) { \ + case torch::kFloat16: \ + LAUNCH_KERNEL_SF(T, half, IdxT); \ + break; \ + case torch::kFloat32: \ + LAUNCH_KERNEL_SF(T, float, IdxT); \ + break; \ + case torch::kBFloat16: \ + LAUNCH_KERNEL_SF(T, __nv_bfloat16, IdxT); \ + break; \ + default: \ + throw std::invalid_argument( \ + "Invalid bias dtype, only supports float16, float32, and " \ + "bfloat16"); \ + break; \ + } \ } while (0) switch (data_type) { @@ -824,5 +1102,6 @@ std::tuple grouped_topk( break; } #undef LAUNCH_KERNEL +#undef LAUNCH_KERNEL_SF return {topk_values, topk_indices}; } diff --git a/csrc/moe/moeTopKFuncs.cuh b/csrc/moe/moeTopKFuncs.cuh new file mode 100644 index 00000000000..70e21cf8773 --- /dev/null +++ b/csrc/moe/moeTopKFuncs.cuh @@ -0,0 +1,257 @@ +/* + * Adapted from + * https://github.com/NVIDIA/TensorRT-LLM/blob/v1.3.0rc2/cpp/tensorrt_llm/kernels/moeTopKFuncs.cuh + * Copyright (c) 2026, The vLLM team. + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include + +namespace vllm { +namespace moe { +namespace reduce_topk { +namespace cg = cooperative_groups; +static constexpr int kWARP_SIZE = 32; + +template +struct TopKRedType { + using T = T_; + static_assert( + std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v, + "Top K reduction only implemented for int, float, float16 and bfloat16"); + + using TypeCmp = std::conditional_t; + using IdxT = std::conditional_t; + + static constexpr int kMoveBits = (sizeof(T) == 4) ? 32 : 16; + static constexpr int kMaxIdx = 65535; + TypeCmp compValIdx; + + static __host__ __device__ inline TypeCmp makeCmpVal(T val, int32_t idx = 0) { + auto valueBits = cub::Traits::TwiddleIn( + reinterpret_cast::UnsignedBits&>(val)); + TypeCmp compactTmp = valueBits; + compactTmp = (compactTmp << kMoveBits) | (0xFFFF & (kMaxIdx - idx)); + // Use 65535 minus idx to give higher priority to elements with smaller + // indices. + return compactTmp; + } + + static __host__ __device__ void unpack(T& value, int32_t& index, + TypeCmp cmp) { + // Since “65535-idx” is always smaller than 65536 and positive, we can + // directly use it as the lower 16 bits + index = kMaxIdx - static_cast((cmp & 0xFFFF)); + + auto compactTmp = cmp >> kMoveBits; + auto valueBits = cub::Traits::TwiddleOut( + reinterpret_cast::UnsignedBits&>(compactTmp)); + value = reinterpret_cast(valueBits); + } + + __host__ __device__ TopKRedType() = default; + + __host__ __device__ TopKRedType(T val, int32_t idx) + : compValIdx(makeCmpVal(val, idx)) {} + + __host__ __device__ operator TypeCmp() const noexcept { return compValIdx; } + + __device__ inline TypeCmp reduce( + cg::thread_block_tile const& warp) { + return cg::reduce(warp, compValIdx, cg::greater{}); + } +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +template +struct TopKIdx { + // by default, empty +}; + +template +struct TopKIdx { + static constexpr int K = K_; + int32_t val[K]; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////// + +#define TOPK_SWAP(I, J) \ + { \ + auto pairMin = min(topK[I].compValIdx, topK[J].compValIdx); \ + auto pairMax = max(topK[I].compValIdx, topK[J].compValIdx); \ + topK[I].compValIdx = pairMax; \ + topK[J].compValIdx = pairMin; \ + } + +template +struct Sort; + +template +struct Sort<1, RedType> { + static __device__ void run(RedType* topK) {} +}; + +template +struct Sort<2, RedType> { + static __device__ void run(RedType* topK) { TOPK_SWAP(0, 1); } +}; + +template +struct Sort<3, RedType> { + static __device__ void run(RedType* topK) { + TOPK_SWAP(0, 1); + TOPK_SWAP(1, 2); + TOPK_SWAP(0, 1); + } +}; + +template +struct Sort<4, RedType> { + static __device__ void run(RedType* topK) { + TOPK_SWAP(0, 2); + TOPK_SWAP(1, 3); + TOPK_SWAP(0, 1); + TOPK_SWAP(2, 3); + TOPK_SWAP(1, 2); + } +}; + +template +__forceinline__ __device__ void reduceTopK( + cg::thread_block_tile const& warp, Type (&out)[K], + int32_t (&outIdx)[K], Type value, int32_t idx, Type const minValue, + int actualK = K) { + static_assert(K > 0, "Top K must have K > 0"); + static_assert(K < kWARP_SIZE, "Top K must have K < kWARP_SIZE"); + using RedType = TopKRedType; + RedType topK{value, idx}; + typename RedType::TypeCmp packedMax{}; +#pragma unroll + for (int kk = 0; kk < actualK; ++kk) { + topK = + kk > 0 && packedMax == topK.compValIdx ? RedType{minValue, idx} : topK; + // get the next largest value + packedMax = topK.reduce(warp); + RedType::unpack(out[kk], outIdx[kk], packedMax); + } +}; + +template +__device__ void reduceTopKFunc(cg::thread_block_tile const& warp, + Type (&out)[K], int32_t (&outIdx)[K], + Type (&value)[N], int32_t (&idx)[N], + Type minValue, int actualK = K) { + static_assert(K > 0, "Top K must have K > 0"); + static_assert(K < kWARP_SIZE, "Top K must have K < kWARP_SIZE"); + static_assert(N > 0, "Top K must have N > 0"); + static_assert(N < 5, + "Only support candidates number less than or equal to 128"); + using RedType = TopKRedType; + RedType topK[N]; +#pragma unroll + for (int nn = 0; nn < N; ++nn) { + topK[nn] = RedType{value[nn], idx[nn]}; + } + + if constexpr (!IsSorted) { + Sort::run(topK); + } + typename RedType::TypeCmp packedMax{}; +#pragma unroll + for (int kk = 0; kk < actualK; ++kk) { + bool update = kk > 0 && packedMax == topK[0].compValIdx; +#pragma unroll + for (int nn = 0; nn < N; ++nn) { + topK[nn] = update && nn == N - 1 ? RedType{minValue, idx[nn]} + : update ? topK[nn + 1] + : topK[nn]; + } + // get the next largest value + packedMax = topK[0].reduce(warp); + RedType::unpack(out[kk], outIdx[kk], packedMax); + } +}; + +template +__forceinline__ __device__ void reduceTopK( + cg::thread_block_tile const& warp, Type (&out)[K], + int32_t (&outIdx)[K], Type (&value)[N], int32_t (&idx)[N], + Type const minValue, int actualK = K) { + static_assert(K > 0, "Top K must have K > 0"); + static_assert(K < kWARP_SIZE, "Top K must have K < kWARP_SIZE"); + static_assert(N > 0, "Top K must have N > 0"); + static_assert( + N <= 16, + "Only support candidates number less than or equal to 16*32=512"); + static_assert(N <= 4 || N % 4 == 0, + "Only support candidates number is a multiple of 4*32=128 or " + "less than or equal to 4"); + using RedType = TopKRedType; + + if constexpr (N <= 4) { + reduceTopKFunc(warp, out, outIdx, value, idx, minValue, + actualK); + } else { + constexpr int numLoops = N / 4; + constexpr int numResults = (numLoops * K - 1) / kWARP_SIZE + 1; + + Type topKBufferValue[numResults]; + int32_t topKBufferIdx[numResults]; + int32_t laneIdx = threadIdx.x % kWARP_SIZE; + + for (int ii = 0; ii < numResults; ++ii) { + topKBufferValue[ii] = minValue; + topKBufferIdx[ii] = ii * kWARP_SIZE - 1; + } + for (int loop = 0; loop < numLoops; ++loop) { + int start = loop * 4; + Type topKValue[K]; + int32_t topKIdx[K]; + Type inValue[4]; + int32_t inIdx[4]; + for (int i = 0; i < 4; ++i) { + inValue[i] = value[start + i]; + inIdx[i] = idx[start + i]; + } + reduceTopKFunc(warp, topKValue, topKIdx, inValue, inIdx, + minValue, actualK); + int inOffset = laneIdx % K; + if (laneIdx >= loop * K && laneIdx < (loop + 1) * K) { + topKBufferValue[0] = topKValue[inOffset]; + topKBufferIdx[0] = topKIdx[inOffset]; + } + if (loop == numLoops - 1 && (laneIdx < (numLoops * K - kWARP_SIZE))) { + topKBufferValue[1] = topKValue[inOffset]; + topKBufferIdx[1] = topKIdx[inOffset]; + } + } + + reduceTopKFunc(warp, out, outIdx, topKBufferValue, + topKBufferIdx, minValue, actualK); + } +}; + +#undef TOPK_SWAP + +} // namespace reduce_topk +} // namespace moe +} // namespace vllm diff --git a/tests/kernels/moe/test_grouped_topk.py b/tests/kernels/moe/test_grouped_topk.py index 2a974206d1d..70c7285acb2 100644 --- a/tests/kernels/moe/test_grouped_topk.py +++ b/tests/kernels/moe/test_grouped_topk.py @@ -8,6 +8,7 @@ Run `pytest tests/kernels/moe/test_grouped_topk.py`. import pytest import torch +import vllm.model_executor.layers.batch_invariant as batch_invariant from vllm.config import ( CompilationConfig, VllmConfig, @@ -27,11 +28,17 @@ from vllm.utils.torch_utils import set_random_seed ) @pytest.mark.parametrize("n_token", [1, 33, 64]) @pytest.mark.parametrize("n_hidden", [1024, 2048]) -@pytest.mark.parametrize("n_expert", [16]) -@pytest.mark.parametrize("topk", [2]) +@pytest.mark.parametrize( + "n_expert,topk,num_expert_group,topk_group", + [ + (16, 2, 8, 2), + (128, 2, 8, 2), + (256, 8, 8, 4), + (384, 8, 1, 1), + (512, 22, 1, 1), + ], +) @pytest.mark.parametrize("renormalize", [True, False]) -@pytest.mark.parametrize("num_expert_group", [8]) -@pytest.mark.parametrize("topk_group", [2]) @pytest.mark.parametrize("scoring_func", ["softmax", "sigmoid"]) @pytest.mark.parametrize("routed_scaling_factor", [1.0, 2.5]) @pytest.mark.parametrize("input_dtype", [torch.bfloat16, torch.float32]) @@ -42,9 +49,9 @@ def test_grouped_topk( n_hidden: int, n_expert: int, topk: int, - renormalize: bool, num_expert_group: int, topk_group: int, + renormalize: bool, scoring_func: str, routed_scaling_factor: float, input_dtype: torch.dtype, @@ -62,6 +69,7 @@ def test_grouped_topk( with set_current_vllm_config(vllm_config), monkeypatch.context() as m: m.setenv("VLLM_USE_FUSED_MOE_GROUPED_TOPK", "0") + m.setattr(batch_invariant, "VLLM_BATCH_INVARIANT", True) grouped_topk = GroupedTopk( topk=topk, renormalize=renormalize, @@ -89,8 +97,7 @@ def test_grouped_topk( e_score_correction_bias=e_score_correction_bias, ) - if renormalize: - torch.testing.assert_close( - baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0 - ) + torch.testing.assert_close( + baseline_topk_weights, test_topk_weights, atol=2e-2, rtol=0 + ) torch.testing.assert_close(baseline_topk_ids, test_topk_ids, atol=0, rtol=0) From 59965affbd6e652a3c8ed229b66ef34a681e5693 Mon Sep 17 00:00:00 2001 From: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Date: Fri, 20 Feb 2026 17:54:27 +0400 Subject: [PATCH 26/74] [BUGFIX] Fix `_dummy_run` missing `prepare_inputs_event` synchronization (#34866) Signed-off-by: Vadim Gimpelson --- vllm/v1/worker/gpu_model_runner.py | 57 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index ba1428c42ee..51c4f5805e5 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -4771,34 +4771,39 @@ class GPUModelRunner( ubatch_slices=ubatch_slices_padded, ) - # If force_attention is True, we always capture attention. Otherwise, - # it only happens for cudagraph_runtime_mode=FULL. - if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL: - if create_mixed_batch: - # In the mixed batch mode (used for FI warmup), we use - # shorter sequence lengths to run faster. - # TODO(luka) better system for describing dummy batches - seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] - else: - seq_lens = max_query_len # type: ignore[assignment] - self.seq_lens.np[:num_reqs] = seq_lens - self.seq_lens.np[num_reqs:] = 0 - self.seq_lens.copy_to_gpu() + # _dummy_run shares pinned CPU buffers (seq_lens, query_start_loc, + # etc.) with execute_model. It must participate in the same event + # protocol so that back-to-back dummy/real steps don't overwrite + # pinned memory while a prior non_blocking H2D DMA is still reading. + with self.synchronize_input_prep(): + # If force_attention is True, we always capture attention. + # Otherwise, it only happens for cudagraph_runtime_mode=FULL. + if force_attention or cudagraph_runtime_mode == CUDAGraphMode.FULL: + if create_mixed_batch: + # In the mixed batch mode (used for FI warmup), we use + # shorter sequence lengths to run faster. + # TODO(luka) better system for describing dummy batches + seq_lens = [1] * num_decode_tokens + [num_prefill_tokens + 1] + else: + seq_lens = max_query_len # type: ignore[assignment] + self.seq_lens.np[:num_reqs] = seq_lens + self.seq_lens.np[num_reqs:] = 0 + self.seq_lens.copy_to_gpu() - cum_num_tokens, _ = self._get_cumsum_and_arange(num_scheduled_tokens) - self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens - self.query_start_loc.copy_to_gpu() + cum_num_tokens, _ = self._get_cumsum_and_arange(num_scheduled_tokens) + self.query_start_loc.np[1 : num_reqs + 1] = cum_num_tokens + self.query_start_loc.copy_to_gpu() - pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL - attn_metadata, _ = self._build_attention_metadata( - num_tokens=num_tokens_unpadded, - num_tokens_padded=num_tokens_padded if pad_attn else None, - num_reqs=num_reqs_padded, - max_query_len=max_query_len, - ubatch_slices=ubatch_slices_padded if pad_attn else ubatch_slices, - for_cudagraph_capture=is_graph_capturing, - slot_mappings=slot_mappings_by_group, - ) + pad_attn = cudagraph_runtime_mode == CUDAGraphMode.FULL + attn_metadata, _ = self._build_attention_metadata( + num_tokens=num_tokens_unpadded, + num_tokens_padded=num_tokens_padded if pad_attn else None, + num_reqs=num_reqs_padded, + max_query_len=max_query_len, + ubatch_slices=(ubatch_slices_padded if pad_attn else ubatch_slices), + for_cudagraph_capture=is_graph_capturing, + slot_mappings=slot_mappings_by_group, + ) with self.maybe_dummy_run_with_lora( self.lora_config, From f9ac19204f0c4c3041d0afbe7d5eb4d63e73f15c Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Fri, 20 Feb 2026 22:19:23 +0800 Subject: [PATCH 27/74] [V0 Deprecation] Remove unused MM placeholders in request output (#34944) Signed-off-by: DarkLight1337 --- vllm/outputs.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/vllm/outputs.py b/vllm/outputs.py index 48f8e9dc040..2c71d2afb1b 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -13,7 +13,6 @@ from typing_extensions import TypeVar from vllm.logger import init_logger from vllm.logprobs import PromptLogprobs, SampleLogprobs from vllm.lora.request import LoRARequest -from vllm.multimodal.inputs import MultiModalPlaceholderDict from vllm.v1.metrics.stats import RequestStateStats logger = init_logger(__name__) @@ -121,7 +120,6 @@ class RequestOutput: encoder_prompt_token_ids: list[int] | None = None, num_cached_tokens: int | None = None, *, - multi_modal_placeholders: MultiModalPlaceholderDict | None = None, kv_transfer_params: dict[str, Any] | None = None, # Forward compatibility, code that uses args added in new release can # still run with older versions of vLLM without breaking. @@ -134,7 +132,6 @@ class RequestOutput: self.request_id = request_id self.prompt = prompt self.prompt_token_ids = prompt_token_ids - self.multi_modal_placeholders = multi_modal_placeholders or {} self.prompt_logprobs = prompt_logprobs self.outputs = outputs self.finished = finished @@ -187,8 +184,7 @@ class RequestOutput: f"finished={self.finished}, " f"metrics={self.metrics}, " f"lora_request={self.lora_request}, " - f"num_cached_tokens={self.num_cached_tokens}, " - f"multi_modal_placeholders={self.multi_modal_placeholders})" + f"num_cached_tokens={self.num_cached_tokens})" ) From ed31a020ee5e383a069a59750261a307bd8ddde4 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Fri, 20 Feb 2026 09:20:46 -0500 Subject: [PATCH 28/74] [Refactor] Extract Harmony streaming SSE event builders into streaming_events.py (#34909) Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Cyrus Leung --- vllm/entrypoints/openai/responses/serving.py | 884 +---------------- .../openai/responses/streaming_events.py | 897 ++++++++++++++++++ 2 files changed, 907 insertions(+), 874 deletions(-) create mode 100644 vllm/entrypoints/openai/responses/streaming_events.py diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index e40b6b8f088..4055095fd2f 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -2,36 +2,22 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import json import time import uuid from collections import deque from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence from contextlib import AsyncExitStack from copy import copy -from dataclasses import dataclass, replace +from dataclasses import replace from http import HTTPStatus from typing import Final import jinja2 from fastapi import Request from openai.types.responses import ( - ResponseCodeInterpreterCallCodeDeltaEvent, - ResponseCodeInterpreterCallCodeDoneEvent, - ResponseCodeInterpreterCallCompletedEvent, - ResponseCodeInterpreterCallInProgressEvent, - ResponseCodeInterpreterCallInterpretingEvent, - ResponseCodeInterpreterToolCallParam, ResponseContentPartAddedEvent, ResponseContentPartDoneEvent, - ResponseFunctionCallArgumentsDeltaEvent, - ResponseFunctionCallArgumentsDoneEvent, ResponseFunctionToolCall, - ResponseFunctionWebSearch, - ResponseMcpCallArgumentsDeltaEvent, - ResponseMcpCallArgumentsDoneEvent, - ResponseMcpCallCompletedEvent, - ResponseMcpCallInProgressEvent, ResponseOutputItem, ResponseOutputItemAddedEvent, ResponseOutputItemDoneEvent, @@ -43,13 +29,8 @@ from openai.types.responses import ( ResponseStatus, ResponseTextDeltaEvent, ResponseTextDoneEvent, - ResponseWebSearchCallCompletedEvent, - ResponseWebSearchCallInProgressEvent, - ResponseWebSearchCallSearchingEvent, - response_function_web_search, response_text_delta_event, ) -from openai.types.responses.response_output_item import McpCall from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob from openai.types.responses.response_reasoning_item import ( Content as ResponseReasoningTextContent, @@ -102,13 +83,17 @@ from vllm.entrypoints.openai.responses.protocol import ( ResponseCreatedEvent, ResponseInProgressEvent, ResponseInputOutputMessage, - ResponseReasoningPartAddedEvent, - ResponseReasoningPartDoneEvent, ResponsesRequest, ResponsesResponse, ResponseUsage, StreamingResponsesResponse, ) +from vllm.entrypoints.openai.responses.streaming_events import ( + HarmonyStreamingState, + emit_content_delta_events, + emit_previous_item_done_events, + emit_tool_action_events, +) from vllm.entrypoints.openai.responses.utils import ( construct_input_messages, construct_tool_dicts, @@ -129,23 +114,6 @@ from vllm.utils import random_uuid logger = init_logger(__name__) -@dataclass -class HarmonyStreamingState: - """Mutable state for harmony streaming event processing.""" - - current_content_index: int = -1 - current_output_index: int = 0 - current_item_id: str = "" - sent_output_item_added: bool = False - is_first_function_call_delta: bool = False - - def reset_for_new_item(self) -> None: - """Reset state when expecting a new output item.""" - self.current_output_index += 1 - self.sent_output_item_added = False - self.is_first_function_call_delta = False - - def _extract_allowed_tools_from_mcp_requests( tools: list[Tool], ) -> dict[str, list[str] | None]: @@ -817,26 +785,6 @@ class OpenAIServingResponses(OpenAIServing): self.response_store[response.id] = response return response - def _is_mcp_tool_by_namespace(self, recipient: str | None) -> bool: - """ - Determine if a tool call is an MCP tool based on recipient prefix. - - - Tools starting with "functions." are function calls - - Everything else is an MCP tool - """ - if recipient is None: - return False - - # Function calls have "functions." prefix - # Everything else is an MCP tool - return not recipient.startswith("functions.") - - _TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = { - "python": "code_interpreter", - "container": "container", - "browser": "web_search_preview", - } - def _topk_logprobs( self, logprobs: dict[int, SampleLogprob], @@ -1605,816 +1553,6 @@ class OpenAIServingResponses(OpenAIServing): ) ) - def _emit_function_call_done_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when a function call completes.""" - function_name = previous_item.recipient[len("functions.") :] - events = [] - events.append( - ResponseFunctionCallArgumentsDoneEvent( - type="response.function_call_arguments.done", - arguments=previous_item.content[0].text, - name=function_name, - item_id=state.current_item_id, - output_index=state.current_output_index, - sequence_number=-1, - ) - ) - function_call_item = ResponseFunctionToolCall( - type="function_call", - arguments=previous_item.content[0].text, - name=function_name, - item_id=state.current_item_id, - output_index=state.current_output_index, - sequence_number=-1, - call_id=f"fc_{random_uuid()}", - status="completed", - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=function_call_item, - ) - ) - return events - - def _emit_mcp_call_done_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when an MCP tool call completes.""" - server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get( - previous_item.recipient, previous_item.recipient - ) - events = [] - events.append( - ResponseMcpCallArgumentsDoneEvent( - type="response.mcp_call_arguments.done", - arguments=previous_item.content[0].text, - name=previous_item.recipient, - item_id=state.current_item_id, - output_index=state.current_output_index, - sequence_number=-1, - ) - ) - events.append( - ResponseMcpCallCompletedEvent( - type="response.mcp_call.completed", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=McpCall( - type="mcp_call", - arguments=previous_item.content[0].text, - name=previous_item.recipient, - id=state.current_item_id, - server_label=server_label, - status="completed", - ), - ) - ) - return events - - def _emit_reasoning_done_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when a reasoning (analysis) item completes.""" - content = ResponseReasoningTextContent( - text=previous_item.content[0].text, - type="reasoning_text", - ) - reasoning_item = ResponseReasoningItem( - type="reasoning", - content=[content], - status="completed", - id=state.current_item_id, - summary=[], - ) - events = [] - events.append( - ResponseReasoningTextDoneEvent( - type="response.reasoning_text.done", - item_id=state.current_item_id, - sequence_number=-1, - output_index=state.current_output_index, - content_index=state.current_content_index, - text=previous_item.content[0].text, - ) - ) - events.append( - ResponseReasoningPartDoneEvent( - type="response.reasoning_part.done", - sequence_number=-1, - item_id=state.current_item_id, - output_index=state.current_output_index, - content_index=state.current_content_index, - part=content, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=reasoning_item, - ) - ) - return events - - def _emit_text_output_done_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when a final text output item completes.""" - text_content = ResponseOutputText( - type="output_text", - text=previous_item.content[0].text, - annotations=[], - ) - events = [] - events.append( - ResponseTextDoneEvent( - type="response.output_text.done", - sequence_number=-1, - output_index=state.current_output_index, - content_index=state.current_content_index, - text=previous_item.content[0].text, - logprobs=[], - item_id=state.current_item_id, - ) - ) - events.append( - ResponseContentPartDoneEvent( - type="response.content_part.done", - sequence_number=-1, - item_id=state.current_item_id, - output_index=state.current_output_index, - content_index=state.current_content_index, - part=text_content, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=ResponseOutputMessage( - id=state.current_item_id, - type="message", - role="assistant", - content=[text_content], - status="completed", - ), - ) - ) - return events - - def _emit_previous_item_done_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit done events for the previous item when expecting a new start.""" - if previous_item.recipient is not None: - # Deal with tool call - if previous_item.recipient.startswith("functions."): - return self._emit_function_call_done_events(previous_item, state) - elif ( - self._is_mcp_tool_by_namespace(previous_item.recipient) - and state.current_item_id is not None - and state.current_item_id.startswith("mcp_") - ): - return self._emit_mcp_call_done_events(previous_item, state) - elif previous_item.channel == "analysis": - return self._emit_reasoning_done_events(previous_item, state) - elif previous_item.channel == "final": - return self._emit_text_output_done_events(previous_item, state) - return [] - - def _emit_final_channel_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for final channel text delta streaming.""" - events = [] - if not state.sent_output_item_added: - state.sent_output_item_added = True - state.current_item_id = f"msg_{random_uuid()}" - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=ResponseOutputMessage( - id=state.current_item_id, - type="message", - role="assistant", - content=[], - status="in_progress", - ), - ) - ) - state.current_content_index += 1 - events.append( - ResponseContentPartAddedEvent( - type="response.content_part.added", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - content_index=state.current_content_index, - part=ResponseOutputText( - type="output_text", - text="", - annotations=[], - logprobs=[], - ), - ) - ) - events.append( - ResponseTextDeltaEvent( - type="response.output_text.delta", - sequence_number=-1, - content_index=state.current_content_index, - output_index=state.current_output_index, - item_id=state.current_item_id, - delta=ctx.last_content_delta, - # TODO, use logprobs from ctx.last_request_output - logprobs=[], - ) - ) - return events - - def _emit_analysis_channel_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for analysis channel reasoning delta streaming.""" - events = [] - if not state.sent_output_item_added: - state.sent_output_item_added = True - state.current_item_id = f"msg_{random_uuid()}" - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=ResponseReasoningItem( - type="reasoning", - id=state.current_item_id, - summary=[], - status="in_progress", - ), - ) - ) - state.current_content_index += 1 - events.append( - ResponseReasoningPartAddedEvent( - type="response.reasoning_part.added", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - content_index=state.current_content_index, - part=ResponseReasoningTextContent( - text="", - type="reasoning_text", - ), - ) - ) - events.append( - ResponseReasoningTextDeltaEvent( - type="response.reasoning_text.delta", - item_id=state.current_item_id, - output_index=state.current_output_index, - content_index=state.current_content_index, - delta=ctx.last_content_delta, - sequence_number=-1, - ) - ) - return events - - def _emit_mcp_tool_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - recipient: str, - ) -> list[StreamingResponsesResponse]: - """Emit events for MCP tool delta streaming.""" - server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient) - events = [] - if not state.sent_output_item_added: - state.sent_output_item_added = True - state.current_item_id = f"mcp_{random_uuid()}" - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=McpCall( - type="mcp_call", - id=state.current_item_id, - name=recipient, - arguments="", - server_label=server_label, - status="in_progress", - ), - ) - ) - events.append( - ResponseMcpCallInProgressEvent( - type="response.mcp_call.in_progress", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseMcpCallArgumentsDeltaEvent( - type="response.mcp_call_arguments.delta", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - delta=ctx.last_content_delta, - ) - ) - return events - - def _emit_code_interpreter_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for code interpreter delta streaming.""" - events = [] - if not state.sent_output_item_added: - state.sent_output_item_added = True - state.current_item_id = f"tool_{random_uuid()}" - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=ResponseCodeInterpreterToolCallParam( - type="code_interpreter_call", - id=state.current_item_id, - code=None, - container_id="auto", - outputs=None, - status="in_progress", - ), - ) - ) - events.append( - ResponseCodeInterpreterCallInProgressEvent( - type="response.code_interpreter_call.in_progress", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseCodeInterpreterCallCodeDeltaEvent( - type="response.code_interpreter_call_code.delta", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - delta=ctx.last_content_delta, - ) - ) - return events - - def _emit_mcp_prefix_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for MCP prefix (mcp.*) delta streaming.""" - events = [] - if not state.sent_output_item_added: - state.sent_output_item_added = True - state.current_item_id = f"mcp_{random_uuid()}" - mcp_name = ctx.parser.current_recipient[len("mcp.") :] - - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=McpCall( - type="mcp_call", - id=state.current_item_id, - name=mcp_name, - arguments="", - server_label=mcp_name, - status="in_progress", - ), - ) - ) - events.append( - ResponseMcpCallInProgressEvent( - type="response.mcp_call.in_progress", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - - events.append( - ResponseMcpCallArgumentsDeltaEvent( - type="response.mcp_call_arguments.delta", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - delta=ctx.last_content_delta, - ) - ) - return events - - def _emit_content_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for content delta streaming based on channel type.""" - if not ctx.last_content_delta: - return [] - - if ( - ctx.parser.current_channel == "final" - and ctx.parser.current_recipient is None - ): - return self._emit_final_channel_delta_events(ctx, state) - elif ( - ctx.parser.current_channel == "analysis" - and ctx.parser.current_recipient is None - ): - return self._emit_analysis_channel_delta_events(ctx, state) - # built-in tools will be triggered on the analysis channel - # However, occasionally built-in tools will - # still be output to commentary. - elif ( - ctx.parser.current_channel == "commentary" - or ctx.parser.current_channel == "analysis" - ) and ctx.parser.current_recipient is not None: - recipient = ctx.parser.current_recipient - # Check for function calls first - they have their own event handling - if recipient.startswith("functions."): - return self._emit_function_call_delta_events(ctx, state) - is_mcp_tool = self._is_mcp_tool_by_namespace(recipient) - if is_mcp_tool: - return self._emit_mcp_tool_delta_events(ctx, state, recipient) - else: - return self._emit_code_interpreter_delta_events(ctx, state) - elif ( - ( - ctx.parser.current_channel == "commentary" - or ctx.parser.current_channel == "analysis" - ) - and ctx.parser.current_recipient is not None - and ctx.parser.current_recipient.startswith("mcp.") - ): - return self._emit_mcp_prefix_delta_events(ctx, state) - - return [] - - def _emit_browser_tool_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for browser tool calls (web search).""" - function_name = previous_item.recipient[len("browser.") :] - parsed_args = json.loads(previous_item.content[0].text) - action = None - - if function_name == "search": - action = response_function_web_search.ActionSearch( - type="search", - query=parsed_args["query"], - ) - elif function_name == "open": - action = response_function_web_search.ActionOpenPage( - type="open_page", - # TODO: translate to url - url=f"cursor:{parsed_args.get('cursor', '')}", - ) - elif function_name == "find": - action = response_function_web_search.ActionFind( - type="find", - pattern=parsed_args["pattern"], - # TODO: translate to url - url=f"cursor:{parsed_args.get('cursor', '')}", - ) - else: - raise ValueError(f"Unknown function name: {function_name}") - - state.current_item_id = f"tool_{random_uuid()}" - events = [] - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=response_function_web_search.ResponseFunctionWebSearch( - # TODO: generate a unique id for web search call - type="web_search_call", - id=state.current_item_id, - action=action, - status="in_progress", - ), - ) - ) - events.append( - ResponseWebSearchCallInProgressEvent( - type="response.web_search_call.in_progress", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseWebSearchCallSearchingEvent( - type="response.web_search_call.searching", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - # enqueue - events.append( - ResponseWebSearchCallCompletedEvent( - type="response.web_search_call.completed", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=ResponseFunctionWebSearch( - type="web_search_call", - id=state.current_item_id, - action=action, - status="completed", - ), - ) - ) - return events - - def _emit_mcp_tool_completion_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when an MCP tool completes during assistant action turn.""" - recipient = previous_item.recipient - server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient) - events = [] - events.append( - ResponseMcpCallArgumentsDoneEvent( - type="response.mcp_call_arguments.done", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - arguments=previous_item.content[0].text, - name=recipient, - ) - ) - events.append( - ResponseMcpCallCompletedEvent( - type="response.mcp_call.completed", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=McpCall( - type="mcp_call", - id=state.current_item_id, - name=recipient, - arguments=previous_item.content[0].text, - server_label=server_label, - status="completed", - ), - ) - ) - return events - - def _emit_code_interpreter_completion_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when code interpreter completes.""" - events = [] - events.append( - ResponseCodeInterpreterCallCodeDoneEvent( - type="response.code_interpreter_call_code.done", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - code=previous_item.content[0].text, - ) - ) - events.append( - ResponseCodeInterpreterCallInterpretingEvent( - type="response.code_interpreter_call.interpreting", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseCodeInterpreterCallCompletedEvent( - type="response.code_interpreter_call.completed", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=ResponseCodeInterpreterToolCallParam( - type="code_interpreter_call", - id=state.current_item_id, - code=previous_item.content[0].text, - container_id="auto", - outputs=[], - status="completed", - ), - ) - ) - return events - - def _emit_mcp_prefix_completion_events( - self, - previous_item, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events when an MCP prefix tool (mcp.*) completes.""" - mcp_name = previous_item.recipient[len("mcp.") :] - events = [] - events.append( - ResponseMcpCallArgumentsDoneEvent( - type="response.mcp_call_arguments.done", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - arguments=previous_item.content[0].text, - name=mcp_name, - ) - ) - events.append( - ResponseMcpCallCompletedEvent( - type="response.mcp_call.completed", - sequence_number=-1, - output_index=state.current_output_index, - item_id=state.current_item_id, - ) - ) - events.append( - ResponseOutputItemDoneEvent( - type="response.output_item.done", - sequence_number=-1, - output_index=state.current_output_index, - item=McpCall( - type="mcp_call", - id=state.current_item_id, - name=mcp_name, - arguments=previous_item.content[0].text, - server_label=mcp_name, - status="completed", - ), - ) - ) - return events - - def _emit_tool_action_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for tool action turn.""" - if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0: - return [] - - events = [] - previous_item = ctx.parser.messages[-1] - - # Handle browser tool - if ( - self.tool_server is not None - and self.tool_server.has_tool("browser") - and previous_item.recipient is not None - and previous_item.recipient.startswith("browser.") - ): - events.extend(self._emit_browser_tool_events(previous_item, state)) - - # Handle tool completion - if ( - self.tool_server is not None - and previous_item.recipient is not None - and state.current_item_id is not None - and state.sent_output_item_added - ): - recipient = previous_item.recipient - # Handle MCP prefix tool completion first - if recipient.startswith("mcp."): - events.extend( - self._emit_mcp_prefix_completion_events(previous_item, state) - ) - else: - # Handle other MCP tool and code interpreter completion - is_mcp_tool = self._is_mcp_tool_by_namespace( - recipient - ) and state.current_item_id.startswith("mcp_") - if is_mcp_tool: - events.extend( - self._emit_mcp_tool_completion_events(previous_item, state) - ) - else: - events.extend( - self._emit_code_interpreter_completion_events( - previous_item, state - ) - ) - - return events - - def _emit_function_call_delta_events( - self, - ctx: StreamingHarmonyContext, - state: HarmonyStreamingState, - ) -> list[StreamingResponsesResponse]: - """Emit events for developer function calls on commentary channel.""" - if not ( - ctx.parser.current_channel == "commentary" - and ctx.parser.current_recipient - and ctx.parser.current_recipient.startswith("functions.") - ): - return [] - - events = [] - if state.is_first_function_call_delta is False: - state.is_first_function_call_delta = True - fc_name = ctx.parser.current_recipient[len("functions.") :] - state.current_item_id = f"fc_{random_uuid()}" - tool_call_item = ResponseFunctionToolCall( - name=fc_name, - type="function_call", - id=state.current_item_id, - call_id=f"call_{random_uuid()}", - arguments="", - status="in_progress", - ) - events.append( - ResponseOutputItemAddedEvent( - type="response.output_item.added", - sequence_number=-1, - output_index=state.current_output_index, - item=tool_call_item, - ) - ) - # Always emit the delta (including on first call) - events.append( - ResponseFunctionCallArgumentsDeltaEvent( - item_id=state.current_item_id, - delta=ctx.last_content_delta, - output_index=state.current_output_index, - sequence_number=-1, - type="response.function_call_arguments.delta", - ) - ) - return events - async def _process_harmony_streaming_events( self, request: ResponsesRequest, @@ -2440,18 +1578,16 @@ class OpenAIServingResponses(OpenAIServing): if ctx.is_expecting_start(): if len(ctx.parser.messages) > 0: previous_item = ctx.parser.messages[-1] - for event in self._emit_previous_item_done_events( - previous_item, state - ): + for event in emit_previous_item_done_events(previous_item, state): yield _increment_sequence_number_and_return(event) state.reset_for_new_item() # Stream the output of a harmony message - for event in self._emit_content_delta_events(ctx, state): + for event in emit_content_delta_events(ctx, state): yield _increment_sequence_number_and_return(event) # Stream tool call outputs - for event in self._emit_tool_action_events(ctx, state): + for event in emit_tool_action_events(ctx, state, self.tool_server): yield _increment_sequence_number_and_return(event) async def responses_stream_generator( diff --git a/vllm/entrypoints/openai/responses/streaming_events.py b/vllm/entrypoints/openai/responses/streaming_events.py new file mode 100644 index 00000000000..cc89f8072be --- /dev/null +++ b/vllm/entrypoints/openai/responses/streaming_events.py @@ -0,0 +1,897 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Streaming SSE event builders for the Responses API. + +Pure functions that translate streaming state + delta data into +OpenAI Response API SSE events. Used by the streaming event +processors in serving.py. +""" + +import json +from dataclasses import dataclass +from typing import Final + +from openai.types.responses import ( + ResponseCodeInterpreterCallCodeDeltaEvent, + ResponseCodeInterpreterCallCodeDoneEvent, + ResponseCodeInterpreterCallCompletedEvent, + ResponseCodeInterpreterCallInProgressEvent, + ResponseCodeInterpreterCallInterpretingEvent, + ResponseCodeInterpreterToolCallParam, + ResponseContentPartAddedEvent, + ResponseContentPartDoneEvent, + ResponseFunctionCallArgumentsDeltaEvent, + ResponseFunctionCallArgumentsDoneEvent, + ResponseFunctionToolCall, + ResponseFunctionWebSearch, + ResponseMcpCallArgumentsDeltaEvent, + ResponseMcpCallArgumentsDoneEvent, + ResponseMcpCallCompletedEvent, + ResponseMcpCallInProgressEvent, + ResponseOutputItemAddedEvent, + ResponseOutputItemDoneEvent, + ResponseOutputMessage, + ResponseOutputText, + ResponseReasoningItem, + ResponseReasoningTextDeltaEvent, + ResponseReasoningTextDoneEvent, + ResponseTextDeltaEvent, + ResponseTextDoneEvent, + ResponseWebSearchCallCompletedEvent, + ResponseWebSearchCallInProgressEvent, + ResponseWebSearchCallSearchingEvent, + response_function_web_search, +) +from openai.types.responses.response_output_item import McpCall +from openai.types.responses.response_reasoning_item import ( + Content as ResponseReasoningTextContent, +) + +from vllm.entrypoints.mcp.tool_server import ToolServer +from vllm.entrypoints.openai.responses.context import StreamingHarmonyContext +from vllm.entrypoints.openai.responses.protocol import ( + ResponseReasoningPartAddedEvent, + ResponseReasoningPartDoneEvent, + StreamingResponsesResponse, +) +from vllm.utils import random_uuid + +TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = { + "python": "code_interpreter", + "container": "container", + "browser": "web_search_preview", +} + + +@dataclass +class HarmonyStreamingState: + """Mutable state for harmony streaming event processing.""" + + current_content_index: int = -1 + current_output_index: int = 0 + current_item_id: str = "" + sent_output_item_added: bool = False + is_first_function_call_delta: bool = False + + def reset_for_new_item(self) -> None: + """Reset state when expecting a new output item.""" + self.current_output_index += 1 + self.sent_output_item_added = False + self.is_first_function_call_delta = False + + +def is_mcp_tool_by_namespace(recipient: str | None) -> bool: + """ + Determine if a tool call is an MCP tool based on recipient prefix. + + - Tools starting with "functions." are function calls + - Everything else is an MCP tool + """ + if recipient is None: + return False + + # Function calls have "functions." prefix + # Everything else is an MCP tool + return not recipient.startswith("functions.") + + +def emit_function_call_done_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when a function call completes.""" + function_name = previous_item.recipient[len("functions.") :] + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseFunctionCallArgumentsDoneEvent( + type="response.function_call_arguments.done", + arguments=previous_item.content[0].text, + name=function_name, + item_id=state.current_item_id, + output_index=state.current_output_index, + sequence_number=-1, + ) + ) + function_call_item = ResponseFunctionToolCall( + type="function_call", + arguments=previous_item.content[0].text, + name=function_name, + item_id=state.current_item_id, + output_index=state.current_output_index, + sequence_number=-1, + call_id=f"fc_{random_uuid()}", + status="completed", + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=function_call_item, + ) + ) + return events + + +def emit_mcp_call_done_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when an MCP tool call completes.""" + server_label = TOOL_NAME_TO_MCP_SERVER_LABEL.get( + previous_item.recipient, previous_item.recipient + ) + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseMcpCallArgumentsDoneEvent( + type="response.mcp_call_arguments.done", + arguments=previous_item.content[0].text, + name=previous_item.recipient, + item_id=state.current_item_id, + output_index=state.current_output_index, + sequence_number=-1, + ) + ) + events.append( + ResponseMcpCallCompletedEvent( + type="response.mcp_call.completed", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=McpCall( + type="mcp_call", + arguments=previous_item.content[0].text, + name=previous_item.recipient, + id=state.current_item_id, + server_label=server_label, + status="completed", + ), + ) + ) + return events + + +def emit_reasoning_done_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when a reasoning (analysis) item completes.""" + content = ResponseReasoningTextContent( + text=previous_item.content[0].text, + type="reasoning_text", + ) + reasoning_item = ResponseReasoningItem( + type="reasoning", + content=[content], + status="completed", + id=state.current_item_id, + summary=[], + ) + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseReasoningTextDoneEvent( + type="response.reasoning_text.done", + item_id=state.current_item_id, + sequence_number=-1, + output_index=state.current_output_index, + content_index=state.current_content_index, + text=previous_item.content[0].text, + ) + ) + events.append( + ResponseReasoningPartDoneEvent( + type="response.reasoning_part.done", + sequence_number=-1, + item_id=state.current_item_id, + output_index=state.current_output_index, + content_index=state.current_content_index, + part=content, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=reasoning_item, + ) + ) + return events + + +def emit_text_output_done_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when a final text output item completes.""" + text_content = ResponseOutputText( + type="output_text", + text=previous_item.content[0].text, + annotations=[], + ) + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseTextDoneEvent( + type="response.output_text.done", + sequence_number=-1, + output_index=state.current_output_index, + content_index=state.current_content_index, + text=previous_item.content[0].text, + logprobs=[], + item_id=state.current_item_id, + ) + ) + events.append( + ResponseContentPartDoneEvent( + type="response.content_part.done", + sequence_number=-1, + item_id=state.current_item_id, + output_index=state.current_output_index, + content_index=state.current_content_index, + part=text_content, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=ResponseOutputMessage( + id=state.current_item_id, + type="message", + role="assistant", + content=[text_content], + status="completed", + ), + ) + ) + return events + + +def emit_previous_item_done_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit done events for the previous item when expecting a new start.""" + if previous_item.recipient is not None: + # Deal with tool call + if previous_item.recipient.startswith("functions."): + return emit_function_call_done_events(previous_item, state) + elif ( + is_mcp_tool_by_namespace(previous_item.recipient) + and state.current_item_id is not None + and state.current_item_id.startswith("mcp_") + ): + return emit_mcp_call_done_events(previous_item, state) + elif previous_item.channel == "analysis": + return emit_reasoning_done_events(previous_item, state) + elif previous_item.channel == "final": + return emit_text_output_done_events(previous_item, state) + return [] + + +def emit_final_channel_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for final channel text delta streaming.""" + events: list[StreamingResponsesResponse] = [] + if not state.sent_output_item_added: + state.sent_output_item_added = True + state.current_item_id = f"msg_{random_uuid()}" + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=ResponseOutputMessage( + id=state.current_item_id, + type="message", + role="assistant", + content=[], + status="in_progress", + ), + ) + ) + state.current_content_index += 1 + events.append( + ResponseContentPartAddedEvent( + type="response.content_part.added", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + content_index=state.current_content_index, + part=ResponseOutputText( + type="output_text", + text="", + annotations=[], + logprobs=[], + ), + ) + ) + events.append( + ResponseTextDeltaEvent( + type="response.output_text.delta", + sequence_number=-1, + content_index=state.current_content_index, + output_index=state.current_output_index, + item_id=state.current_item_id, + delta=ctx.last_content_delta, + # TODO, use logprobs from ctx.last_request_output + logprobs=[], + ) + ) + return events + + +def emit_analysis_channel_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for analysis channel reasoning delta streaming.""" + events: list[StreamingResponsesResponse] = [] + if not state.sent_output_item_added: + state.sent_output_item_added = True + state.current_item_id = f"msg_{random_uuid()}" + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=ResponseReasoningItem( + type="reasoning", + id=state.current_item_id, + summary=[], + status="in_progress", + ), + ) + ) + state.current_content_index += 1 + events.append( + ResponseReasoningPartAddedEvent( + type="response.reasoning_part.added", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + content_index=state.current_content_index, + part=ResponseReasoningTextContent( + text="", + type="reasoning_text", + ), + ) + ) + events.append( + ResponseReasoningTextDeltaEvent( + type="response.reasoning_text.delta", + item_id=state.current_item_id, + output_index=state.current_output_index, + content_index=state.current_content_index, + delta=ctx.last_content_delta, + sequence_number=-1, + ) + ) + return events + + +def emit_mcp_tool_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, + recipient: str, +) -> list[StreamingResponsesResponse]: + """Emit events for MCP tool delta streaming.""" + server_label = TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient) + events: list[StreamingResponsesResponse] = [] + if not state.sent_output_item_added: + state.sent_output_item_added = True + state.current_item_id = f"mcp_{random_uuid()}" + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=McpCall( + type="mcp_call", + id=state.current_item_id, + name=recipient, + arguments="", + server_label=server_label, + status="in_progress", + ), + ) + ) + events.append( + ResponseMcpCallInProgressEvent( + type="response.mcp_call.in_progress", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseMcpCallArgumentsDeltaEvent( + type="response.mcp_call_arguments.delta", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + delta=ctx.last_content_delta, + ) + ) + return events + + +def emit_code_interpreter_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for code interpreter delta streaming.""" + events: list[StreamingResponsesResponse] = [] + if not state.sent_output_item_added: + state.sent_output_item_added = True + state.current_item_id = f"tool_{random_uuid()}" + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=ResponseCodeInterpreterToolCallParam( + type="code_interpreter_call", + id=state.current_item_id, + code=None, + container_id="auto", + outputs=None, + status="in_progress", + ), + ) + ) + events.append( + ResponseCodeInterpreterCallInProgressEvent( + type="response.code_interpreter_call.in_progress", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseCodeInterpreterCallCodeDeltaEvent( + type="response.code_interpreter_call_code.delta", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + delta=ctx.last_content_delta, + ) + ) + return events + + +def emit_mcp_prefix_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for MCP prefix (mcp.*) delta streaming.""" + events: list[StreamingResponsesResponse] = [] + if not state.sent_output_item_added: + state.sent_output_item_added = True + state.current_item_id = f"mcp_{random_uuid()}" + mcp_name = ctx.parser.current_recipient[len("mcp.") :] + + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=McpCall( + type="mcp_call", + id=state.current_item_id, + name=mcp_name, + arguments="", + server_label=mcp_name, + status="in_progress", + ), + ) + ) + events.append( + ResponseMcpCallInProgressEvent( + type="response.mcp_call.in_progress", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + + events.append( + ResponseMcpCallArgumentsDeltaEvent( + type="response.mcp_call_arguments.delta", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + delta=ctx.last_content_delta, + ) + ) + return events + + +def emit_function_call_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for developer function calls on commentary channel.""" + if not ( + ctx.parser.current_channel == "commentary" + and ctx.parser.current_recipient + and ctx.parser.current_recipient.startswith("functions.") + ): + return [] + + events: list[StreamingResponsesResponse] = [] + if state.is_first_function_call_delta is False: + state.is_first_function_call_delta = True + fc_name = ctx.parser.current_recipient[len("functions.") :] + state.current_item_id = f"fc_{random_uuid()}" + tool_call_item = ResponseFunctionToolCall( + name=fc_name, + type="function_call", + id=state.current_item_id, + call_id=f"call_{random_uuid()}", + arguments="", + status="in_progress", + ) + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=tool_call_item, + ) + ) + # Always emit the delta (including on first call) + events.append( + ResponseFunctionCallArgumentsDeltaEvent( + item_id=state.current_item_id, + delta=ctx.last_content_delta, + output_index=state.current_output_index, + sequence_number=-1, + type="response.function_call_arguments.delta", + ) + ) + return events + + +def emit_content_delta_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for content delta streaming based on channel type.""" + if not ctx.last_content_delta: + return [] + + if ctx.parser.current_channel == "final" and ctx.parser.current_recipient is None: + return emit_final_channel_delta_events(ctx, state) + elif ( + ctx.parser.current_channel == "analysis" + and ctx.parser.current_recipient is None + ): + return emit_analysis_channel_delta_events(ctx, state) + # built-in tools will be triggered on the analysis channel + # However, occasionally built-in tools will + # still be output to commentary. + elif ( + ctx.parser.current_channel == "commentary" + or ctx.parser.current_channel == "analysis" + ) and ctx.parser.current_recipient is not None: + recipient = ctx.parser.current_recipient + # Check for function calls first - they have their own event handling + if recipient.startswith("functions."): + return emit_function_call_delta_events(ctx, state) + if is_mcp_tool_by_namespace(recipient): + return emit_mcp_tool_delta_events(ctx, state, recipient) + else: + return emit_code_interpreter_delta_events(ctx, state) + elif ( + ( + ctx.parser.current_channel == "commentary" + or ctx.parser.current_channel == "analysis" + ) + and ctx.parser.current_recipient is not None + and ctx.parser.current_recipient.startswith("mcp.") + ): + return emit_mcp_prefix_delta_events(ctx, state) + + return [] + + +def emit_browser_tool_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events for browser tool calls (web search).""" + function_name = previous_item.recipient[len("browser.") :] + parsed_args = json.loads(previous_item.content[0].text) + action = None + + if function_name == "search": + action = response_function_web_search.ActionSearch( + type="search", + query=parsed_args["query"], + ) + elif function_name == "open": + action = response_function_web_search.ActionOpenPage( + type="open_page", + # TODO: translate to url + url=f"cursor:{parsed_args.get('cursor', '')}", + ) + elif function_name == "find": + action = response_function_web_search.ActionFind( + type="find", + pattern=parsed_args["pattern"], + # TODO: translate to url + url=f"cursor:{parsed_args.get('cursor', '')}", + ) + else: + raise ValueError(f"Unknown function name: {function_name}") + + state.current_item_id = f"tool_{random_uuid()}" + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseOutputItemAddedEvent( + type="response.output_item.added", + sequence_number=-1, + output_index=state.current_output_index, + item=response_function_web_search.ResponseFunctionWebSearch( + # TODO: generate a unique id for web search call + type="web_search_call", + id=state.current_item_id, + action=action, + status="in_progress", + ), + ) + ) + events.append( + ResponseWebSearchCallInProgressEvent( + type="response.web_search_call.in_progress", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseWebSearchCallSearchingEvent( + type="response.web_search_call.searching", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + # enqueue + events.append( + ResponseWebSearchCallCompletedEvent( + type="response.web_search_call.completed", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=ResponseFunctionWebSearch( + type="web_search_call", + id=state.current_item_id, + action=action, + status="completed", + ), + ) + ) + return events + + +def emit_mcp_tool_completion_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when an MCP tool completes during assistant action turn.""" + recipient = previous_item.recipient + server_label = TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient) + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseMcpCallArgumentsDoneEvent( + type="response.mcp_call_arguments.done", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + arguments=previous_item.content[0].text, + name=recipient, + ) + ) + events.append( + ResponseMcpCallCompletedEvent( + type="response.mcp_call.completed", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=McpCall( + type="mcp_call", + id=state.current_item_id, + name=recipient, + arguments=previous_item.content[0].text, + server_label=server_label, + status="completed", + ), + ) + ) + return events + + +def emit_code_interpreter_completion_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when code interpreter completes.""" + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseCodeInterpreterCallCodeDoneEvent( + type="response.code_interpreter_call_code.done", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + code=previous_item.content[0].text, + ) + ) + events.append( + ResponseCodeInterpreterCallInterpretingEvent( + type="response.code_interpreter_call.interpreting", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseCodeInterpreterCallCompletedEvent( + type="response.code_interpreter_call.completed", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=ResponseCodeInterpreterToolCallParam( + type="code_interpreter_call", + id=state.current_item_id, + code=previous_item.content[0].text, + container_id="auto", + outputs=[], + status="completed", + ), + ) + ) + return events + + +def emit_mcp_prefix_completion_events( + previous_item, + state: HarmonyStreamingState, +) -> list[StreamingResponsesResponse]: + """Emit events when an MCP prefix tool (mcp.*) completes.""" + mcp_name = previous_item.recipient[len("mcp.") :] + events: list[StreamingResponsesResponse] = [] + events.append( + ResponseMcpCallArgumentsDoneEvent( + type="response.mcp_call_arguments.done", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + arguments=previous_item.content[0].text, + name=mcp_name, + ) + ) + events.append( + ResponseMcpCallCompletedEvent( + type="response.mcp_call.completed", + sequence_number=-1, + output_index=state.current_output_index, + item_id=state.current_item_id, + ) + ) + events.append( + ResponseOutputItemDoneEvent( + type="response.output_item.done", + sequence_number=-1, + output_index=state.current_output_index, + item=McpCall( + type="mcp_call", + id=state.current_item_id, + name=mcp_name, + arguments=previous_item.content[0].text, + server_label=mcp_name, + status="completed", + ), + ) + ) + return events + + +def emit_tool_action_events( + ctx: StreamingHarmonyContext, + state: HarmonyStreamingState, + tool_server: ToolServer | None, +) -> list[StreamingResponsesResponse]: + """Emit events for tool action turn.""" + if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0: + return [] + + events: list[StreamingResponsesResponse] = [] + previous_item = ctx.parser.messages[-1] + + # Handle browser tool + if ( + tool_server is not None + and tool_server.has_tool("browser") + and previous_item.recipient is not None + and previous_item.recipient.startswith("browser.") + ): + events.extend(emit_browser_tool_events(previous_item, state)) + + # Handle tool completion + if ( + tool_server is not None + and previous_item.recipient is not None + and state.current_item_id is not None + and state.sent_output_item_added + ): + recipient = previous_item.recipient + # Handle MCP prefix tool completion first + if recipient.startswith("mcp."): + events.extend(emit_mcp_prefix_completion_events(previous_item, state)) + else: + # Handle other MCP tool and code interpreter completion + is_mcp_tool = is_mcp_tool_by_namespace( + recipient + ) and state.current_item_id.startswith("mcp_") + if is_mcp_tool: + events.extend(emit_mcp_tool_completion_events(previous_item, state)) + else: + events.extend( + emit_code_interpreter_completion_events(previous_item, state) + ) + + return events From 1fe462168c381f604a5ef9d491a230a3dd861d2c Mon Sep 17 00:00:00 2001 From: Huamin Li <3ericli@gmail.com> Date: Fri, 20 Feb 2026 06:21:56 -0800 Subject: [PATCH 29/74] [perf] Avoid dtype promotion sync in mamba_get_block_table_tensor (#34870) Signed-off-by: Huamin Li <3ericli@gmail.com> Co-authored-by: Cyrus Leung --- vllm/v1/attention/backends/utils.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/v1/attention/backends/utils.py b/vllm/v1/attention/backends/utils.py index eda50155dda..1b030eaf140 100644 --- a/vllm/v1/attention/backends/utils.py +++ b/vllm/v1/attention/backends/utils.py @@ -855,8 +855,12 @@ def mamba_get_block_table_tensor( (seq_lens - 1) // kv_cache_spec.block_size, min=0, ) + # Use int32 for arithmetic to avoid dtype promotion overhead, + # then convert to int64 for gather (which requires Long indices) offsets = torch.arange( - 1 + kv_cache_spec.num_speculative_blocks, device=block_table.device + 1 + kv_cache_spec.num_speculative_blocks, + device=block_table.device, + dtype=torch.int32, ) - indices_to_gather = start_indices.unsqueeze(1) + offsets + indices_to_gather = (start_indices.unsqueeze(1) + offsets).to(torch.int64) return torch.gather(block_table, 1, indices_to_gather) From 6ce80f7071b009badaa2c473e96ec55a134790d2 Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Fri, 20 Feb 2026 15:38:11 +0000 Subject: [PATCH 30/74] Ensure that MkDocs v2 does not get installed (#34958) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- requirements/docs.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/docs.txt b/requirements/docs.txt index 0997b52d2e3..952e7c09bae 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,4 @@ -mkdocs +mkdocs<2.0.0 mkdocs-api-autonav mkdocs-material mkdocstrings-python From a6d0299c75f2c0687334d50d302801ade083c784 Mon Sep 17 00:00:00 2001 From: Yanan Cao Date: Fri, 20 Feb 2026 08:36:51 -0800 Subject: [PATCH 31/74] [Kernel] [Helion] [6/N] Add num_tokens dimension to silu_mul autotuning and dispatching (#34185) Signed-off-by: Yanan Cao --- tests/kernels/helion/test_silu_mul_fp8.py | 86 +- vllm/kernels/helion/configs/silu_mul_fp8.json | 55298 +++++++++++++++- vllm/kernels/helion/ops/silu_mul_fp8.py | 89 +- 3 files changed, 55236 insertions(+), 237 deletions(-) diff --git a/tests/kernels/helion/test_silu_mul_fp8.py b/tests/kernels/helion/test_silu_mul_fp8.py index da6405d6c83..887f20b9f56 100644 --- a/tests/kernels/helion/test_silu_mul_fp8.py +++ b/tests/kernels/helion/test_silu_mul_fp8.py @@ -54,8 +54,8 @@ def reset_config_manager_singleton(): class TestSiluMulFp8ConfigPicker: def test_config_picker_exact_match(self): config_keys = [ - "intermediate_2048_batchsize_256", - "intermediate_4096_batchsize_256", + "intermediate_2048_numtokens_256", + "intermediate_4096_numtokens_256", ] input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda") @@ -63,12 +63,12 @@ class TestSiluMulFp8ConfigPicker: args = (input_tensor, scale) selected_key = pick_silu_mul_fp8_config(args, config_keys) - assert selected_key == "intermediate_2048_batchsize_256" + assert selected_key == "intermediate_2048_numtokens_256" def test_config_picker_closest_match(self): config_keys = [ - "intermediate_2048_batchsize_256", - "intermediate_4096_batchsize_256", + "intermediate_2048_numtokens_256", + "intermediate_4096_numtokens_256", ] # Use 7000 (intermediate_size=3500) which is closer to 4096 than 2048 input_tensor = torch.randn(32, 7000, dtype=torch.bfloat16, device="cuda") @@ -76,10 +76,10 @@ class TestSiluMulFp8ConfigPicker: args = (input_tensor, scale) selected_key = pick_silu_mul_fp8_config(args, config_keys) - assert selected_key == "intermediate_4096_batchsize_256" + assert selected_key == "intermediate_4096_numtokens_256" def test_config_picker_fallback_to_default(self): - config_keys = ["default", "some_other_key"] + config_keys = ["default"] input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda") scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") @@ -101,9 +101,9 @@ class TestSiluMulFp8ConfigPicker: @pytest.mark.parametrize("intermediate_size", [2048, 4096, 5120]) def test_config_picker_different_sizes(self, intermediate_size): config_keys = [ - "intermediate_2048_batchsize_256", - "intermediate_4096_batchsize_256", - "intermediate_5120_batchsize_256", + "intermediate_2048_numtokens_256", + "intermediate_4096_numtokens_256", + "intermediate_5120_numtokens_256", ] input_tensor = torch.randn( @@ -113,9 +113,73 @@ class TestSiluMulFp8ConfigPicker: args = (input_tensor, scale) selected_key = pick_silu_mul_fp8_config(args, config_keys) - expected_key = f"intermediate_{intermediate_size}_batchsize_256" + expected_key = f"intermediate_{intermediate_size}_numtokens_256" assert selected_key == expected_key + def test_config_picker_numtokens_ceiling(self): + """Pick the smallest numtokens >= input num_tokens.""" + config_keys = [ + "intermediate_4096_numtokens_8", + "intermediate_4096_numtokens_32", + "intermediate_4096_numtokens_128", + "intermediate_4096_numtokens_256", + ] + # 20 tokens -> should pick numtokens_32 (smallest >= 20) + input_tensor = torch.randn(20, 8192, dtype=torch.bfloat16, device="cuda") + scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") + + selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys) + assert selected_key == "intermediate_4096_numtokens_32" + + def test_config_picker_numtokens_exact(self): + """Exact num_tokens match is preferred over ceiling.""" + config_keys = [ + "intermediate_4096_numtokens_8", + "intermediate_4096_numtokens_32", + "intermediate_4096_numtokens_128", + ] + input_tensor = torch.randn(32, 8192, dtype=torch.bfloat16, device="cuda") + scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") + + selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys) + assert selected_key == "intermediate_4096_numtokens_32" + + def test_config_picker_numtokens_fallback_to_largest(self): + """Fall back to the largest numtokens when input exceeds all.""" + config_keys = [ + "intermediate_4096_numtokens_8", + "intermediate_4096_numtokens_32", + "intermediate_4096_numtokens_128", + ] + # 512 tokens -> exceeds all available, should pick largest (128) + input_tensor = torch.randn(512, 8192, dtype=torch.bfloat16, device="cuda") + scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") + + selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys) + assert selected_key == "intermediate_4096_numtokens_128" + + def test_config_picker_malformed_key_raises(self): + """Malformed config keys should raise ValueError.""" + config_keys = ["intermediate_4096_badformat_256"] + input_tensor = torch.randn(32, 8192, dtype=torch.bfloat16, device="cuda") + scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") + + with pytest.raises(ValueError, match="Malformed config key"): + pick_silu_mul_fp8_config((input_tensor, scale), config_keys) + + def test_config_picker_default_ignored_when_valid_keys_exist(self): + """'default' is skipped in favor of a real match.""" + config_keys = [ + "default", + "intermediate_4096_numtokens_32", + "intermediate_4096_numtokens_128", + ] + input_tensor = torch.randn(64, 8192, dtype=torch.bfloat16, device="cuda") + scale = torch.tensor([0.5], dtype=torch.float32, device="cuda") + + selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys) + assert selected_key == "intermediate_4096_numtokens_128" + class TestSiluMulFp8Correctness: @pytest.mark.parametrize("batch_size", [1, 8, 32, 128]) diff --git a/vllm/kernels/helion/configs/silu_mul_fp8.json b/vllm/kernels/helion/configs/silu_mul_fp8.json index c26ca087db3..0f0de04a1cc 100644 --- a/vllm/kernels/helion/configs/silu_mul_fp8.json +++ b/vllm/kernels/helion/configs/silu_mul_fp8.json @@ -1,9 +1,54 @@ { "nvidia_h200": { - "intermediate_2048_batchsize_256": { + "intermediate_2048_numtokens_256": { "block_sizes": [ 64, - 128 + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_256": { + "block_sizes": [ + 32, + 512 ], "loop_orders": [ [ @@ -20,6 +65,7 @@ "range_unroll_factors": [ 0 ], + "range_warp_specializes": [], "range_num_stages": [ 0 ], @@ -34,21 +80,65 @@ "", "" ], - "num_warps": 32, - "num_stages": 1, + "num_warps": 2, + "num_stages": 2, "indexing": [ "pointer", "tensor_descriptor", "pointer", "pointer" ], - "pid_type": "flat", - "range_warp_specializes": [] + "pid_type": "flat" }, - "intermediate_4096_batchsize_256": { + "default": { "block_sizes": [ - 16, - 64 + 1, + 512 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_256": { + "block_sizes": [ + 32, + 8 ], "loop_orders": [ [ @@ -57,7 +147,7 @@ ] ], "flatten_loops": [ - false + true ], "l2_groupings": [ 1 @@ -65,6 +155,97 @@ "range_unroll_factors": [ 0 ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_256": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_256": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], "range_num_stages": [ 0 ], @@ -87,13 +268,192 @@ "pointer", "pointer" ], - "pid_type": "flat", - "range_warp_specializes": [] + "pid_type": "flat" }, - "default": { + "intermediate_7688_numtokens_256": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_256": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_1": { "block_sizes": [ 1, - 512 + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_1": { + "block_sizes": [ + 1, + 1 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_1": { + "block_sizes": [ + 1, + 32 ], "loop_orders": [ [ @@ -104,12 +464,5098 @@ "flatten_loops": [ false ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_2": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_2": { + "block_sizes": [ + 1, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_2": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_2": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_2": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_2": { + "block_sizes": [ + 1, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_4": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_4": { + "block_sizes": [ + 1, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_4": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_4": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_8": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_8": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_8": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_8": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_16": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_16": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_16": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_24": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_24": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_24": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_24": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_24": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_24": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_32": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_32": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_32": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_32": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_40": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_48": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_48": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_48": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_48": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_48": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_48": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_56": { + "block_sizes": [ + 2, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_56": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_56": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_56": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_56": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_56": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_64": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_64": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_64": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_64": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_64": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_64": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_72": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_72": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_72": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_72": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_72": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_72": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_80": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_80": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_80": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_80": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], "l2_groupings": [ 4 ], "range_unroll_factors": [ 0 ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_80": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_80": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_88": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_88": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_88": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "first", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_88": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_96": { + "block_sizes": [ + 128, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_96": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_96": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_96": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_96": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_96": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_104": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_104": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_104": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_104": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_104": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_104": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_112": { + "block_sizes": [ + 32, + 1024 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_112": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_112": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_112": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_120": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_120": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_128": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_128": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_128": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_128": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_136": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_136": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], "range_num_stages": [ 0 ], @@ -125,11 +5571,8290 @@ "" ], "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_136": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_136": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_136": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_136": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_144": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_144": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_144": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_144": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_144": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_144": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_152": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_152": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_152": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_152": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_160": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_160": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_160": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_160": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_168": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_168": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_168": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_168": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_168": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_168": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_176": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_176": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_176": { + "block_sizes": [ + 4, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_176": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_184": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_184": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_184": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_184": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_192": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_192": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_192": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_192": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_192": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_192": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_200": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_200": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_200": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_208": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, "num_stages": 2, "indexing": [ "tensor_descriptor", "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_208": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_216": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_216": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_216": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_216": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", "tensor_descriptor", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_216": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_216": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_224": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_224": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_224": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_224": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_224": { + "block_sizes": [ + 256, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_224": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_232": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_232": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_232": { + "block_sizes": [ + 16, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_232": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_232": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_232": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_240": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_240": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_248": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_248": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_248": { + "block_sizes": [ + 256, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_248": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_248": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_248": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_272": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_272": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_272": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_288": { + "block_sizes": [ + 4, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_288": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_288": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_288": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_288": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_288": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_304": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_304": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_304": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_304": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_320": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_320": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_320": { + "block_sizes": [ + 512, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_320": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_320": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_320": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_336": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_336": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_336": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_336": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_352": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_352": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_352": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_352": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_368": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_368": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_368": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_368": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_384": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_384": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_384": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_400": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_400": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_416": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_416": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_416": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_416": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_416": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_416": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_432": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_432": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_432": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_432": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_432": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_432": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_448": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_448": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_448": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_448": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_464": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_464": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_464": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_464": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_480": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_480": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_480": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_480": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_480": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_480": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_496": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_496": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_496": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_496": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_512": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_512": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_512": { + "block_sizes": [ + 128, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_512": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", "pointer" ], "pid_type": "flat", @@ -137,55 +13862,10 @@ } }, "nvidia_h100_pcie": { - "intermediate_2048_batchsize_256": { + "intermediate_2048_numtokens_256": { "block_sizes": [ - 1, - 512 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat", - "range_warp_specializes": [] - }, - "intermediate_4096_batchsize_256": { - "block_sizes": [ - 256, - 128 + 64, + 32 ], "loop_orders": [ [ @@ -200,32 +13880,77 @@ 1 ], "range_unroll_factors": [ - 2 + 0 ], + "range_warp_specializes": [], "range_num_stages": [ - 3 + 0 ], "range_multi_buffers": [ - false + null ], "range_flattens": [ - true + null ], "load_eviction_policies": [ - "last", - "last", + "", + "", "" ], - "num_warps": 32, - "num_stages": 3, + "num_warps": 8, + "num_stages": 1, "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", + "pointer", + "pointer", + "pointer", "pointer" ], - "pid_type": "persistent_blocked", - "range_warp_specializes": [] + "pid_type": "flat" + }, + "intermediate_4096_numtokens_256": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" }, "default": { "block_sizes": [ @@ -247,6 +13972,7 @@ "range_unroll_factors": [ 0 ], + "range_warp_specializes": [], "range_num_stages": [ 0 ], @@ -269,60 +13995,13739 @@ "tensor_descriptor", "pointer" ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_256": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_256": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_256": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_7688_numtokens_256": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_256": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_1": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_1": { + "block_sizes": [ + 1, + 1 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_2": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_2": { + "block_sizes": [ + 1, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_2": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_2": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_2": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_2": { + "block_sizes": [ + 1, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_4": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_4": { + "block_sizes": [ + 1, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_4": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_4": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_8": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_8": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_8": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_8": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_16": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_16": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_16": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_24": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_24": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_24": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_24": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_24": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_24": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_32": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_32": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_32": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_32": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_40": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_48": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_48": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_48": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_48": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_48": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_48": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_56": { + "block_sizes": [ + 2, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_56": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_56": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_56": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_56": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_56": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_64": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_64": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_64": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_64": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_64": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_64": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_72": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_72": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_72": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_72": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_72": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_72": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_80": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_80": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_80": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_80": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_80": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_80": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_88": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_88": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_88": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "first", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_88": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_96": { + "block_sizes": [ + 128, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_96": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_96": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_96": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_96": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_96": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_104": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_104": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_104": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_104": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_104": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_104": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_112": { + "block_sizes": [ + 32, + 1024 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_112": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_112": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_112": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_120": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_120": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_128": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_128": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_128": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_128": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_136": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_136": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_136": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_136": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_136": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_136": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_144": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_144": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_144": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_144": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_144": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_144": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_152": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_152": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_152": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_152": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_160": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_160": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_160": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_160": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_168": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_168": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_168": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_168": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_168": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_168": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_176": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_176": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_176": { + "block_sizes": [ + 4, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_176": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_184": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_184": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_184": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_184": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_192": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_192": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_192": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_192": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_192": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_192": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_200": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_200": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_200": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_208": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_208": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_216": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_216": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_216": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_216": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_216": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_216": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_224": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_224": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_224": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_224": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_224": { + "block_sizes": [ + 256, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_224": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_232": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_232": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_232": { + "block_sizes": [ + 16, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_232": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_232": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_232": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_240": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_240": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_248": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_248": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_248": { + "block_sizes": [ + 256, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_248": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_248": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_248": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_272": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_272": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_272": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_288": { + "block_sizes": [ + 4, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_288": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_288": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_288": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_288": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_288": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_304": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_304": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_304": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_304": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_320": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_320": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_320": { + "block_sizes": [ + 512, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_320": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_320": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_320": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_336": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_336": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_336": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_336": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_352": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_352": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_352": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_352": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_368": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_368": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_368": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_368": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_384": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_384": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_384": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_400": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_400": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_416": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_416": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_416": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_416": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_416": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_416": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_432": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_432": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_432": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_432": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_432": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_432": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_448": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_448": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_448": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_448": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_464": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_464": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_464": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_464": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_480": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_480": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_480": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_480": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_480": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_480": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_496": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_496": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_496": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_496": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_512": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_512": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_512": { + "block_sizes": [ + 128, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_512": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], "pid_type": "flat", "range_warp_specializes": [] } }, - "nvidia_h100_sxm5": { - "intermediate_2048_batchsize_256": { + "nvidia_h100_80gb_hbm3": { + "intermediate_2048_numtokens_256": { "block_sizes": [ - 1, - 512 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat", - "range_warp_specializes": [] - }, - "intermediate_4096_batchsize_256": { - "block_sizes": [ - 256, - 128 + 64, + 32 ], "loop_orders": [ [ @@ -337,32 +27742,77 @@ 1 ], "range_unroll_factors": [ - 2 + 0 ], + "range_warp_specializes": [], "range_num_stages": [ - 3 + 0 ], "range_multi_buffers": [ - false + null ], "range_flattens": [ - true + null ], "load_eviction_policies": [ - "last", - "last", + "", + "", "" ], - "num_warps": 32, - "num_stages": 3, + "num_warps": 8, + "num_stages": 1, "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", + "pointer", + "pointer", + "pointer", "pointer" ], - "pid_type": "persistent_blocked", - "range_warp_specializes": [] + "pid_type": "flat" + }, + "intermediate_4096_numtokens_256": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" }, "default": { "block_sizes": [ @@ -384,6 +27834,7 @@ "range_unroll_factors": [ 0 ], + "range_warp_specializes": [], "range_num_stages": [ 0 ], @@ -406,60 +27857,13739 @@ "tensor_descriptor", "pointer" ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_256": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_256": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_256": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_7688_numtokens_256": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_256": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_1": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_1": { + "block_sizes": [ + 1, + 1 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_2": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_2": { + "block_sizes": [ + 1, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_2": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_2": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_2": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_2": { + "block_sizes": [ + 1, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_4": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_4": { + "block_sizes": [ + 1, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_4": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_4": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_8": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_8": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_8": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_8": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_16": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_16": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_16": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_24": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_24": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_24": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_24": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_24": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_24": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_32": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_32": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_32": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_32": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_40": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_48": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_48": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_48": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_48": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_48": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_48": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_56": { + "block_sizes": [ + 2, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_56": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_56": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_56": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_56": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_56": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_64": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_64": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_64": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_64": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_64": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_64": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_72": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_72": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_72": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_72": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_72": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_72": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_80": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_80": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_80": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_80": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_80": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_80": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_88": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_88": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_88": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "first", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_88": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_96": { + "block_sizes": [ + 128, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_96": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_96": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_96": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_96": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_96": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_104": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_104": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_104": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_104": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_104": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_104": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_112": { + "block_sizes": [ + 32, + 1024 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_112": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_112": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_112": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_120": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_120": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_128": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_128": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_128": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_128": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_136": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_136": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_136": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_136": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_136": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_136": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_144": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_144": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_144": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_144": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_144": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_144": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_152": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_152": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_152": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_152": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_160": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_160": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_160": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_160": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_168": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_168": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_168": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_168": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_168": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_168": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_176": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_176": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_176": { + "block_sizes": [ + 4, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_176": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_184": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_184": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_184": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_184": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_192": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_192": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_192": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_192": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_192": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_192": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_200": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_200": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_200": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_208": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_208": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_216": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_216": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_216": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_216": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_216": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_216": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_224": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_224": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_224": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_224": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_224": { + "block_sizes": [ + 256, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_224": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_232": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_232": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_232": { + "block_sizes": [ + 16, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_232": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_232": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_232": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_240": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_240": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_248": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_248": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_248": { + "block_sizes": [ + 256, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_248": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_248": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_248": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_272": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_272": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_272": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_288": { + "block_sizes": [ + 4, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_288": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_288": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_288": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_288": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_288": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_304": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_304": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_304": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_304": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_320": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_320": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_320": { + "block_sizes": [ + 512, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_320": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_320": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_320": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_336": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_336": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_336": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_336": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_352": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_352": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_352": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_352": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_368": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_368": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_368": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_368": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_384": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_384": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_384": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_400": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_400": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_416": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_416": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_416": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_416": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_416": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_416": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_432": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_432": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_432": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_432": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_432": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_432": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_448": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_448": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_448": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_448": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_464": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_464": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_464": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_464": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_480": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_480": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_480": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_480": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_480": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_480": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_496": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_496": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_496": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_496": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_512": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_512": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_512": { + "block_sizes": [ + 128, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_512": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], "pid_type": "flat", "range_warp_specializes": [] } }, "nvidia_h100": { - "intermediate_2048_batchsize_256": { + "intermediate_2048_numtokens_256": { "block_sizes": [ - 1, - 512 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat", - "range_warp_specializes": [] - }, - "intermediate_4096_batchsize_256": { - "block_sizes": [ - 256, - 128 + 64, + 32 ], "loop_orders": [ [ @@ -474,32 +41604,77 @@ 1 ], "range_unroll_factors": [ - 2 + 0 ], + "range_warp_specializes": [], "range_num_stages": [ - 3 + 0 ], "range_multi_buffers": [ - false + null ], "range_flattens": [ - true + null ], "load_eviction_policies": [ - "last", - "last", + "", + "", "" ], - "num_warps": 32, - "num_stages": 3, + "num_warps": 8, + "num_stages": 1, "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", + "pointer", + "pointer", + "pointer", "pointer" ], - "pid_type": "persistent_blocked", - "range_warp_specializes": [] + "pid_type": "flat" + }, + "intermediate_4096_numtokens_256": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" }, "default": { "block_sizes": [ @@ -521,6 +41696,7 @@ "range_unroll_factors": [ 0 ], + "range_warp_specializes": [], "range_num_stages": [ 0 ], @@ -543,8 +41719,13732 @@ "tensor_descriptor", "pointer" ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_256": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_256": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_256": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_7688_numtokens_256": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_256": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_1": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_1": { + "block_sizes": [ + 1, + 1 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_1": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_2": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_2": { + "block_sizes": [ + 1, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_2": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_2": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_2": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_2": { + "block_sizes": [ + 1, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_4": { + "block_sizes": [ + 1, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_4": { + "block_sizes": [ + 1, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_4": { + "block_sizes": [ + 1, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_4": { + "block_sizes": [ + 1, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_4": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_8": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_8": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_8": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_8": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_8": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_16": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_16": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_16": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_16": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_24": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_24": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_24": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_24": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_24": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_24": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_32": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_32": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_32": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_32": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_32": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_40": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_40": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_40": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_48": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_48": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_48": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_48": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_48": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_48": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_56": { + "block_sizes": [ + 2, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_56": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_56": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_56": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_56": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_56": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_64": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_64": { + "block_sizes": [ + 4, + 64 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_64": { + "block_sizes": [ + 2, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_64": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_64": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_64": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_72": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_72": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_72": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_72": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_72": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_72": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_80": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_80": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_80": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_80": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_80": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_80": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_88": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_88": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_88": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "first", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_88": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_88": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_96": { + "block_sizes": [ + 128, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_96": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_96": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_96": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_96": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_96": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_104": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_104": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_104": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_104": { + "block_sizes": [ + 8, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_104": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_104": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_112": { + "block_sizes": [ + 32, + 1024 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_112": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_112": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_112": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_112": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_120": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_120": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_120": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_120": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_128": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_128": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_128": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_128": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_128": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_136": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 3, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_136": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_136": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_136": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_136": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 3, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_136": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_144": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_144": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_144": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_144": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_144": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_144": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_152": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_152": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_152": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_152": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_152": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_160": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_160": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_160": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_160": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_160": { + "block_sizes": [ + 128, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_168": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_168": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_168": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_168": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_168": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_168": { + "block_sizes": [ + 32, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_176": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_176": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_176": { + "block_sizes": [ + 4, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_176": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_176": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_184": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_184": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_184": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_184": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_184": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_192": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_192": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_192": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_192": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_192": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_192": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_200": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_200": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_200": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_200": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_208": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_208": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_208": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "last", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_208": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_216": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_216": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_216": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_216": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_216": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_216": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_224": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_224": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_224": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_224": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_224": { + "block_sizes": [ + 256, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_224": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_232": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_232": { + "block_sizes": [ + 64, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_232": { + "block_sizes": [ + 16, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_232": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_232": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_232": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_240": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_240": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_240": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_240": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_248": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_248": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_248": { + "block_sizes": [ + 256, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_248": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_248": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_248": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_272": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 1, + 0 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_272": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_272": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_272": { + "block_sizes": [ + 64, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_288": { + "block_sizes": [ + 4, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_288": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_288": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_288": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_288": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_288": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_304": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_304": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_304": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_304": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_304": { + "block_sizes": [ + 64, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_320": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_320": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_320": { + "block_sizes": [ + 512, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_320": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_320": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_320": { + "block_sizes": [ + 128, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_336": { + "block_sizes": [ + 2, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_336": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_336": { + "block_sizes": [ + 64, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_336": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_336": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_352": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_352": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_352": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_352": { + "block_sizes": [ + 8, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_352": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_368": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_368": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_368": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_368": { + "block_sizes": [ + 32, + 4 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_368": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_384": { + "block_sizes": [ + 64, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_384": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_384": { + "block_sizes": [ + 8, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_384": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_400": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_400": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_400": { + "block_sizes": [ + 256, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_400": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_416": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_416": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_416": { + "block_sizes": [ + 64, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_416": { + "block_sizes": [ + 128, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_416": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_416": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_432": { + "block_sizes": [ + 16, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_432": { + "block_sizes": [ + 32, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_432": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 4, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_432": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_432": { + "block_sizes": [ + 16, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_432": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_448": { + "block_sizes": [ + 8, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_448": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_448": { + "block_sizes": [ + 32, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_448": { + "block_sizes": [ + 16, + 256 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_448": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "last", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_464": { + "block_sizes": [ + 32, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_464": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_464": { + "block_sizes": [ + 16, + 64 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_464": { + "block_sizes": [ + 8, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "first" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_464": { + "block_sizes": [ + 128, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_480": { + "block_sizes": [ + 4, + 16 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 2, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_480": { + "block_sizes": [ + 4, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "first" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_480": { + "block_sizes": [ + 8, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_480": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_480": { + "block_sizes": [ + 64, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "last" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_480": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "last", + "", + "" + ], + "num_warps": 8, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_496": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "first", + "" + ], + "num_warps": 8, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_496": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 4 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 2, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "tensor_descriptor", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_496": { + "block_sizes": [ + 32, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "tensor_descriptor" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_496": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 4, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_496": { + "block_sizes": [ + 256, + 8 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2048_numtokens_512": { + "block_sizes": [ + 32, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "last" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_2880_numtokens_512": { + "block_sizes": [ + 16, + 32 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 1, + "indexing": [ + "pointer", + "tensor_descriptor", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_4096_numtokens_512": { + "block_sizes": [ + 128, + 512 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 16, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_8192_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + false + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 32, + "num_stages": 1, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_11008_numtokens_512": { + "block_sizes": [ + 32, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 1 + ], + "range_unroll_factors": [ + 0 + ], + "range_warp_specializes": [], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "", + "", + "" + ], + "num_warps": 1, + "num_stages": 1, + "indexing": [ + "tensor_descriptor", + "pointer", + "pointer", + "pointer" + ], + "pid_type": "flat" + }, + "intermediate_14336_numtokens_512": { + "block_sizes": [ + 16, + 128 + ], + "loop_orders": [ + [ + 0, + 1 + ] + ], + "flatten_loops": [ + true + ], + "l2_groupings": [ + 2 + ], + "range_unroll_factors": [ + 0 + ], + "range_num_stages": [ + 0 + ], + "range_multi_buffers": [ + null + ], + "range_flattens": [ + null + ], + "load_eviction_policies": [ + "first", + "", + "" + ], + "num_warps": 1, + "num_stages": 2, + "indexing": [ + "pointer", + "pointer", + "pointer", + "pointer" + ], "pid_type": "flat", "range_warp_specializes": [] } } -} \ No newline at end of file +} diff --git a/vllm/kernels/helion/ops/silu_mul_fp8.py b/vllm/kernels/helion/ops/silu_mul_fp8.py index a45943b1a69..954f5df3abf 100644 --- a/vllm/kernels/helion/ops/silu_mul_fp8.py +++ b/vllm/kernels/helion/ops/silu_mul_fp8.py @@ -3,6 +3,7 @@ from typing import Any +import regex as re import torch from vllm.logger import init_logger @@ -53,44 +54,78 @@ def silu_mul_fp8(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: return out.view(output_shape) +@silu_mul_fp8.register_input_generator # type: ignore[misc] +def generate_silu_mul_fp8_inputs() -> dict[str, tuple[Any, ...]]: + intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336] + + # Use the same num_tokens values as vLLM's default cudagraph capture sizes. + # See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula. + num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16)) + + inputs = {} + for num_tokens in num_tokens_list: + for intermediate_size in intermediate_sizes: + # Input tensor has shape (num_tokens, 2 * intermediate_size) + # because silu_mul splits it into two halves + input_tensor = torch.randn( + num_tokens, + 2 * intermediate_size, + device="cuda", + dtype=torch.bfloat16, + ) + scale = torch.tensor([1.0], device="cuda", dtype=torch.float32) + + config_key = f"intermediate_{intermediate_size}_numtokens_{num_tokens}" + inputs[config_key] = (input_tensor, scale) + + return inputs + + @silu_mul_fp8.register_config_picker # type: ignore[misc] def pick_silu_mul_fp8_config( args: tuple[Any, ...], config_keys: list[str] ) -> str | None: + """Pick the best pre-tuned config for the given input shape. + + Selection strategy: + 1. Find the closest intermediate_size among available configs + (exact match preferred). + 2. Among the num_tokens values tuned for that intermediate_size, pick + the smallest num_tokens >= the input's num_tokens. If the input is + larger than all available num_tokens, fall back to the largest. + + Config keys must be "default" or follow the format + "intermediate_{int}_numtokens_{int}". + """ if not config_keys: return None - input_tensor, scale = args + input_tensor, _scale = args intermediate_size = input_tensor.shape[-1] // 2 - - # TODO(gmagosfm): Rerun autotuning to capture config for - # other batch sizes. - target_key = f"intermediate_{intermediate_size}_batchsize_256" - if target_key in config_keys: - return target_key - - intermediate_sizes = [] + num_tokens = input_tensor.view(-1, input_tensor.shape[-1]).shape[0] + configs: dict[int, list[int]] = {} for key in config_keys: - if key.startswith("intermediate_") and "_batchsize_256" in key: - try: - size_str = key.split("_")[1] - size = int(size_str) - intermediate_sizes.append((abs(size - intermediate_size), key)) - except (ValueError, IndexError): - continue + if key == "default": + continue + match = re.fullmatch(r"intermediate_(\d+)_numtokens_(\d+)", key) + if not match: + raise ValueError( + f"Malformed config key '{key}', " + f"expected format 'intermediate_{{int}}_numtokens_{{int}}'" + ) + isize_str, ntokens_str = match.groups() + configs.setdefault(int(isize_str), []).append(int(ntokens_str)) - if intermediate_sizes: - _, best_key = min(intermediate_sizes) - logger.debug( - "No exact config for intermediate_size=%d, using closest match: %s", - intermediate_size, - best_key, - ) - return best_key - if "default" in config_keys: - return "default" + if not configs: + return "default" if "default" in config_keys else None - return None + best_isize = min(configs, key=lambda s: abs(s - intermediate_size)) + available_ntokens = sorted(configs[best_isize]) + best_ntokens = next( + (n for n in available_ntokens if n >= num_tokens), available_ntokens[-1] + ) + + return f"intermediate_{best_isize}_numtokens_{best_ntokens}" def silu_mul_fp8_baseline(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: From e4a5d8c653fc00adb06922bddcb7fec14b01a62b Mon Sep 17 00:00:00 2001 From: Zhengxu Chen Date: Fri, 20 Feb 2026 11:46:45 -0500 Subject: [PATCH 32/74] [compile] Move torch_aot_compile directory under torch_compile_cache (#34831) Signed-off-by: zhxchen17 --- vllm/compilation/decorators.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/vllm/compilation/decorators.py b/vllm/compilation/decorators.py index 3651c835f7e..f97467ad6fc 100644 --- a/vllm/compilation/decorators.py +++ b/vllm/compilation/decorators.py @@ -407,10 +407,10 @@ def _support_torch_compile( if envs.VLLM_USE_AOT_COMPILE: """ When using torch.compile in AOT mode, we store the cache artifacts - under VLLM_CACHE_ROOT/torch_aot_compile/{hash}/rank_i_j. The {hash} - contains all of the factors except for the source files being - traced through, because we don't actually know which source files - to check at this point (before dynamo runs). + under VLLM_CACHE_ROOT/torch_compile_cache/torch_aot_compile/{hash} + The {hash} contains all of the factors except for the source files + being traced through, because we don't actually know which source + files to check at this point (before dynamo runs). On loading we will actually look at the source files being traced through. If any source file have changed (compared with the serialized backend artifacts), then we need to generate a new AOT @@ -424,6 +424,7 @@ def _support_torch_compile( hash_key = hashlib.sha256(str(factors).encode()).hexdigest() cache_dir = os.path.join( envs.VLLM_CACHE_ROOT, + "torch_compile_cache", "torch_aot_compile", hash_key, ) From f8639940844bcc10e3f374d2bb5aa33ae52a2624 Mon Sep 17 00:00:00 2001 From: Zhengxu Chen Date: Fri, 20 Feb 2026 11:47:14 -0500 Subject: [PATCH 33/74] [compile] Fix torch.compile time discrepancy in logging. (#34912) Signed-off-by: zhxchen17 Co-authored-by: Cyrus Leung --- vllm/compilation/backends.py | 10 ++++------ vllm/compilation/monitor.py | 5 +++-- vllm/compilation/piecewise_backend.py | 10 ++++++++++ 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/vllm/compilation/backends.py b/vllm/compilation/backends.py index 50d0df589c2..09fd1f75091 100644 --- a/vllm/compilation/backends.py +++ b/vllm/compilation/backends.py @@ -249,7 +249,7 @@ class CompilerManager: if graph_index == 0: # before compiling the first graph, record the start time global compilation_start_time - compilation_start_time = time.time() + compilation_start_time = time.perf_counter() compilation_counter.num_backend_compilations += 1 @@ -261,8 +261,7 @@ class CompilerManager: if graph_index == num_graphs - 1: # after loading the last graph for this shape, record the time. # there can be multiple graphs due to piecewise compilation. - now = time.time() - elapsed = now - compilation_start_time + elapsed = time.perf_counter() - compilation_start_time compilation_config.compilation_time += elapsed logger.info_once( "Directly load the compiled graph(s) for compile range %s " @@ -362,8 +361,7 @@ class CompilerManager: # after compiling the last graph, record the end time if graph_index == num_graphs - 1: - now = time.time() - elapsed = now - compilation_start_time + elapsed = time.perf_counter() - compilation_start_time compilation_config.compilation_time += elapsed logger.info_once( "Compiling a graph for compile range %s takes %.2f s", @@ -974,7 +972,7 @@ class VllmBackend: compilation_counter.num_graphs_seen += 1 from .monitor import torch_compile_start_time - dynamo_time = time.time() - torch_compile_start_time + dynamo_time = time.perf_counter() - torch_compile_start_time logger.info_once( "Dynamo bytecode transform time: %.2f s", dynamo_time, scope="local" ) diff --git a/vllm/compilation/monitor.py b/vllm/compilation/monitor.py index 2bad5f0a16f..43b9ae508a5 100644 --- a/vllm/compilation/monitor.py +++ b/vllm/compilation/monitor.py @@ -14,7 +14,7 @@ torch_compile_start_time: float = 0.0 def start_monitoring_torch_compile(vllm_config: VllmConfig) -> None: global torch_compile_start_time - torch_compile_start_time = time.time() + torch_compile_start_time = time.perf_counter() compilation_config: CompilationConfig = vllm_config.compilation_config path = vllm_config.compile_debug_dump_path() @@ -30,10 +30,11 @@ def start_monitoring_torch_compile(vllm_config: VllmConfig) -> None: def end_monitoring_torch_compile(vllm_config: VllmConfig) -> None: compilation_config: CompilationConfig = vllm_config.compilation_config + total_compile_time: float = time.perf_counter() - torch_compile_start_time if compilation_config.mode == CompilationMode.VLLM_COMPILE: logger.info_once( "torch.compile takes %.2f s in total", - compilation_config.compilation_time, + total_compile_time, scope="local", ) global context_manager diff --git a/vllm/compilation/piecewise_backend.py b/vllm/compilation/piecewise_backend.py index 4f6ae250535..f9eb245893d 100644 --- a/vllm/compilation/piecewise_backend.py +++ b/vllm/compilation/piecewise_backend.py @@ -5,6 +5,7 @@ import dataclasses import io import json import pickle +import time from collections.abc import Callable from pickle import Pickler from typing import Any @@ -164,7 +165,16 @@ class PiecewiseBackend: if self.is_last_graph and not self.to_be_compiled_ranges: # no specific sizes to compile # save the hash of the inductor graph for the next run + time_before_saving = time.perf_counter() self.vllm_backend.compiler_manager.save_to_file() + elapsed = time.perf_counter() - time_before_saving + if elapsed > 1: + logger.info_once( + "Saved compiler manager cache in %.2f seconds.", + elapsed, + scope="local", + ) + end_monitoring_torch_compile(self.vllm_config) # Call the completion callback (e.g., to save AOT compiled function) if self.on_compilation_complete is not None: From fac1507f03c78d8717853c8a15ad1d887d71cc1d Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Fri, 20 Feb 2026 13:17:42 -0500 Subject: [PATCH 34/74] [CI] Remove failing prime-rl integration test (#34843) Signed-off-by: mgoin Signed-off-by: Michael Goin --- .buildkite/scripts/run-prime-rl-test.sh | 64 ---------------------- .buildkite/test-amd.yaml | 29 ---------- .buildkite/test_areas/e2e_integration.yaml | 13 ----- 3 files changed, 106 deletions(-) delete mode 100755 .buildkite/scripts/run-prime-rl-test.sh diff --git a/.buildkite/scripts/run-prime-rl-test.sh b/.buildkite/scripts/run-prime-rl-test.sh deleted file mode 100755 index a3f2bf8bf40..00000000000 --- a/.buildkite/scripts/run-prime-rl-test.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -# Setup script for Prime-RL integration tests -# This script prepares the environment for running Prime-RL tests with nightly vLLM - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -PRIME_RL_REPO="https://github.com/PrimeIntellect-ai/prime-rl.git" -PRIME_RL_DIR="${REPO_ROOT}/prime-rl" - -if command -v rocm-smi &> /dev/null || command -v rocminfo &> /dev/null; then - echo "AMD GPU detected. Prime-RL currently only supports NVIDIA. Skipping..." - exit 0 -fi - -echo "Setting up Prime-RL integration test environment..." - -# Clean up any existing Prime-RL directory -if [ -d "${PRIME_RL_DIR}" ]; then - echo "Removing existing Prime-RL directory..." - rm -rf "${PRIME_RL_DIR}" -fi - -# Install UV if not available -if ! command -v uv &> /dev/null; then - echo "Installing UV package manager..." - curl -LsSf https://astral.sh/uv/install.sh | sh - source "$HOME"/.local/bin/env -fi - -# Clone Prime-RL repository at specific branch for reproducible tests -PRIME_RL_BRANCH="integ-vllm-main" -echo "Cloning Prime-RL repository at branch: ${PRIME_RL_BRANCH}..." -git clone --branch "${PRIME_RL_BRANCH}" --single-branch "${PRIME_RL_REPO}" "${PRIME_RL_DIR}" -cd "${PRIME_RL_DIR}" - -echo "Setting up UV project environment..." -export UV_PROJECT_ENVIRONMENT=/usr/local -ln -s /usr/bin/python3 /usr/local/bin/python - -# Remove vllm pin from pyproject.toml -echo "Removing vllm pin from pyproject.toml..." -sed -i '/vllm==/d' pyproject.toml - -# Sync Prime-RL dependencies -echo "Installing Prime-RL dependencies..." -uv sync --inexact && uv sync --inexact --all-extras - -# Verify installation -echo "Verifying installations..." -uv run python -c "import vllm; print(f'vLLM version: {vllm.__version__}')" -uv run python -c "import prime_rl; print('Prime-RL imported successfully')" - -echo "Prime-RL integration test environment setup complete!" - -echo "Running Prime-RL integration tests..." -export WANDB_MODE=offline # this makes this test not require a WANDB_API_KEY -uv run pytest -vs tests/integration/test_rl.py -m gpu - -echo "Prime-RL integration tests completed!" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 052c85c22d5..ba6edb92f40 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1622,21 +1622,6 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER=1 VLLM_USE_AITER_UNIFIED_ATTENTION=1 pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58 -##### RL Integration Tests ##### -- label: Prime-RL Integration Test # 15min - mirror_hardwares: [amdexperimental] - agent_pool: mi325_2 - # grade: Blocking - timeout_in_minutes: 30 - optional: true - num_gpus: 2 - working_dir: "/vllm-workspace" - source_file_dependencies: - - vllm/ - - .buildkite/scripts/run-prime-rl-test.sh - commands: - - bash .buildkite/scripts/run-prime-rl-test.sh - ##### EPLB Accuracy Tests ##### - label: DeepSeek V2-Lite Accuracy mirror_hardwares: [amdexperimental, amdproduction] @@ -3201,20 +3186,6 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - VLLM_ROCM_USE_AITER_MHA=0 VLLM_ROCM_USE_AITER=1 VLLM_USE_AITER_UNIFIED_ATTENTION=1 pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58 -##### RL Integration Tests ##### -- label: Prime-RL Integration Test # 15min - mirror_hardwares: [amdexperimental] - agent_pool: mi355_2 - timeout_in_minutes: 30 - optional: true - num_gpus: 2 - working_dir: "/vllm-workspace" - source_file_dependencies: - - vllm/ - - .buildkite/scripts/run-prime-rl-test.sh - commands: - - bash .buildkite/scripts/run-prime-rl-test.sh - ##### EPLB Accuracy Tests ##### - label: DeepSeek V2-Lite Accuracy mirror_hardwares: [amdexperimental, amdproduction] diff --git a/.buildkite/test_areas/e2e_integration.yaml b/.buildkite/test_areas/e2e_integration.yaml index 958bff5c95b..d95b73073d6 100644 --- a/.buildkite/test_areas/e2e_integration.yaml +++ b/.buildkite/test_areas/e2e_integration.yaml @@ -28,16 +28,3 @@ steps: working_dir: "/vllm-workspace" commands: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1 - -- label: Prime-RL Integration (2 GPUs) - timeout_in_minutes: 30 - optional: true - soft_fail: true - num_devices: 2 - working_dir: "/vllm-workspace" - source_file_dependencies: - - vllm/ - - .buildkite/scripts/run-prime-rl-test.sh - commands: - - nvidia-smi - - bash .buildkite/scripts/run-prime-rl-test.sh From f24b2de3d3812301932645e3002adba46a8c0055 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:51:58 -0500 Subject: [PATCH 35/74] [Test] Add FP8 KV Cache Testing for MLA Backends (#34473) Signed-off-by: wzhao18 --- tests/v1/attention/test_mla_backends.py | 95 ++++++++++++++++++------- 1 file changed, 68 insertions(+), 27 deletions(-) diff --git a/tests/v1/attention/test_mla_backends.py b/tests/v1/attention/test_mla_backends.py index ba70c825174..32c0b906427 100644 --- a/tests/v1/attention/test_mla_backends.py +++ b/tests/v1/attention/test_mla_backends.py @@ -19,8 +19,13 @@ from tests.v1.attention.utils import ( ) from vllm import _custom_ops as ops from vllm.config.vllm import set_current_vllm_config -from vllm.model_executor.layers.attention.mla_attention import QueryLenSupport +from vllm.model_executor.layers.attention.mla_attention import ( + QueryLenSupport, + _DecodeConcatQuantFP8, +) from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase +from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape +from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE from vllm.v1.attention.backend import CommonAttentionMetadata @@ -50,6 +55,7 @@ if not flash_attn_supports_mla(): if not is_flashmla_dense_supported()[0]: BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHMLA) + SPEC_DECODE_BACKENDS = [] for backend in BACKENDS_TO_TEST: builder_cls, _ = try_get_attention_backend(backend) @@ -144,9 +150,8 @@ def create_and_prepopulate_kv_cache( common_attn_metadata: Common attention metadata randomize_blocks: Whether to randomly permute blocks or use sequential order - kv_cache_dtype: Optional kv cache dtype string. When set to - "fp8_ds_mla" the cache is populated using the - fp8 DeepSeek MLA layout via concat_and_cache_mla. + kv_cache_dtype: Optional kv cache dtype string. For fp8 cache dtype, + the cache is populated via concat_and_cache_mla. scale: Scaling factor forwarded to concat_and_cache_mla when the fp8 cache layout is requested. @@ -163,18 +168,21 @@ def create_and_prepopulate_kv_cache( block_table = common_attn_metadata.block_table_tensor slot_mapping = common_attn_metadata.slot_mapping + fp8_attention = kv_cache_dtype and kv_cache_dtype.startswith("fp8") use_fp8_ds_mla = kv_cache_dtype == "fp8_ds_mla" - if use_fp8_ds_mla: - if not kv_c_contexts: - raise ValueError( - "kv_c_contexts cannot be empty when using fp8_ds_mla cache dtype" - ) - kv_lora_rank = kv_c_contexts[0].shape[-1] - rope_dim = k_pe_contexts[0].shape[-1] - entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim + if fp8_attention: + if use_fp8_ds_mla: + kv_lora_rank = kv_c_contexts[0].shape[-1] + rope_dim = k_pe_contexts[0].shape[-1] + # 4 * 4: 4 float32 scale values for 128-element tiles + # 2 * rope_dim: 16-bit RoPE values + kv_entry_size = kv_lora_rank + 4 * 4 + 2 * rope_dim + else: + kv_entry_size = head_size + kv_cache = torch.zeros( - num_blocks, block_size, entry_size, dtype=torch.uint8, device=device + num_blocks, block_size, kv_entry_size, dtype=torch.uint8, device=device ) scale_tensor = ( scale @@ -201,14 +209,14 @@ def create_and_prepopulate_kv_cache( start = start_block_idx * block_size - if use_fp8_ds_mla: + if fp8_attention: slots = torch.arange(context_len, device=device, dtype=torch.long) + start ops.concat_and_cache_mla( kv_c_context, k_pe_context.squeeze(1), kv_cache, slots, - kv_cache_dtype="fp8_ds_mla", + kv_cache_dtype=kv_cache_dtype, scale=scale_tensor, ) else: @@ -329,8 +337,9 @@ class MockSparseMLAAttentionLayer: output: torch.Tensor, ) -> torch.Tensor: """Forward for sparse MLA - uses forward_mqa for all tokens.""" - # Write to KV cache kv_cache_dtype = getattr(self.impl, "kv_cache_dtype", "auto") + + # Write to KV cache if kv_cache.numel() > 0: ops.concat_and_cache_mla( kv_c, @@ -426,6 +435,12 @@ class MockMLAAttentionLayer(AttentionLayerBase): self._k_scale_float = 1.0 self._v_scale_float = 1.0 + self._decode_concat_quant_fp8_op = _DecodeConcatQuantFP8( + static=True, + group_shape=GroupShape.PER_TENSOR, + compile_native=True, + ) + def get_attn_backend(self): raise NotImplementedError @@ -443,16 +458,21 @@ class MockMLAAttentionLayer(AttentionLayerBase): ) -> torch.Tensor: """Replicates MLAAttention.forward_impl logic for testing.""" # Write to KV cache + kv_cache_dtype = getattr(self.impl, "kv_cache_dtype", "auto") + fp8_attention = kv_cache_dtype.startswith("fp8") if kv_cache.numel() > 0: ops.concat_and_cache_mla( kv_c, k_pe.squeeze(1), kv_cache, attn_metadata.slot_mapping.flatten(), - kv_cache_dtype="auto", + kv_cache_dtype=kv_cache_dtype, scale=self._k_scale, ) + if fp8_attention and kv_cache_dtype != "fp8_ds_mla": + kv_cache = kv_cache.view(current_platform.fp8_dtype()) + # Determine decode vs prefill split num_decode_tokens = attn_metadata.num_decode_tokens or 0 has_decode = (attn_metadata.num_decodes or 0) > 0 @@ -491,8 +511,14 @@ class MockMLAAttentionLayer(AttentionLayerBase): # Convert from (N, B, L) to (B, N, L) mqa_ql_nope = mqa_ql_nope.transpose(0, 1) - # Pass as tuple to forward_mqa - mqa_q = (mqa_ql_nope, mqa_q_pe) + if fp8_attention and self.impl.supports_quant_query_input: + assert mqa_ql_nope.shape[0] == mqa_q_pe.shape[0] + assert mqa_ql_nope.shape[1] == mqa_q_pe.shape[1] + mqa_q = self._decode_concat_quant_fp8_op( + mqa_ql_nope, mqa_q_pe, self._q_scale + ) + else: + mqa_q = (mqa_ql_nope, mqa_q_pe) attn_out, _ = self.impl.forward_mqa(mqa_q, kv_cache, attn_metadata, self) @@ -526,6 +552,7 @@ def run_attention_backend( qk_rope_head_dim: int, v_head_dim: int, mock_kv_b_proj, + kv_cache_dtype: str = "auto", ) -> torch.Tensor: """Run attention computation using the specified backend's AttentionImpl.""" @@ -550,7 +577,7 @@ def run_attention_backend( num_kv_heads=num_kv_heads, alibi_slopes=None, sliding_window=None, - kv_cache_dtype="auto", + kv_cache_dtype=kv_cache_dtype, logits_soft_cap=None, attn_type="decoder", kv_sharing_target_layer_name=None, @@ -630,12 +657,14 @@ def run_attention_backend( ) @pytest.mark.parametrize("model", ["deepseek-ai/DeepSeek-R1"]) @pytest.mark.parametrize("tensor_parallel_size", [1, 4, 8, 16]) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"]) def test_backend_correctness( default_vllm_config, dist_init, batch_spec_name: str, model: str, tensor_parallel_size: int, + kv_cache_dtype: str, ): """ Test that all backends produce similar outputs to a reference implementation @@ -658,9 +687,18 @@ def test_backend_correctness( head counts. """ + # Filter backends to those that support the requested kv_cache_dtype + backends_to_test = [ + b + for b in BACKENDS_TO_TEST + if kv_cache_dtype in b.get_class().supported_kv_cache_dtypes + ] + if not backends_to_test: + pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}") + batch_spec = BATCH_SPECS[batch_spec_name] is_spec_decode_test = batch_spec_name.startswith("spec_decode") - unique_block_sizes = sorted(set(BACKEND_BLOCK_SIZES.values())) + unique_block_sizes = sorted(set(BACKEND_BLOCK_SIZES[b] for b in backends_to_test)) default_block_size = unique_block_sizes[0] required_blocks = sum( (seq_len + default_block_size - 1) // default_block_size @@ -694,6 +732,7 @@ def test_backend_correctness( block_size=default_block_size, hf_config_override=hf_config_override, ) + vllm_config.cache_config.cache_dtype = kv_cache_dtype # For spec decode tests, add a speculative_config to set the reorder_batch_threshold if is_spec_decode_test: @@ -751,7 +790,7 @@ def test_backend_correctness( kv_b_proj_weight = torch.cat([W_UK, W_UV], dim=-1) - for i, backend in enumerate(BACKENDS_TO_TEST): + for i, backend in enumerate(backends_to_test): all_sdpa_outputs.append([]) for i in range(batch_size): @@ -785,7 +824,7 @@ def test_backend_correctness( # pipeline (MHA-style). This ensures the reference implementation # matches each backend's actual decode/prefill pipeline path. is_decode = [] - for backend_idx, backend in enumerate(BACKENDS_TO_TEST): + for backend_idx, backend in enumerate(backends_to_test): builder_cls, _ = try_get_attention_backend(backend) if is_spec_decode_test: query_len_support = getattr( @@ -885,7 +924,7 @@ def test_backend_correctness( sdpa_out_i_prefill = sdpa_out_i_prefill.transpose(1, 2).squeeze(0) sdpa_out_i_prefill = sdpa_out_i_prefill.flatten(start_dim=-2) - for backend_idx, backend in enumerate(BACKENDS_TO_TEST): + for backend_idx, backend in enumerate(backends_to_test): if is_decode[backend_idx]: all_sdpa_outputs[backend_idx].append(sdpa_out_i_decode) else: @@ -905,7 +944,7 @@ def test_backend_correctness( kv_c_vllm = torch.cat(all_kv_c_vllm, dim=0) k_pe_vllm = torch.cat(all_k_pe_vllm, dim=0) sdpa_outputs = {} - for backend_idx, backend in enumerate(BACKENDS_TO_TEST): + for backend_idx, backend in enumerate(backends_to_test): sdpa_outputs[backend] = torch.cat(all_sdpa_outputs[backend_idx], dim=0) # Create mock kv_b_proj using the same weights as reference implementation @@ -973,12 +1012,13 @@ def test_backend_correctness( num_blocks=num_blocks_for_size, common_attn_metadata=common_attn_metadata, randomize_blocks=True, + kv_cache_dtype=kv_cache_dtype, ) kv_cache_per_block_size[block_size] = kv_cache # 4. Run vLLM backends and compare failures = [] - for backend_idx, backend_name in enumerate(BACKENDS_TO_TEST): + for backend_idx, backend_name in enumerate(backends_to_test): # Skip backends that don't support spec decode for spec decode tests if is_spec_decode_test and backend_name not in SPEC_DECODE_BACKENDS: continue @@ -997,7 +1037,7 @@ def test_backend_correctness( head_size=vllm_config.model_config.get_head_size(), dtype=vllm_config.model_config.dtype, sliding_window=vllm_config.model_config.get_sliding_window(), - cache_dtype_str=vllm_config.cache_config.cache_dtype, + cache_dtype_str=kv_cache_dtype, ) backend_output = run_attention_backend( @@ -1016,6 +1056,7 @@ def test_backend_correctness( qk_rope_head_dim, v_head_dim, mock_kv_b_proj, + kv_cache_dtype=kv_cache_dtype, ) # Use backend_idx to get the correct SDPA output for this backend From aaefc58ee0f023ec7bd3671ca83aae1b8a8f271d Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 20 Feb 2026 16:25:50 -0500 Subject: [PATCH 36/74] [CI] Revert PRs 34818 and 33600 (#34979) --- .../processing/test_tensor_schema.py | 5 +- tests/models/utils.py | 9 +- tests/v1/spec_decode/test_eagle.py | 2 +- vllm/config/cache.py | 11 +- vllm/config/vllm.py | 97 +++-- vllm/engine/arg_utils.py | 3 +- .../attention/chunked_local_attention.py | 11 +- .../layers/attention/mla_attention.py | 17 +- vllm/platforms/cuda.py | 344 +++++++++--------- vllm/platforms/interface.py | 7 - vllm/v1/attention/backend.py | 19 +- vllm/v1/engine/core.py | 9 +- vllm/v1/executor/multiproc_executor.py | 4 - vllm/v1/executor/ray_executor.py | 5 - vllm/v1/executor/uniproc_executor.py | 2 - vllm/v1/worker/gpu_model_runner.py | 5 +- 16 files changed, 249 insertions(+), 301 deletions(-) diff --git a/tests/models/multimodal/processing/test_tensor_schema.py b/tests/models/multimodal/processing/test_tensor_schema.py index c81a8fe09d3..8f79936478d 100644 --- a/tests/models/multimodal/processing/test_tensor_schema.py +++ b/tests/models/multimodal/processing/test_tensor_schema.py @@ -13,7 +13,6 @@ import torch.nn as nn from PIL import Image from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config -from vllm.config.cache import CacheConfig from vllm.config.multimodal import ( AudioDummyOptions, BaseDummyOptions, @@ -132,9 +131,7 @@ def initialize_dummy_model( ): temp_file = tempfile.mkstemp()[1] current_device = torch.get_default_device() - vllm_config = VllmConfig( - model_config=model_config, cache_config=CacheConfig(block_size=16) - ) + vllm_config = VllmConfig(model_config=model_config) with set_current_vllm_config(vllm_config=vllm_config): init_distributed_environment( world_size=1, diff --git a/tests/models/utils.py b/tests/models/utils.py index 8c1fb63d67a..4830f18dccf 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -457,9 +457,6 @@ def dummy_hf_overrides( # Kimi uses `num_expert_group` instead of `n_group`. if n_group is None: n_group = getattr(text_config, "num_expert_group", None) - # InternS1Pro uses `router_n_groups` instead of `n_group`. - if n_group is None: - n_group = getattr(text_config, "router_n_groups", None) num_experts = n_group * 2 if n_group is not None else 2 # we use three layers for Gemma-3n to check @@ -489,14 +486,12 @@ def dummy_hf_overrides( # Only set MoE related config when the model has MoE layers. # Otherwise all models detected as MoE by _get_transformers_backend_cls. if model_arch_config.num_experts > 0: - orig_topk = getattr(text_config, "num_experts_per_tok", 2) - topk = min(orig_topk, 2) update_dict.update( { "num_experts": num_experts, - "num_experts_per_tok": topk, + "num_experts_per_tok": 2, # Kimi uses `num_experts_per_token`. - "num_experts_per_token": topk, + "num_experts_per_token": 2, "num_local_experts": num_experts, # Otherwise there will not be any expert layers "first_k_dense_replace": 0, diff --git a/tests/v1/spec_decode/test_eagle.py b/tests/v1/spec_decode/test_eagle.py index 65e97b7ad5b..8b180168dff 100644 --- a/tests/v1/spec_decode/test_eagle.py +++ b/tests/v1/spec_decode/test_eagle.py @@ -78,7 +78,7 @@ def _create_proposer( device = current_platform.device_type vllm_config = VllmConfig( model_config=model_config, - cache_config=CacheConfig(block_size=16), + cache_config=CacheConfig(), speculative_config=speculative_config, device_config=DeviceConfig(device=device), parallel_config=ParallelConfig(), diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 313a4577b50..daceaa6c2bb 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -19,6 +19,7 @@ else: logger = init_logger(__name__) +BlockSize = Literal[1, 8, 16, 32, 64, 128, 256] CacheDType = Literal[ "auto", "bfloat16", @@ -38,11 +39,13 @@ KVOffloadingBackend = Literal["native", "lmcache"] class CacheConfig: """Configuration for the KV cache.""" - block_size: SkipValidation[int] = None # type: ignore[assignment] - """Size of a contiguous cache block in number of tokens. + block_size: SkipValidation[BlockSize] = None # type: ignore[assignment] + """Size of a contiguous cache block in number of tokens. On CUDA devices, + only block sizes up to 32 are supported. - This is None until the platform sets it. Always an int by the time - the engine starts.""" + This config has no static default. If left unspecified by the user, it will + be set in `Platform.check_and_update_config()` based on the current + platform.""" gpu_memory_utilization: float = Field(default=0.9, gt=0, le=1) """The fraction of GPU memory to be used for the model executor, which can range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index fffe769e7a4..e951e6f2c8a 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -915,6 +915,32 @@ class VllmConfig: ) current_platform.check_and_update_config(self) + # If DCP, ensure the block size is right. + if self.parallel_config.decode_context_parallel_size > 1: + if self.parallel_config.dcp_kv_cache_interleave_size > 1 and ( + self.parallel_config.cp_kv_cache_interleave_size + != self.parallel_config.dcp_kv_cache_interleave_size + ): + self.parallel_config.cp_kv_cache_interleave_size = ( + self.parallel_config.dcp_kv_cache_interleave_size + ) + logger.warning_once( + "cp_kv_cache_interleave_size is overridden by dcp_kv_cache" + "_interleave_size. And dcp-kv-cache-interleave-size will be " + "deprecated when PCP is fully supported." + ) + assert ( + self.parallel_config.cp_kv_cache_interleave_size + <= self.cache_config.block_size + and self.cache_config.block_size + % self.parallel_config.cp_kv_cache_interleave_size + == 0 + ), ( + f"Block_size({self.cache_config.block_size}) should be greater " + "than or equal to and divisible by cp_kv_cache_interleave_size " + f"({self.parallel_config.cp_kv_cache_interleave_size})." + ) + # Do this after all the updates to compilation_config.mode effective_dp_size = ( self.parallel_config.data_parallel_size @@ -1082,6 +1108,26 @@ class VllmConfig: # Default to enable HMA if not explicitly disabled by user or logic above. self.scheduler_config.disable_hybrid_kv_cache_manager = False + if self.cache_config.mamba_cache_mode == "align": + assert ( + self.cache_config.block_size + <= self.scheduler_config.max_num_batched_tokens + ), ( + "In Mamba cache align mode, block_size " + f"({self.cache_config.block_size}) must be <= " + "max_num_batched_tokens " + f"({self.scheduler_config.max_num_batched_tokens})." + ) + if self.scheduler_config.long_prefill_token_threshold > 0: + assert ( + self.scheduler_config.long_prefill_token_threshold + >= self.cache_config.block_size + ) + assert not self.scheduler_config.disable_chunked_mm_input, ( + "Chunked MM input is required because we need the flexibility to " + "schedule a multiple of block_size tokens even if they are in the " + "middle of a mm input" + ) if self.compilation_config.debug_dump_path: self.compilation_config.debug_dump_path = ( self.compilation_config.debug_dump_path.absolute().expanduser() @@ -1442,57 +1488,6 @@ class VllmConfig: f"compilation_config={self.compilation_config!r}" ) - def validate_block_size(self) -> None: - """Validate block_size against DCP and mamba constraints. - - Called after Platform.update_block_size_for_backend() has - finalised block_size, so that the checks see the real value - rather than the initial None sentinel. - """ - block_size = self.cache_config.block_size - assert block_size is not None, ( - "validate_block_size called before block_size was set" - ) - - # DCP interleave-size compatibility - if self.parallel_config.decode_context_parallel_size > 1: - if self.parallel_config.dcp_kv_cache_interleave_size > 1 and ( - self.parallel_config.cp_kv_cache_interleave_size - != self.parallel_config.dcp_kv_cache_interleave_size - ): - self.parallel_config.cp_kv_cache_interleave_size = ( - self.parallel_config.dcp_kv_cache_interleave_size - ) - logger.warning_once( - "cp_kv_cache_interleave_size is overridden by dcp_kv_cache" - "_interleave_size. And dcp-kv-cache-interleave-size will be " - "deprecated when PCP is fully supported." - ) - assert ( - self.parallel_config.cp_kv_cache_interleave_size <= block_size - and block_size % self.parallel_config.cp_kv_cache_interleave_size == 0 - ), ( - f"Block_size({block_size}) should be greater " - "than or equal to and divisible by cp_kv_cache_interleave_size " - f"({self.parallel_config.cp_kv_cache_interleave_size})." - ) - - # Mamba cache align-mode constraints - if self.cache_config.mamba_cache_mode == "align": - assert block_size <= self.scheduler_config.max_num_batched_tokens, ( - "In Mamba cache align mode, block_size " - f"({block_size}) must be <= " - "max_num_batched_tokens " - f"({self.scheduler_config.max_num_batched_tokens})." - ) - if self.scheduler_config.long_prefill_token_threshold > 0: - assert self.scheduler_config.long_prefill_token_threshold >= block_size - assert not self.scheduler_config.disable_chunked_mm_input, ( - "Chunked MM input is required because we need the flexibility " - "to schedule a multiple of block_size tokens even if they are " - "in the middle of a mm input" - ) - @model_validator(mode="after") def validate_mamba_block_size(self) -> "VllmConfig": if self.model_config is None: diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 1d9a924bdce..8ea96de4913 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -59,6 +59,7 @@ from vllm.config import ( get_attr_docs, ) from vllm.config.cache import ( + BlockSize, CacheDType, KVOffloadingBackend, MambaCacheMode, @@ -430,7 +431,7 @@ class EngineArgs: max_parallel_loading_workers: int | None = ( ParallelConfig.max_parallel_loading_workers ) - block_size: int = None # type: ignore[assignment] + block_size: BlockSize = CacheConfig.block_size enable_prefix_caching: bool | None = None prefix_caching_hash_algo: PrefixCachingHashAlgo = ( CacheConfig.prefix_caching_hash_algo diff --git a/vllm/model_executor/layers/attention/chunked_local_attention.py b/vllm/model_executor/layers/attention/chunked_local_attention.py index 522981820d6..e33733c0cc1 100644 --- a/vllm/model_executor/layers/attention/chunked_local_attention.py +++ b/vllm/model_executor/layers/attention/chunked_local_attention.py @@ -30,8 +30,9 @@ from vllm.v1.kv_cache_interface import ( def create_chunked_local_attention_backend( underlying_attn_backend: AttentionBackend, attention_chunk_size: int, + block_size: int, ) -> type[AttentionBackend]: - prefix = f"ChunkedLocalAttention_{attention_chunk_size}_" + prefix = f"ChunkedLocalAttention_{attention_chunk_size}_{block_size}_" underlying_builder = underlying_attn_backend.get_builder_cls() assert issubclass(underlying_builder, AttentionMetadataBuilder) @@ -54,9 +55,7 @@ def create_chunked_local_attention_backend( fast_build: bool = False, ): cm, make_virtual_batches_block_table = make_local_attention_virtual_batches( - attention_chunk_size, - common_attn_metadata, - self.kv_cache_spec.block_size, + attention_chunk_size, common_attn_metadata, block_size ) metadata = super().build(common_prefix_len, cm, fast_build) metadata.make_virtual_batches_block_table = make_virtual_batches_block_table @@ -98,13 +97,13 @@ class ChunkedLocalAttention(Attention): block_size = cache_config.block_size else: kv_cache_dtype = "auto" - block_size = None + block_size = 16 underlying_attn_backend = get_attn_backend( head_size, dtype, kv_cache_dtype, block_size ) attn_backend = create_chunked_local_attention_backend( - underlying_attn_backend, attention_chunk_size + underlying_attn_backend, attention_chunk_size, block_size ) super().__init__( diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 4fe25b02779..98ff02e9d4a 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -407,24 +407,17 @@ class MLAAttention(nn.Module, AttentionLayerBase): ) # Attributes for forward_impl method - self._vllm_config = get_current_vllm_config() - self._chunked_prefill_workspace_size: int | None = None + self.chunked_prefill_workspace_size = ( + MLACommonMetadataBuilder.determine_chunked_prefill_workspace_size( + get_current_vllm_config() + ) + ) self._decode_concat_quant_fp8_op = _DecodeConcatQuantFP8( static=True, group_shape=GroupShape.PER_TENSOR, compile_native=True, ) - @property - def chunked_prefill_workspace_size(self) -> int: - if self._chunked_prefill_workspace_size is None: - self._chunked_prefill_workspace_size = ( - MLACommonMetadataBuilder.determine_chunked_prefill_workspace_size( - self._vllm_config - ) - ) - return self._chunked_prefill_workspace_size - def forward( self, q: torch.Tensor, diff --git a/vllm/platforms/cuda.py b/vllm/platforms/cuda.py index 921054f73e6..c2fcde4ab1c 100644 --- a/vllm/platforms/cuda.py +++ b/vllm/platforms/cuda.py @@ -163,12 +163,122 @@ class CudaPlatformBase(Platform): @classmethod def check_and_update_config(cls, vllm_config: "VllmConfig") -> None: + from vllm.v1.attention.backends.registry import AttentionBackendEnum + parallel_config = vllm_config.parallel_config model_config = vllm_config.model_config if parallel_config.worker_cls == "auto": parallel_config.worker_cls = "vllm.v1.worker.gpu_worker.Worker" + cache_config = vllm_config.cache_config + if cache_config and cache_config.block_size is None: + cache_config.block_size = 16 + + # TODO(lucas): handle this more gracefully + # Note: model_config may be None during testing + # Note: block_size is initialized in + # HybridAttentionMambaModelConfig.verify_and_update_config + # for models with both attention and mamba, + # and doesn't need to be reinitialized here + if ( + model_config is not None + and model_config.use_mla + and cache_config.block_size is not None + ): + use_sparse = hasattr(vllm_config.model_config.hf_config, "index_topk") + # If `--attention-config.backend` is not set and we are using MLA, + # then we default to FlashMLA backend for non-blackwell GPUs, + # else we default to CutlassMLA. For each case, we force the + # required block_size. + use_flashmla = False + use_cutlass_mla = False + use_flashinfer_mla = False + use_flashmla_sparse = False + use_flashinfer_mla_sparse = False + + from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported + + if vllm_config.attention_config.backend is None: + # Default case + hf_text_config = model_config.hf_text_config + qk_nope_head_dim = getattr(hf_text_config, "qk_nope_head_dim", 1) + if ( + cls.is_device_capability_family(100) + and not use_sparse + and qk_nope_head_dim == 128 + ): + # Blackwell => Force FlashInfer MLA (unless sparse, i.e. DSv3.2) + # and only if qk_nope_head_dim == 128 (kernel constraint) + use_flashinfer_mla = True + # Set the backend in AttentionConfig so it's used during + # backend selection + vllm_config.attention_config.backend = ( + AttentionBackendEnum.FLASHINFER_MLA + ) + elif cls.is_device_capability_family(100) and not use_sparse: + # Fall back to CUTLASS_MLA as 2nd priority on Blackwell + use_cutlass_mla = True + elif is_flashmla_dense_supported()[0]: + # Non-Blackwell with FlashMLA support + use_flashmla = True + else: + # Fallback: will use Triton MLA or other compatible backend + pass + else: + # Forced case + backend = vllm_config.attention_config.backend + use_flashmla = backend == AttentionBackendEnum.FLASHMLA + use_cutlass_mla = backend == AttentionBackendEnum.CUTLASS_MLA + use_flashinfer_mla = backend == AttentionBackendEnum.FLASHINFER_MLA + use_flashmla_sparse = backend == AttentionBackendEnum.FLASHMLA_SPARSE + use_flashinfer_mla_sparse = ( + backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE + ) + + if ( + use_flashmla + and is_flashmla_dense_supported()[0] + and cache_config.block_size % 64 != 0 + ): + cache_config.block_size = 64 + logger.info("Forcing kv cache block size to 64 for FlashMLA backend.") + + if use_cutlass_mla and cache_config.block_size % 128 != 0: + cache_config.block_size = 128 + logger.info( + "Forcing kv cache block size to 128 for CUTLASS_MLA backend." + ) + + if ( + use_flashinfer_mla + and cache_config.block_size != 32 + and cache_config.block_size % 64 != 0 + ): + cache_config.block_size = 64 + logger.info( + "Forcing kv cache block size to 64 for FlashInferMLA backend." + ) + + if use_sparse: + if not (use_flashmla_sparse or use_flashinfer_mla_sparse): + use_flashmla_sparse = True + + if use_flashmla_sparse and cache_config.block_size != 64: + cache_config.block_size = 64 + logger.info( + "Forcing kv cache block size to 64 for FlashMLASparse backend." + ) + elif use_flashinfer_mla_sparse and cache_config.block_size not in ( + 32, + 64, + ): + cache_config.block_size = 64 + logger.info( + "Forcing kv cache block size to 64 for FlashInferMLASparse " + "backend." + ) + scheduler_config = vllm_config.scheduler_config # Note: model_config may be None during testing if ( @@ -183,49 +293,6 @@ class CudaPlatformBase(Platform): ) scheduler_config.disable_chunked_mm_input = True - @classmethod - def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: - cache_config = vllm_config.cache_config - if cache_config.block_size is not None: - # User specified --block-size; keep it. - return - - model_config = vllm_config.model_config - # model_config may be None during testing. - # Skip hybrid models — their block_size is managed by - # HybridAttentionMambaModelConfig. - if model_config is None or model_config.is_hybrid: - cache_config.block_size = 16 - return - - from vllm.config.vllm import ( - get_layers_from_vllm_config, - set_current_vllm_config, - ) - from vllm.model_executor.layers.attention_layer_base import ( - AttentionLayerBase, - ) - - attn_layers = get_layers_from_vllm_config( - vllm_config, - AttentionLayerBase, - ) - if not attn_layers: - cache_config.block_size = 16 - return - - first_layer = next(iter(attn_layers.values())) - backend_cls = first_layer.get_attn_backend() - with set_current_vllm_config(vllm_config): - preferred = backend_cls.get_preferred_block_size(16) - if preferred != 16: - logger.info( - "Setting kv cache block size to %d for %s backend.", - preferred, - backend_cls.get_name(), - ) - cache_config.block_size = preferred - @classmethod def get_current_memory_usage( cls, device: torch.types.Device | None = None @@ -242,10 +309,10 @@ class CudaPlatformBase(Platform): num_heads: int | None = None, ) -> tuple[ list[tuple["AttentionBackendEnum", int]], - dict["AttentionBackendEnum", tuple[int, list[str]]], + dict["AttentionBackendEnum", list[str]], ]: valid_backends_priorities = [] - invalid_reasons: dict[AttentionBackendEnum, tuple[int, list[str]]] = {} + invalid_reasons = {} backend_priorities = _get_backend_priorities( attn_selector_config.use_mla, @@ -262,155 +329,84 @@ class CudaPlatformBase(Platform): except ImportError: invalid_reasons_i = ["ImportError"] if invalid_reasons_i: - invalid_reasons[backend] = (priority, invalid_reasons_i) + invalid_reasons[backend] = invalid_reasons_i else: valid_backends_priorities.append((backend, priority)) return valid_backends_priorities, invalid_reasons - @classmethod - def select_attention_backend( - cls, - selected_backend: "AttentionBackendEnum | None", - attn_selector_config: "AttentionSelectorConfig", - device_capability: "DeviceCapability", - raise_on_invalid: bool = True, - num_heads: int | None = None, - ) -> "AttentionBackendEnum | None": - """Select the best attention backend for the given configuration. - - Args: - selected_backend: User-specified backend, or None for auto-selection - attn_selector_config: Configuration for attention selection - device_capability: Device capability info - raise_on_invalid: If True, raise ValueError when no valid backend - num_heads: Number of attention heads per GPU, used for backend - priority ordering on Blackwell GPUs - - Returns: - The selected backend enum, or None if no valid backend found - and raise_on_invalid is False - """ - # First try checking just the selected backend, if there is one. - if selected_backend is not None: - try: - backend_class = selected_backend.get_class() - validation_errors = backend_class.validate_configuration( - device_capability=device_capability, - **attn_selector_config._asdict(), - ) - except ImportError: - validation_errors = ["ImportError"] - if validation_errors: - if raise_on_invalid: - raise ValueError( - f"Selected backend {selected_backend} is not valid for " - f"this configuration. Reason: {validation_errors}" - ) - return None - return selected_backend - - # No selected backend, so find the best valid one. - valid_backends_priorities, invalid_reasons = cls.get_valid_backends( - device_capability=device_capability, - attn_selector_config=attn_selector_config, - num_heads=num_heads, - ) - - if len(valid_backends_priorities) == 0: - if raise_on_invalid: - reasons_str = ( - "{" - + ", ".join( - f"{backend.name}: [{', '.join(reasons)}]" - for backend, (_, reasons) in invalid_reasons.items() - ) - + "}" - ) - config_str = attn_selector_config.__repr__() - raise ValueError( - f"No valid attention backend found for {cls.device_name} " - f"with {config_str}. Reasons: {reasons_str}." - ) - return None - - # Select the one with the highest priority (lowest index). - sorted_backends = sorted(valid_backends_priorities, key=lambda x: x[1]) - chosen_backend, chosen_priority = sorted_backends[0] - - # If the user specified --block-size (but not --attention-backend), - # check whether that constraint precluded any higher-priority backends. - if attn_selector_config.block_size is not None: - excluded = [ - backend - for backend, (priority, reasons) in invalid_reasons.items() - if priority < chosen_priority - and reasons == ["block_size not supported"] - ] - if excluded: - names = ", ".join(b.name for b in excluded) - logger.warning( - "--block-size %d excluded higher-priority backend(s) " - "%s. Using %s instead, which may result in reduced " - "performance. Consider removing --block-size to " - "auto-select the optimal block size.", - attn_selector_config.block_size, - names, - chosen_backend.name, - ) - - return chosen_backend - @classmethod def get_attn_backend_cls( cls, - selected_backend: "AttentionBackendEnum | None", + selected_backend: "AttentionBackendEnum", attn_selector_config: "AttentionSelectorConfig", num_heads: int | None = None, ) -> str: device_capability = cls.get_device_capability() assert device_capability is not None - chosen_backend = cls.select_attention_backend( - selected_backend=selected_backend, + attn_selector_config = attn_selector_config._replace(block_size=None) + # First try checking just the selected backend, if there is one. + if selected_backend is not None: + try: + backend_class = selected_backend.get_class() + invalid_reasons = backend_class.validate_configuration( + device_capability=device_capability, + **attn_selector_config._asdict(), + ) + except ImportError: + invalid_reasons = ["ImportError"] + if invalid_reasons: + raise ValueError( + f"Selected backend {selected_backend} is not valid for " + f"this configuration. Reason: {invalid_reasons}" + ) + else: + logger.info("Using %s backend.", selected_backend) + return selected_backend.get_path() + + # No selected backend or the selected backend is invalid, + # so we try finding a valid backend. + valid_backends_priorities, invalid_reasons = cls.get_valid_backends( + device_capability=device_capability, attn_selector_config=attn_selector_config, num_heads=num_heads, - device_capability=device_capability, - raise_on_invalid=True, ) - assert chosen_backend is not None # raise_on_invalid=True guarantees this - - # Log the selection - if selected_backend is not None: - logger.info("Using %s backend.", chosen_backend) - else: - # Get all valid backends for logging - valid_backends_priorities, invalid_reasons = cls.get_valid_backends( - device_capability=device_capability, - attn_selector_config=attn_selector_config, - num_heads=num_heads, + reasons_str = ( + "{" + + ", ".join( + f"{backend.name}: [{', '.join(reasons)}]" + for backend, reasons in invalid_reasons.items() ) - reasons_str = ( - "{" - + ", ".join( - f"{backend.name}: [{', '.join(reasons)}]" - for backend, (_, reasons) in invalid_reasons.items() - ) - + "}" - ) - config_str = attn_selector_config.__repr__() - logger.debug_once( - f"Some attention backends are not valid for {cls.device_name} with " - f"{config_str}. Reasons: {reasons_str}." - ) - logger.info_once( - "Using %s attention backend out of potential backends: %s", - chosen_backend.name, - tuple(backend.name for backend, _ in valid_backends_priorities), - scope="local", + + "}" + ) + config_str = attn_selector_config.__repr__() + logger.debug_once( + f"Some attention backends are not valid for {cls.device_name} with " + f"{config_str}. Reasons: {reasons_str}." + ) + if len(valid_backends_priorities) == 0: + raise ValueError( + f"No valid attention backend found for {cls.device_name} " + f"with {config_str}. Reasons: {reasons_str}." ) - return chosen_backend.get_path() + # We have found some valid backends. Select the one with the + # highest priority. + sorted_indices = sorted( + range(len(valid_backends_priorities)), + key=lambda i: valid_backends_priorities[i][1], + ) + selected_index = sorted_indices[0] + selected_backend = valid_backends_priorities[selected_index][0] + logger.info_once( + "Using %s attention backend out of potential backends: %s.", + selected_backend.name, + "[" + ", ".join(f"'{b[0].name}'" for b in valid_backends_priorities) + "]", + scope="local", + ) + + return selected_backend.get_path() @classmethod def get_supported_vit_attn_backends(cls) -> list["AttentionBackendEnum"]: diff --git a/vllm/platforms/interface.py b/vllm/platforms/interface.py index ba44fa6d967..6794c05f5e5 100644 --- a/vllm/platforms/interface.py +++ b/vllm/platforms/interface.py @@ -406,13 +406,6 @@ class Platform: """ pass - @classmethod - def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None: - """ - Ensure block_size is compatible with the attention backend. - """ - pass - @classmethod def verify_model_arch(cls, model_arch: str) -> None: """ diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index f31e2635a0f..9c004d7724d 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -4,7 +4,7 @@ from abc import ABC, abstractmethod from dataclasses import dataclass, replace from enum import Enum -from typing import TYPE_CHECKING, Any, ClassVar, Generic, Protocol, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Generic, Protocol, TypeVar, get_args import numpy as np import torch @@ -144,9 +144,15 @@ class AttentionBackend(ABC): @classmethod def supports_block_size(cls, block_size: int | None) -> bool: + from vllm.config.cache import BlockSize + if block_size is None: return True + valid_sizes = get_args(BlockSize) + if block_size not in valid_sizes: + return False + supported_kernel_block_sizes = cls.get_supported_kernel_block_sizes() if not supported_kernel_block_sizes: return True @@ -161,17 +167,6 @@ class AttentionBackend(ABC): return True return False - @classmethod - def get_preferred_block_size(cls, default_block_size: int = 16) -> int: - supported_sizes = cls.get_supported_kernel_block_sizes() - if not supported_sizes: - return default_block_size - - if cls.supports_block_size(default_block_size): - return default_block_size - - return min(s.base if isinstance(s, MultipleOf) else s for s in supported_sizes) - @classmethod def is_mla(cls) -> bool: return False diff --git a/vllm/v1/engine/core.py b/vllm/v1/engine/core.py index b805abe8a66..a258fe29506 100644 --- a/vllm/v1/engine/core.py +++ b/vllm/v1/engine/core.py @@ -114,14 +114,7 @@ class EngineCore: num_gpu_blocks, num_cpu_blocks, kv_cache_config = self._initialize_kv_caches( vllm_config ) - if kv_cache_config.kv_cache_groups: - vllm_config.cache_config.block_size = min( - g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups - ) - elif vllm_config.cache_config.block_size is None: - # Attention-free models (encoder-only, SSM) — use default. - vllm_config.cache_config.block_size = 16 - vllm_config.validate_block_size() + vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks vllm_config.cache_config.num_cpu_blocks = num_cpu_blocks self.collective_rpc("initialize_cache", args=(num_gpu_blocks, num_cpu_blocks)) diff --git a/vllm/v1/executor/multiproc_executor.py b/vllm/v1/executor/multiproc_executor.py index 9cc7dc63ad8..b63cbd6586f 100644 --- a/vllm/v1/executor/multiproc_executor.py +++ b/vllm/v1/executor/multiproc_executor.py @@ -41,7 +41,6 @@ from vllm.distributed.parallel_state import ( ) from vllm.envs import enable_envs_cache from vllm.logger import init_logger -from vllm.platforms import current_platform from vllm.tracing import instrument, maybe_init_worker_tracer from vllm.utils.network_utils import ( get_distributed_init_method, @@ -580,9 +579,6 @@ class WorkerProc: self._init_message_queues(input_shm_handle, vllm_config) self.worker.load_model() - # Set block size based on the attention backends - current_platform.update_block_size_for_backend(vllm_config) - # Enable environment variable cache (e.g. assume no more # environment variable overrides after this point) enable_envs_cache() diff --git a/vllm/v1/executor/ray_executor.py b/vllm/v1/executor/ray_executor.py index 6c939a59387..ad51526ae94 100644 --- a/vllm/v1/executor/ray_executor.py +++ b/vllm/v1/executor/ray_executor.py @@ -385,11 +385,6 @@ class RayDistributedExecutor(Executor): self.collective_rpc("init_device") self.collective_rpc("load_model") - def _update_block_size(worker): - current_platform.update_block_size_for_backend(worker.vllm_config) - - self.collective_rpc(_update_block_size) - for pp_rank in range(self.parallel_config.pipeline_parallel_size): self.pp_tp_workers.append([]) for tp_rank in range(self.parallel_config.tensor_parallel_size): diff --git a/vllm/v1/executor/uniproc_executor.py b/vllm/v1/executor/uniproc_executor.py index 290c4dc8bbc..b9c7b550170 100644 --- a/vllm/v1/executor/uniproc_executor.py +++ b/vllm/v1/executor/uniproc_executor.py @@ -12,7 +12,6 @@ import torch.distributed as dist import vllm.envs as envs from vllm.logger import init_logger -from vllm.platforms import current_platform from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput from vllm.v1.engine import ReconfigureDistributedRequest, ReconfigureRankType @@ -47,7 +46,6 @@ class UniProcExecutor(Executor): self.driver_worker.init_worker(all_kwargs=[kwargs]) self.driver_worker.init_device() self.driver_worker.load_model() - current_platform.update_block_size_for_backend(self.vllm_config) def _distributed_args(self) -> tuple[str, int, int]: """Return (distributed_init_method, rank, local_rank).""" diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 51c4f5805e5..9ef8584c7f8 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -513,7 +513,6 @@ class GPUModelRunner( custom_logitsprocs: Sequence[str | type[LogitsProcessor]] = ( tuple(logits_processors) if logits_processors is not None else () ) - placeholder_block_size = self.cache_config.block_size or 16 self.input_batch = InputBatch( max_num_reqs=self.max_num_reqs, # We need to use the encoder length for encoder-decoer @@ -523,8 +522,8 @@ class GPUModelRunner( device=self.device, pin_memory=self.pin_memory, vocab_size=self.model_config.get_vocab_size(), - block_sizes=[placeholder_block_size], - kernel_block_sizes=[placeholder_block_size], + block_sizes=[self.cache_config.block_size], + kernel_block_sizes=[self.cache_config.block_size], is_spec_decode=bool(self.vllm_config.speculative_config), logitsprocs=build_logitsprocs( self.vllm_config, From 0632ed8778cab44de6152eb873d09fa40c241962 Mon Sep 17 00:00:00 2001 From: Ryan Rock Date: Fri, 20 Feb 2026 15:33:04 -0600 Subject: [PATCH 37/74] [AMD][CI] Fix test_custom_allreduce for A100 testgroup (#34735) Signed-off-by: Ryan Rock --- tests/distributed/test_custom_all_reduce.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/distributed/test_custom_all_reduce.py b/tests/distributed/test_custom_all_reduce.py index f6e274be938..68abc2b98c3 100644 --- a/tests/distributed/test_custom_all_reduce.py +++ b/tests/distributed/test_custom_all_reduce.py @@ -33,6 +33,7 @@ def graph_allreduce( ): with monkeypatch.context() as m: m.delenv("CUDA_VISIBLE_DEVICES", raising=False) + m.delenv("HIP_VISIBLE_DEVICES", raising=False) device = torch.device(f"cuda:{rank}") torch.cuda.set_device(device) init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port) @@ -92,6 +93,7 @@ def eager_allreduce( ): with monkeypatch.context() as m: m.delenv("CUDA_VISIBLE_DEVICES", raising=False) + m.delenv("HIP_VISIBLE_DEVICES", raising=False) device = torch.device(f"cuda:{rank}") torch.cuda.set_device(device) init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port) From ea5f903f80fec5afd4960a3846b8a84b0e53ca6e Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Fri, 20 Feb 2026 16:37:31 -0500 Subject: [PATCH 38/74] Bump Flashinfer Version and Re-enable DeepSeek NVFP4 AR+Norm Fusion (#34899) Signed-off-by: wzhao18 Co-authored-by: Cyrus Leung --- docker/Dockerfile | 2 +- docker/Dockerfile.nightly_torch | 4 ++-- docker/versions.json | 2 +- requirements/cuda.txt | 2 +- vllm/model_executor/models/config.py | 25 +------------------------ 5 files changed, 6 insertions(+), 29 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 71cef521bed..cc2ccc11cdc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -582,7 +582,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # This is ~1.1GB and only changes when FlashInfer version bumps # https://docs.flashinfer.ai/installation.html # From versions.json: .flashinfer.version -ARG FLASHINFER_VERSION=0.6.3 +ARG FLASHINFER_VERSION=0.6.4 RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --system flashinfer-cubin==${FLASHINFER_VERSION} \ && uv pip install --system flashinfer-jit-cache==${FLASHINFER_VERSION} \ diff --git a/docker/Dockerfile.nightly_torch b/docker/Dockerfile.nightly_torch index b4d590016b1..6f6f147c438 100644 --- a/docker/Dockerfile.nightly_torch +++ b/docker/Dockerfile.nightly_torch @@ -217,13 +217,13 @@ RUN pip install setuptools==75.6.0 packaging==23.2 ninja==1.11.1.3 build==1.2.2. # build flashinfer for torch nightly from source around 10 mins -# release version: v0.6.3 +# release version: v0.6.4 # todo(elainewy): cache flashinfer build result for faster build ENV CCACHE_DIR=/root/.cache/ccache RUN --mount=type=cache,target=/root/.cache/ccache \ --mount=type=cache,target=/root/.cache/uv \ echo "git clone flashinfer..." \ - && git clone --depth 1 --branch v0.6.3 --recursive https://github.com/flashinfer-ai/flashinfer.git \ + && git clone --depth 1 --branch v0.6.4 --recursive https://github.com/flashinfer-ai/flashinfer.git \ && cd flashinfer \ && git submodule update --init --recursive \ && echo "finish git clone flashinfer..." \ diff --git a/docker/versions.json b/docker/versions.json index 6277e0b6faf..24f4b6e7d1b 100644 --- a/docker/versions.json +++ b/docker/versions.json @@ -68,7 +68,7 @@ "default": "true" }, "FLASHINFER_VERSION": { - "default": "0.6.3" + "default": "0.6.4" }, "GDRCOPY_CUDA_VERSION": { "default": "12.8" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 15e4ebbf4d5..84fe347306c 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -10,4 +10,4 @@ torchaudio==2.10.0 # These must be updated alongside torch torchvision==0.25.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version # FlashInfer should be updated together with the Dockerfile -flashinfer-python==0.6.3 +flashinfer-python==0.6.4 diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index e67a7700541..27cf3a7929f 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -536,34 +536,12 @@ class HybridAttentionMambaModelConfig(VerifyAndUpdateConfig): ) -class DeepseekV3ForCausalLM(VerifyAndUpdateConfig): - @classmethod - def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: - """Disable AR-RMS-Quant fusion for DeepSeekV3 in NVFP4""" - # TODO: https://github.com/vllm-project/vllm/issues/34395 - - # disable AR-rms-fp4 fusion for DSv3+ - ar_rms_enabled = vllm_config.compilation_config.pass_config.fuse_allreduce_rms - nvfp4 = vllm_config.model_config.is_nvfp4_quantized() - - # Disable by default, warn if manually enabled: - if ar_rms_enabled is None and nvfp4: - vllm_config.compilation_config.pass_config.fuse_allreduce_rms = False - if ar_rms_enabled and nvfp4: - logger.warning( - "Allreduce-rms fusion broken for DeepSeekV3 with NVFP4 quant," - "see https://github.com/vllm-project/vllm/issues/34395." - ) - - -class DeepseekV32ForCausalLM(DeepseekV3ForCausalLM): +class DeepseekV32ForCausalLM(VerifyAndUpdateConfig): @classmethod def verify_and_update_config(cls, vllm_config: "VllmConfig") -> None: """ Updated fp8 cache to custom "fp8_ds_mla" format for DeepSeekV32 """ - super().verify_and_update_config(vllm_config) - hf_config = vllm_config.model_config.hf_config # Mirror the check in vllm/model_executor/models/deepseek_v2.py @@ -654,7 +632,6 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { "MambaForCausalLM": MambaModelConfig, "Mamba2ForCausalLM": MambaModelConfig, "FalconMambaForCausalLM": MambaModelConfig, - "DeepseekV3ForCausalLM": DeepseekV3ForCausalLM, "DeepseekV32ForCausalLM": DeepseekV32ForCausalLM, "NemotronHForCausalLM": NemotronHForCausalLMConfig, "NemotronHPuzzleForCausalLM": NemotronHForCausalLMConfig, From 0e22cd618b5da36404365518aad5a522aea008e7 Mon Sep 17 00:00:00 2001 From: Lucas Wilkinson Date: Fri, 20 Feb 2026 20:19:19 -0500 Subject: [PATCH 39/74] Revert "[Llama4,Quantization] Simplify and generalize logic for Q/K permutations in quantized self-attn layers " (#34997) --- vllm/model_executor/models/llama4.py | 97 +++++++++++++++++++--------- 1 file changed, 68 insertions(+), 29 deletions(-) diff --git a/vllm/model_executor/models/llama4.py b/vllm/model_executor/models/llama4.py index b84b4e2ae51..4050bf0453e 100644 --- a/vllm/model_executor/models/llama4.py +++ b/vllm/model_executor/models/llama4.py @@ -44,6 +44,9 @@ from vllm.model_executor.layers.linear import ( RowParallelLinear, ) from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.compressed_tensors import ( + compressed_tensors as ct, +) from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.model_loader.weight_utils import ( default_weight_loader, @@ -828,38 +831,74 @@ class Llama4ForCausalLM(LlamaForCausalLM, MixtureOfExperts): name: str, loaded_weight: torch.Tensor, ) -> tuple[str, torch.Tensor]: - modules = name.split(".") - # Permute Q/K weights and corresponding scales for rotary embedding. - # This pathway is validated against modelopt and compressed-tensors ckpts, - # and for per-tensor, per-group (e.g. GPTQ), and per-channel quant schemes. - # Note: permutations are not feasible only for per-block (e.g. DeepSeek 128x128) - # For per-block quantization, consider not quantizing q/k_proj. - is_weight = modules[-1] in ("weight", "weight_packed") - is_weight_scale = ( - modules[-1] == "weight_scale" - and loaded_weight.numel() > 1 # no need to permute per-tensor scales - ) - is_k_proj = "wk" in modules or "k_proj" in modules - is_q_proj = "wq" in modules or "q_proj" in modules - - if (is_weight or is_weight_scale) and (is_k_proj or is_q_proj): - original_ndim = loaded_weight.ndim - if original_ndim == 1: - loaded_weight = loaded_weight.unsqueeze(-1) - - f_out, f_in = loaded_weight.shape - n_heads = ( - self.config.num_key_value_heads - if is_k_proj - else self.config.num_attention_heads + # Helper function to permute the weight's channels + def permute( + w: torch.Tensor, + n_heads: int, + is_nvfp4_weight_scale: bool, + is_ct_int8_or_fp8_weight_scale: bool, + ): + # Calculate the expected shape of the weight. + # Do not rely on w's shape, as it may be in another layout. + attn_in = self.config.head_dim * n_heads + attn_out = ( + self.config.hidden_size + if not is_ct_int8_or_fp8_weight_scale + else w.shape[-1] ) - loaded_weight = ( - loaded_weight.view(n_heads, f_out // n_heads // 2, 2, f_in) + + # If the weight is FP4 packed as uint8, we need to divide attn_out + # by 2. + if w.dtype == torch.uint8 and w.shape[1] * 2 == attn_out: + attn_out = attn_out // 2 + + # If the weight is a weight scale, we need to divide attn_out by + # block size, which is currently 16. + elif ( + w.dtype == torch.float8_e4m3fn + and is_nvfp4_weight_scale + and w.shape[1] * 16 == attn_out + ): + attn_out = attn_out // 16 + + return ( + w.view(n_heads, attn_in // n_heads // 2, 2, attn_out) .transpose(1, 2) - .reshape(f_out, f_in) + .reshape(attn_in, attn_out) ) - if original_ndim == 1: - loaded_weight = loaded_weight.squeeze(-1) + modules = name.split(".") + + # Permute Q/K weights and weight block scales for rotary embedding + is_weight = modules[-1] == "weight" + is_nvfp4_weight_scale = ( + modules[-1] == "weight_scale" and loaded_weight.dtype == torch.float8_e4m3fn + ) + is_ct_int8_or_fp8_weight_scale = False + if modules[-1] == "weight_scale" and isinstance( + self.model.quant_config, ct.CompressedTensorsConfig + ): + from compressed_tensors import CompressionFormat + + is_ct_int8_or_fp8_weight_scale = self.model.quant_config.quant_format in [ + CompressionFormat.int_quantized.value, + CompressionFormat.float_quantized.value, + ] and loaded_weight.dtype in [torch.float16, torch.bfloat16, torch.float32] + + if is_weight or is_nvfp4_weight_scale or is_ct_int8_or_fp8_weight_scale: + if "wk" in modules or "k_proj" in modules: + loaded_weight = permute( + loaded_weight, + self.config.num_key_value_heads, + is_nvfp4_weight_scale, + is_ct_int8_or_fp8_weight_scale, + ) + elif "wq" in modules or "q_proj" in modules: + loaded_weight = permute( + loaded_weight, + self.config.num_attention_heads, + is_nvfp4_weight_scale, + is_ct_int8_or_fp8_weight_scale, + ) return name, loaded_weight From a55caf6ae9a561dd816692060ef49681d9d6786d Mon Sep 17 00:00:00 2001 From: yugong333 Date: Fri, 20 Feb 2026 19:54:35 -0800 Subject: [PATCH 40/74] [LoRA] Support Quantized Adapters (#30286) Signed-off-by: Yu Gong Signed-off-by: wz1qqx Signed-off-by: mgoin Co-authored-by: wz1qqx <55830058+wz1qqx@users.noreply.github.com> Co-authored-by: wz1qqx Co-authored-by: Michael Goin Co-authored-by: Jee Jee Li Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Cyrus Leung --- vllm/lora/ops/triton_ops/__init__.py | 8 + .../ops/triton_ops/fused_moe_lora_fp8_op.py | 1032 +++++++++++++++++ 2 files changed, 1040 insertions(+) create mode 100644 vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py diff --git a/vllm/lora/ops/triton_ops/__init__.py b/vllm/lora/ops/triton_ops/__init__.py index 7e8b9a79add..76587376a3c 100644 --- a/vllm/lora/ops/triton_ops/__init__.py +++ b/vllm/lora/ops/triton_ops/__init__.py @@ -2,6 +2,11 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.lora.ops.triton_ops.fused_moe_lora_fp8_op import ( + fused_moe_lora_expand_fp8, + fused_moe_lora_fp8, + fused_moe_lora_shrink_fp8, +) from vllm.lora.ops.triton_ops.fused_moe_lora_op import ( fused_moe_lora, fused_moe_lora_expand, @@ -18,4 +23,7 @@ __all__ = [ "fused_moe_lora", "fused_moe_lora_shrink", "fused_moe_lora_expand", + "fused_moe_lora_fp8", + "fused_moe_lora_shrink_fp8", + "fused_moe_lora_expand_fp8", ] diff --git a/vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py b/vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py new file mode 100644 index 00000000000..015d434165d --- /dev/null +++ b/vllm/lora/ops/triton_ops/fused_moe_lora_fp8_op.py @@ -0,0 +1,1032 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +from typing import List # noqa: UP035 + +import torch + +from vllm.distributed import ( + tensor_model_parallel_all_gather, + tensor_model_parallel_all_reduce, +) +from vllm.triton_utils import tl, triton +from vllm.utils.torch_utils import direct_register_custom_op + +from .utils import supports_pdl + + +@triton.jit +def _get_lora_id( + lora_ids, + token_lora_mapping_ptr, + lora_idx, + pid_m, + top_k_num, + naive_block_assignment: tl.constexpr, +): + """Returns lora_id""" + if naive_block_assignment: + token_idx = pid_m // top_k_num + return tl.load(token_lora_mapping_ptr + token_idx) + else: + return tl.load(lora_ids + lora_idx) + + +@triton.jit +def _get_expert_id( + expert_ids_ptr, + lora_id, + pid_m, + stride_el, + max_loras, + naive_block_assignment: tl.constexpr, +): + """Returns expert_id""" + if naive_block_assignment: + return tl.load(expert_ids_ptr + pid_m) + else: + ind = lora_id * stride_el + pid_m + return tl.load(expert_ids_ptr + ind, ind < max_loras * stride_el, -1) + + +@triton.jit +def _get_token_offs( + sorted_token_ids_ptr, + lora_id, + pid_m, + offs, + stride_tl, + max_loras, + num_valid_tokens, + naive_block_assignment: tl.constexpr, + BLOCK_SIZE_M: tl.constexpr, +): + """Returns token offsets""" + if naive_block_assignment: + return tl.where(offs == 0, pid_m, num_valid_tokens) + else: + offs_token_id = pid_m * BLOCK_SIZE_M + offs + token_ind = stride_tl * lora_id + offs_token_id + return tl.load( + sorted_token_ids_ptr + token_ind, token_ind < max_loras * stride_tl, 0 + ) + + +_LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {} + + +def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device): + """ + `_LORA_PTR_DICT` collects the required information during `profile_run`, + After this, it remains constant and subsequent usage is through LUT. + Refer to: + https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py + """ + key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights) + + if (ptr_tensor := _LORA_PTR_DICT.get(key)) is not None: + return ptr_tensor + + tensor_ptrs = [] + for lora_weight in lora_weights: + tensor_ptrs.append(lora_weight.data_ptr()) + ptr_tensor = torch.tensor(tensor_ptrs, device=device, dtype=torch.uint64) + + _LORA_PTR_DICT[key] = ptr_tensor + return _LORA_PTR_DICT.get(key) + + +def _adjust_kernel_inputs( + num_active_loras: int, + sorted_token_ids: torch.Tensor | None, + expert_ids: torch.Tensor, +): + """ + helper function to adjust kernel inputs when sorted_token_ids is None + """ + if sorted_token_ids is None: + stride_tl = 0 + stride_el = 0 + grid_lora_dim = 1 + else: + stride_tl = sorted_token_ids.stride(0) + stride_el = expert_ids.stride(0) + grid_lora_dim = num_active_loras + return grid_lora_dim, stride_tl, stride_el + + +@triton.jit( + do_not_specialize=[ + "num_valid_tokens", + "EM", + "stride_tl", + "stride_el", + "slice_a_size", + "slice_c_size", + ] +) +def _fused_moe_lora_kernel_fp8( + a_ptr, + b_ptr, + c_ptr, + a_scale_ptr, + b_scale_ptr, + topk_weights_ptr, + sorted_token_ids_ptr, + expert_ids_ptr, + num_tokens_post_padded_ptr, + token_lora_mapping_ptr, + # Matrix dimensions + N, + K, + EM, + num_valid_tokens, + num_experts, + top_k_num, + lora_ids, + adapter_enabled, + max_loras, # <<< PR2: rename, used for masks when grid axis-2 != max_loras + # The stride variables represent how much to increase the ptr by when + # moving by 1 element in a particular dimension. E.g. `stride_am` is + # how much to increase `a_ptr` by to get the element one row down + # (A has M rows). + stride_am, + stride_ak, + stride_bl, + stride_be, + stride_bk, + stride_bn, + stride_cm, + stride_cn, + stride_tl, + stride_el, + stride_asm, + stride_ask, + stride_bsl, + stride_bse, + stride_bsk, + stride_bsn, + # block size for block-wise quantization + group_n: tl.constexpr, + group_k: tl.constexpr, + slice_a_size, + slice_c_size, + # Meta-parameters + num_slice_a: tl.constexpr, + num_slice_c: tl.constexpr, + # top_k_num or 1 depending on input token + # is expanded by top_k or not + token_mapping_factor: tl.constexpr, + # whether use naive block assignment + naive_block_assignment: tl.constexpr, + MUL_ROUTED_WEIGHT: tl.constexpr, + ADD_INPUTS: tl.constexpr, + USE_B_L2_CACHE: tl.constexpr, # new, enable .ca load for B + BLOCK_SIZE_M: tl.constexpr, + BLOCK_SIZE_N: tl.constexpr, + BLOCK_SIZE_K: tl.constexpr, + GROUP_SIZE_M: tl.constexpr, + SPLIT_K: tl.constexpr, + USE_GDC: tl.constexpr, + launch_pdl: tl.constexpr, + IS_PRIMARY: tl.constexpr, + use_fp8_w8a8: tl.constexpr, + use_int8_w8a8: tl.constexpr, + use_int8_w8a16: tl.constexpr, + per_channel_quant: tl.constexpr, +): + pid = tl.program_id(axis=0) + slice_id = tl.program_id(axis=1) + grid_k = tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K) + + # calculate pid_m,pid_n + lora_idx = tl.program_id(axis=2) + pid_sk = pid % SPLIT_K + pid_m_n = pid // SPLIT_K + num_pid_m = tl.cdiv(EM, BLOCK_SIZE_M) + num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) + + num_pid_in_group = GROUP_SIZE_M * num_pid_n + group_id = pid_m_n // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + ((pid_m_n % num_pid_in_group) % group_size_m) + pid_n = (pid_m_n % num_pid_in_group) // group_size_m + + offs = tl.arange(0, BLOCK_SIZE_M).to(tl.int64) + + # Get lora_id + lora_id = _get_lora_id( + lora_ids, + token_lora_mapping_ptr, + lora_idx, + pid_m, + top_k_num, + naive_block_assignment, + ) + if lora_id == -1: + return + moe_enabled = tl.load(adapter_enabled + lora_id) + if moe_enabled == 0: + return + if lora_id >= max_loras: + return + + # Non-naive only: check num_tokens_post_padded + if not naive_block_assignment: + num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr + lora_id) + if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded: + return + + # Get expert_id + expert_id = _get_expert_id( + expert_ids_ptr, + lora_id, + pid_m, + stride_el, + max_loras, + naive_block_assignment, + ) + if expert_id == -1: + return + + # Get token offsets + offs_token = _get_token_offs( + sorted_token_ids_ptr, + lora_id, + pid_m, + offs, + stride_tl, + max_loras, + num_valid_tokens, + naive_block_assignment, + BLOCK_SIZE_M, + ) + # get a_ptr,b_ptr,c_ptr + cur_a_ptr = a_ptr + (slice_id % num_slice_a) * slice_a_size + cur_b_ptr = tl.load(b_ptr + slice_id).to(tl.pointer_type(c_ptr.dtype.element_ty)) + cur_c_ptr = c_ptr + (slice_id % num_slice_c) * slice_c_size + + # remove modulo wrap-around + offs_bn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N).to(tl.int32) + offs_k = pid_sk * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) + token_mask = offs_token < num_valid_tokens + + # get a_ptrs,b_ptrs + a_ptrs = cur_a_ptr + ( + offs_token[:, None] // token_mapping_factor * stride_am + + offs_k[None, :] * stride_ak + ) + + b_ptrs = ( + cur_b_ptr + + lora_id * stride_bl + + expert_id * stride_be + + offs_k[:, None] * stride_bk + + offs_bn[None, :] * stride_bn + ) + + if USE_GDC and IS_PRIMARY: + # GDC launch dependents hints the runtime system to launch dependent kernels. + tl.extra.cuda.gdc_launch_dependents() + + # accumulator + accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) + + if USE_GDC and not IS_PRIMARY: + tl.extra.cuda.gdc_wait() + + for k in range(0, grid_k): + k_remaining = K - k * (BLOCK_SIZE_K * SPLIT_K) + # GDC wait waits for ALL programs in the prior kernel to complete + # before continuing. + # pre-fetch lora weight + # add (offs_bn < N) mask; optional .ca for B + b_mask = (offs_k[:, None] < k_remaining) & (offs_bn[None, :] < N) + if USE_B_L2_CACHE: + b = tl.load(b_ptrs, mask=b_mask, other=0.0, cache_modifier=".ca") + else: + b = tl.load(b_ptrs, mask=b_mask, other=0.0) + + if USE_GDC and not IS_PRIMARY: + tl.extra.cuda.gdc_wait() + a = tl.load( + a_ptrs, + mask=token_mask[:, None] & (offs_k[None, :] < k_remaining), + other=0.0, + ) + accumulator += tl.dot(a, b) + # Advance the ptrs to the next K block. + a_ptrs += BLOCK_SIZE_K * SPLIT_K * stride_ak + b_ptrs += BLOCK_SIZE_K * SPLIT_K * stride_bk + + if MUL_ROUTED_WEIGHT: + moe_weight = tl.load(topk_weights_ptr + offs_token, mask=token_mask, other=0.0) + accumulator = accumulator * moe_weight[:, None] + accumulator = accumulator.to(c_ptr.dtype.element_ty) + # Write back the block of the output + offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) + c_ptrs = cur_c_ptr + stride_cm * offs_token[:, None] + stride_cn * offs_cn[None, :] + c_mask = token_mask[:, None] & (offs_cn[None, :] < N) + + if SPLIT_K == 1: + if ADD_INPUTS: + prev = tl.load(c_ptrs, mask=c_mask, other=0.0) + tl.store(c_ptrs, prev + accumulator, mask=c_mask) + else: + tl.store(c_ptrs, accumulator, mask=c_mask) + else: + tl.atomic_add(c_ptrs, accumulator, mask=c_mask, sem="relaxed") + + +@torch.inference_mode() +def _fused_moe_lora_shrink_fp8( + a_intermediate_cache1: torch.Tensor, + # (num_slices, num_tokens, top_k_num, max_lora_rank) + qcurr_hidden_states: torch.Tensor, # (num_tokens, K,) + lora_a_stacked: list[ + torch.Tensor + ], # [(max_loras, num_experts, max_lora_rank, K,),...] + topk_weights: torch.Tensor, # (num_tokens, top_k_num) + sorted_token_ids: torch.Tensor | None, # (max_loras, _) + expert_ids: torch.Tensor, # (max_loras, _ ,) or (num_tokens * top_k,) + num_tokens_post_padded: torch.Tensor | None, # (max_loras, ) + token_lora_mapping: torch.Tensor, + top_k_num: int, + lora_ids: torch.Tensor, + adapter_enabled: torch.Tensor, + ## adding for kernel + device: torch.device, + N: int, + M: int, + EM: int, + K: int, + num_tokens: int, + num_experts: int, + num_slices: int, + block_size_m: int, + block_size_n: int, + block_size_k: int, + group_size_m: int, + num_warps: int, + num_stages: int, + split_k: int, + num_active_loras: int, + lora_a_scale_stacked: list[torch.Tensor], + mul_routed_weight: bool = False, + use_gdc: bool = False, + act_scale: torch.Tensor | None = None, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + per_channel_quant: bool = False, + block_shape: List[int] | None = None, # noqa: UP006, UP007 +) -> None: + if use_fp8_w8a8 or use_int8_w8a8: + assert lora_a_scale_stacked is not None, ( + "lora_a_scale_stacked must be provided for w8a8 quantization" + ) + assert block_shape is None or triton.cdiv( + lora_a_stacked[0].size(-2), block_shape[0] + ) == lora_a_scale_stacked[0].size(-2), ( + "Incompatible block shape for lora_a_scale_stacked.size(-2) " + ) + assert block_shape is None or triton.cdiv( + lora_a_stacked[0].size(-1), block_shape[1] + ) == lora_a_scale_stacked[0].size(-1), ( + "Incompatible block shape for lora_a_scale_stacked.size(-1) " + ) + elif use_int8_w8a16: + assert lora_a_scale_stacked is not None, ( + "lora_a_scale_stacked must be provided for w8a16 quantization" + ) + assert block_shape is None or block_shape[0] == 0, ( + "Block shape for activation must be 0 for w8a16" + ) + else: + assert act_scale is None + assert lora_a_scale_stacked is None + + if block_shape is not None: + block_size_k = min(block_size_k, min(block_shape[0], block_shape[1])) + + if lora_a_scale_stacked is not None: + b_scale_ptr = _get_ptr(lora_a_scale_stacked, device) + w1_lora_a_scale_stacked = lora_a_scale_stacked[0] + + w1_lora_a_stacked = lora_a_stacked[0] + shrink_config = { + "BLOCK_SIZE_M": block_size_m, + "BLOCK_SIZE_N": block_size_n, + "BLOCK_SIZE_K": block_size_k, + "GROUP_SIZE_M": group_size_m, + "num_warps": num_warps, + "num_stages": num_stages, + "SPLIT_K": split_k, + "USE_GDC": use_gdc, + "launch_pdl": use_gdc, # triton kernel metadata + } + + b_ptr = _get_ptr(lora_a_stacked, device) + + grid_lora_dim, stride_tl, stride_el = _adjust_kernel_inputs( + num_active_loras, sorted_token_ids, expert_ids + ) + + grid = lambda META: ( + split_k + * triton.cdiv(EM, META["BLOCK_SIZE_M"]) + * triton.cdiv(N, META["BLOCK_SIZE_N"]), + len(lora_a_stacked), + grid_lora_dim, + ) + _fused_moe_lora_kernel_fp8[grid]( + qcurr_hidden_states, + b_ptr, + a_intermediate_cache1, + act_scale, + b_scale_ptr if lora_a_scale_stacked is not None else None, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + token_lora_mapping, + N, + K, + EM, + num_tokens, + num_experts, + top_k_num, + lora_ids, + adapter_enabled, + lora_a_stacked[0].shape[0], + qcurr_hidden_states.stride(0), + qcurr_hidden_states.stride(1), + w1_lora_a_stacked.stride(0), + w1_lora_a_stacked.stride(1), + w1_lora_a_stacked.stride(3), + w1_lora_a_stacked.stride(2), + a_intermediate_cache1.stride(2), + a_intermediate_cache1.stride(3), + stride_tl, + stride_el, + act_scale.stride(0) if act_scale is not None and act_scale.ndim == 2 else 0, + act_scale.stride(1) if act_scale is not None and act_scale.ndim == 2 else 0, + w1_lora_a_scale_stacked.stride(0) + if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim >= 2 + else 0, + w1_lora_a_scale_stacked.stride(1) + if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim >= 2 + else 0, + w1_lora_a_scale_stacked.stride(3) + if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim == 4 + else 0, + w1_lora_a_scale_stacked.stride(2) + if lora_a_scale_stacked is not None and w1_lora_a_scale_stacked.ndim == 4 + else 0, + 0 if block_shape is None else block_shape[0], + 0 if block_shape is None else block_shape[1], + slice_a_size=qcurr_hidden_states.numel(), + slice_c_size=a_intermediate_cache1.numel() // num_slices, + num_slice_a=1, + num_slice_c=num_slices, + token_mapping_factor=1 if mul_routed_weight else top_k_num, + naive_block_assignment=sorted_token_ids is None, + MUL_ROUTED_WEIGHT=False, + ADD_INPUTS=False, + USE_B_L2_CACHE=True, # new + IS_PRIMARY=True, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + per_channel_quant=per_channel_quant, + **shrink_config, + ) + + +@torch.inference_mode() +def _fused_moe_lora_expand_fp8( + output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),) + a_intermediate_cache1: torch.Tensor, # (num_slices, M, top_k_num, max_lora_rank) + lora_b_stacked: list[ + torch.Tensor + ], # [(max_loras, num_experts, max_lora_rank, K,),...] + topk_weights: torch.Tensor, # (num_tokens, top_k_num) + sorted_token_ids: torch.Tensor | None, # (max_loras, _) + expert_ids: torch.Tensor, # (max_loras, _ ,) or (num_tokens * top_k,) + num_tokens_post_padded: torch.Tensor | None, # (max_loras, ) + token_lora_mapping: torch.Tensor, + top_k_num: int, + lora_ids: torch.Tensor, + adapter_enabled: torch.Tensor, + ## adding for kernel + device: torch.device, + N: int, + M: int, + EM: int, + K: int, + num_tokens: int, + num_experts: int, + num_slices: int, + max_lora_rank: int, + w1_output_dim_size: int, + block_size_m: int, + block_size_n: int, + block_size_k: int, + group_size_m: int, + num_warps: int, + num_stages: int, + split_k: int, + num_active_loras: int, + lora_b_scale_stacked: list[torch.Tensor], + mul_routed_weight: bool = False, + offset: int = 0, + use_gdc: bool = False, + act_scale: torch.Tensor | None = None, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + per_channel_quant: bool = False, + block_shape: List[int] | None = None, # noqa: UP006, UP007 +) -> None: + if use_fp8_w8a8 or use_int8_w8a8: + assert lora_b_scale_stacked is not None, ( + "lora_b_scale_stacked must be provided for w8a8 quantization" + ) + assert block_shape is None or triton.cdiv( + lora_b_stacked[0].size(-2), block_shape[0] + ) == lora_b_scale_stacked[0].size(-2), ( + "Incompatible block shape for lora_b_scale_stacked.size(-2) " + ) + assert block_shape is None or triton.cdiv( + lora_b_stacked[0].size(-1), block_shape[1] + ) == lora_b_scale_stacked[0].size(-1), ( + "Incompatible block shape for lora_b_scale_stacked.size(-1) " + ) + elif use_int8_w8a16: + assert lora_b_scale_stacked is not None, ( + "lora_b_scale_stacked must be provided for w8a16 quantization" + ) + assert block_shape is None or block_shape[0] == 0, ( + "Block shape for activation must be 0 for w8a16" + ) + else: + assert act_scale is None + assert lora_b_scale_stacked is None + + if lora_b_scale_stacked is not None: + b_scale_ptr = _get_ptr(lora_b_scale_stacked, device) + w1_lora_b_scale_stacked = lora_b_scale_stacked[0] + + if block_shape is not None: + block_size_k = min(block_size_k, min(block_shape[0], block_shape[1])) + + b_ptr = _get_ptr(lora_b_stacked, device) + K = max_lora_rank + N = w1_output_dim_size + + w1_lora_b_stacked = lora_b_stacked[0] + + a_intermediate_cache1 = a_intermediate_cache1.view( + -1, a_intermediate_cache1.shape[3] + ) + + expand_config = { + "BLOCK_SIZE_M": block_size_m, + "BLOCK_SIZE_N": block_size_n, + "BLOCK_SIZE_K": block_size_k, + "GROUP_SIZE_M": group_size_m, + "num_warps": num_warps, + "num_stages": num_stages, + "SPLIT_K": 1, # Set split_k = 1 for expand calls + "USE_GDC": use_gdc, + "launch_pdl": use_gdc, # triton kernel metadata + } + + grid_lora_dim, stride_tl, stride_el = _adjust_kernel_inputs( + num_active_loras, sorted_token_ids, expert_ids + ) + + grid = lambda META: ( + triton.cdiv(EM, META["BLOCK_SIZE_M"]) * triton.cdiv(N, META["BLOCK_SIZE_N"]), + len(lora_b_stacked), + grid_lora_dim, + ) + + # Fast path: directly accumulate into the corresponding slice interval of output. + out_view = output[:, :, offset : offset + num_slices * N] + slice_c_size = N * out_view.stride(2) + + _fused_moe_lora_kernel_fp8[grid]( + a_intermediate_cache1, + b_ptr, + out_view, + act_scale, + b_scale_ptr if lora_b_scale_stacked is not None else None, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + token_lora_mapping, + N, + K, + EM, + num_tokens, + num_experts, + top_k_num, + lora_ids, + adapter_enabled, + lora_b_stacked[0].shape[0], + a_intermediate_cache1.stride(0), + a_intermediate_cache1.stride(1), + w1_lora_b_stacked.stride(0), + w1_lora_b_stacked.stride(1), + w1_lora_b_stacked.stride(3), + w1_lora_b_stacked.stride(2), + out_view.stride(1), + out_view.stride(2), + stride_tl, + stride_el, + act_scale.stride(0) if act_scale is not None and act_scale.ndim == 2 else 0, + act_scale.stride(1) if act_scale is not None and act_scale.ndim == 2 else 0, + w1_lora_b_scale_stacked.stride(0) + if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim >= 2 + else 0, + w1_lora_b_scale_stacked.stride(1) + if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim >= 2 + else 0, + w1_lora_b_scale_stacked.stride(3) + if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim == 4 + else 0, + w1_lora_b_scale_stacked.stride(2) + if lora_b_scale_stacked is not None and w1_lora_b_scale_stacked.ndim == 4 + else 0, + 0 if block_shape is None else block_shape[0], + 0 if block_shape is None else block_shape[1], + slice_a_size=a_intermediate_cache1.numel() // num_slices, + slice_c_size=slice_c_size, + num_slice_a=num_slices, + num_slice_c=num_slices, + token_mapping_factor=1, + naive_block_assignment=sorted_token_ids is None, + MUL_ROUTED_WEIGHT=mul_routed_weight, + ADD_INPUTS=True, + USE_B_L2_CACHE=True, # new + IS_PRIMARY=False, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + per_channel_quant=per_channel_quant, + **expand_config, + ) + + +@torch.inference_mode() +def _fused_moe_lora_fp8( + output: torch.Tensor, # (num_tokens, top_k_num, N*len(lora_a_stacked),) + qcurr_hidden_states: torch.Tensor, # (num_tokens, K,) + lora_a_stacked: list[ + torch.Tensor + ], # [(max_loras, num_experts, max_lora_rank, K,),...] + lora_b_stacked: list[ + torch.Tensor + ], # [(max_loras, num_experts, N, max_lora_rank,),...] + topk_weights: torch.Tensor, # (num_tokens, top_k_num) + sorted_token_ids: torch.Tensor | None, # (max_loras, _) + expert_ids: torch.Tensor, # (max_loras, _ ,) or (num_tokens * top_k,) + num_tokens_post_padded: torch.Tensor | None, # (max_loras, ) + token_lora_mapping: torch.Tensor, + max_lora_rank: int, + top_k_num: int, + lora_ids: torch.Tensor, + num_active_loras: int, + adapter_enabled: torch.Tensor, + shrink_block_size_m: int, + shrink_block_size_n: int, + shrink_block_size_k: int, + shrink_group_size_m: int, + shrink_num_warps: int, + shrink_num_stages: int, + shrink_split_k: int, + expand_block_size_m: int, + expand_block_size_n: int, + expand_block_size_k: int, + expand_group_size_m: int, + expand_num_warps: int, + expand_num_stages: int, + expand_split_k: int, + lora_a_scale_stacked: list[torch.Tensor], + lora_b_scale_stacked: list[torch.Tensor], + shrink_act_scale: torch.Tensor | None = None, + expand_act_scale: torch.Tensor | None = None, + mul_routed_weight: bool = False, + fully_sharded: bool = False, + offset: int = 0, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + per_channel_quant: bool = False, + block_shape: List[int] | None = None, # noqa: UP006, UP007 +) -> None: + assert len(lora_a_stacked) == len(lora_b_stacked) > 0 + assert topk_weights.dim() == qcurr_hidden_states.dim() == 2 + if sorted_token_ids is None: + assert expert_ids.dim() == 1 + else: + assert sorted_token_ids is not None + assert num_tokens_post_padded is not None + assert ( + sorted_token_ids.dim() + == expert_ids.dim() + == topk_weights.dim() + == qcurr_hidden_states.dim() + == 2 + ) + assert ( + sorted_token_ids.shape[0] + == expert_ids.shape[0] + == num_tokens_post_padded.shape[0] + ) + assert output.shape[0] == topk_weights.shape[0] + assert top_k_num == topk_weights.shape[1] + device = qcurr_hidden_states.device + num_slices = len(lora_a_stacked) + w1_lora_b_stacked = lora_b_stacked[0] + num_experts = lora_a_stacked[0].shape[1] + N = max_lora_rank + M = topk_weights.shape[0] + K = qcurr_hidden_states.shape[1] + num_tokens = M * top_k_num + w1_output_dim_size = w1_lora_b_stacked.shape[2] + assert shrink_block_size_m == expand_block_size_m + EM = ( + sorted_token_ids.shape[1] + if sorted_token_ids is not None + else num_tokens * shrink_block_size_m + ) + + a_intermediate_cache1 = torch.zeros( + (num_slices, M, top_k_num, max_lora_rank), + dtype=output.dtype, + device=device, + ) + + use_gdc = supports_pdl(device) and not fully_sharded + _fused_moe_lora_shrink_fp8( + a_intermediate_cache1, + qcurr_hidden_states, + lora_a_stacked, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + token_lora_mapping, + top_k_num, + lora_ids, + adapter_enabled, + ## adding for kernel + device, + N, + M, + EM, + K, + num_tokens, + num_experts, + num_slices, + shrink_block_size_m, + shrink_block_size_n, + shrink_block_size_k, + shrink_group_size_m, + shrink_num_warps, + shrink_num_stages, + shrink_split_k, + num_active_loras, + lora_a_scale_stacked, + mul_routed_weight=mul_routed_weight, + use_gdc=use_gdc, + act_scale=shrink_act_scale, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + + if fully_sharded: + if max_lora_rank == w1_lora_b_stacked.shape[-1]: + a_intermediate_cache1 = tensor_model_parallel_all_reduce( + a_intermediate_cache1 + ) + else: + a_intermediate_cache1 = tensor_model_parallel_all_gather( + a_intermediate_cache1 + ) + + # reset max_lora_rank to the full rank after allgather + max_lora_rank = a_intermediate_cache1.shape[-1] + + _fused_moe_lora_expand_fp8( + output, + a_intermediate_cache1, + lora_b_stacked, + topk_weights, + sorted_token_ids, + expert_ids, + num_tokens_post_padded, + token_lora_mapping, + top_k_num, + lora_ids, + adapter_enabled, + ## adding for kernel + device, + N, + M, + EM, + K, + num_tokens, + num_experts, + num_slices, + max_lora_rank, + w1_output_dim_size, + expand_block_size_m, + expand_block_size_n, + expand_block_size_k, + expand_group_size_m, + expand_num_warps, + expand_num_stages, + expand_split_k, + num_active_loras, + lora_b_scale_stacked, + mul_routed_weight=mul_routed_weight, + offset=offset, + use_gdc=use_gdc, + act_scale=expand_act_scale, + use_fp8_w8a8=use_fp8_w8a8, + use_int8_w8a8=use_int8_w8a8, + use_int8_w8a16=use_int8_w8a16, + per_channel_quant=per_channel_quant, + block_shape=block_shape, + ) + + +def _fused_moe_lora_fp8_fake( + output: torch.Tensor, + qcurr_hidden_states: torch.Tensor, + lora_a_stacked: list[torch.Tensor], + lora_b_stacked: list[torch.Tensor], + topk_weights: torch.Tensor, + sorted_token_ids: torch.Tensor | None, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor | None, + token_lora_mapping: torch.Tensor, + max_lora_rank: int, + top_k_num: int, + lora_ids: torch.Tensor, + num_active_loras: int, + adapter_enabled: torch.Tensor, + shrink_block_size_m: int, + shrink_block_size_n: int, + shrink_block_size_k: int, + shrink_group_size_m: int, + shrink_num_warps: int, + shrink_num_stages: int, + shrink_split_k: int, + expand_block_size_m: int, + expand_block_size_n: int, + expand_block_size_k: int, + expand_group_size_m: int, + expand_num_warps: int, + expand_num_stages: int, + expand_split_k: int, + lora_a_scale_stacked: list[torch.Tensor], + lora_b_scale_stacked: list[torch.Tensor], + mul_routed_weight: bool = False, + fully_sharded: bool = False, + offset: int = 0, + shrink_act_scale: torch.Tensor | None = None, + expand_act_scale: torch.Tensor | None = None, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + per_channel_quant: bool = False, + block_shape: List[int] | None = None, # noqa: UP006, UP007 +) -> None: + return + + +def _fused_moe_lora_shrink_fp8_fake( + a_intermediate_cache1: torch.Tensor, + qcurr_hidden_states: torch.Tensor, + lora_a_stacked: list[torch.Tensor], + topk_weights: torch.Tensor, + sorted_token_ids: torch.Tensor | None, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor | None, + token_lora_mapping: torch.Tensor, + top_k_num: int, + lora_ids: torch.Tensor, + adapter_enabled: torch.Tensor, + device: torch.device, + N: int, + M: int, + EM: int, + K: int, + num_tokens: int, + num_experts: int, + num_slices: int, + block_size_m: int, + block_size_n: int, + block_size_k: int, + group_size_m: int, + num_warps: int, + num_stages: int, + split_k: int, + num_active_loras: int, + lora_a_scale_stacked: list[torch.Tensor], + mul_routed_weight: bool = False, + use_gdc: bool = False, + act_scale: torch.Tensor | None = None, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + per_channel_quant: bool = False, + block_shape: List[int] | None = None, # noqa: UP006, UP007 +) -> None: + return + + +def _fused_moe_lora_expand_fp8_fake( + output: torch.Tensor, + a_intermediate_cache1: torch.Tensor, + lora_b_stacked: list[torch.Tensor], + topk_weights: torch.Tensor, + sorted_token_ids: torch.Tensor | None, + expert_ids: torch.Tensor, + num_tokens_post_padded: torch.Tensor | None, + token_lora_mapping: torch.Tensor, + top_k_num: int, + lora_ids: torch.Tensor, + adapter_enabled: torch.Tensor, + device: torch.device, + N: int, + M: int, + EM: int, + K: int, + num_tokens: int, + num_experts: int, + num_slices: int, + max_lora_rank: int, + w1_output_dim_size: int, + block_size_m: int, + block_size_n: int, + block_size_k: int, + group_size_m: int, + num_warps: int, + num_stages: int, + split_k: int, + num_active_loras: int, + act_scale: torch.Tensor, + lora_b_scale_stacked: list[torch.Tensor], + mul_routed_weight: bool = False, + offset: int = 0, + use_fp8_w8a8: bool = False, + use_int8_w8a8: bool = False, + use_int8_w8a16: bool = False, + per_channel_quant: bool = False, + block_shape: List[int] | None = None, # noqa: UP006, UP007 + use_gdc: bool = False, +) -> None: + return + + +try: + direct_register_custom_op( + op_name="fused_moe_lora_fp8", + op_func=_fused_moe_lora_fp8, + mutates_args=["output"], + fake_impl=_fused_moe_lora_fp8_fake, + ) + + direct_register_custom_op( + op_name="fused_moe_lora_shrink_fp8", + op_func=_fused_moe_lora_shrink_fp8, + mutates_args=["a_intermediate_cache1"], + fake_impl=_fused_moe_lora_shrink_fp8_fake, + ) + + direct_register_custom_op( + op_name="fused_moe_lora_expand_fp8", + op_func=_fused_moe_lora_expand_fp8, + mutates_args=["output"], + fake_impl=_fused_moe_lora_expand_fp8_fake, + ) + + fused_moe_lora_fp8 = torch.ops.vllm.fused_moe_lora_fp8 + fused_moe_lora_shrink_fp8 = torch.ops.vllm.fused_moe_lora_shrink_fp8 + fused_moe_lora_expand_fp8 = torch.ops.vllm.fused_moe_lora_expand_fp8 + +except AttributeError: + fused_moe_lora_fp8 = _fused_moe_lora_fp8 + fused_moe_lora_shrink_fp8 = _fused_moe_lora_shrink_fp8 + fused_moe_lora_expand_fp8 = _fused_moe_lora_expand_fp8 From e739c29ea451869e073a31b5d8cbc6b88f162e8d Mon Sep 17 00:00:00 2001 From: Vlad Tiberiu Mihailescu Date: Fri, 20 Feb 2026 19:54:55 -0800 Subject: [PATCH 41/74] [CI/Build] Add opentelemetry libs in default vllm build (requirements/common.txt) (#34466) Signed-off-by: Vlad Mihailescu --- examples/online_serving/opentelemetry/README.md | 10 +--------- requirements/common.txt | 4 ++++ 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/examples/online_serving/opentelemetry/README.md b/examples/online_serving/opentelemetry/README.md index ae5d84d8ef1..4361b36f5c1 100644 --- a/examples/online_serving/opentelemetry/README.md +++ b/examples/online_serving/opentelemetry/README.md @@ -1,14 +1,6 @@ # Setup OpenTelemetry POC -1. Install OpenTelemetry packages: - - ```bash - pip install \ - 'opentelemetry-sdk>=1.26.0,<1.27.0' \ - 'opentelemetry-api>=1.26.0,<1.27.0' \ - 'opentelemetry-exporter-otlp>=1.26.0,<1.27.0' \ - 'opentelemetry-semantic-conventions-ai>=0.4.1,<0.5.0' - ``` +> **Note:** The core OpenTelemetry packages (`opentelemetry-sdk`, `opentelemetry-api`, `opentelemetry-exporter-otlp`, `opentelemetry-semantic-conventions-ai`) are bundled with vLLM. Manual installation is not required. 1. Start Jaeger in a docker container: diff --git a/requirements/common.txt b/requirements/common.txt index ef320c5e219..ec7ce5df9e8 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -53,3 +53,7 @@ model-hosting-container-standards >= 0.1.13, < 1.0.0 mcp grpcio grpcio-reflection +opentelemetry-sdk >= 1.27.0 +opentelemetry-api >= 1.27.0 +opentelemetry-exporter-otlp >= 1.27.0 +opentelemetry-semantic-conventions-ai >= 0.4.1 From 9d7577b2bdf9de1155f1078e591446453723a88c Mon Sep 17 00:00:00 2001 From: Yanan Cao Date: Fri, 20 Feb 2026 19:55:51 -0800 Subject: [PATCH 42/74] [Kernel] [Helion] [9/N] Canonicalize GPU variant names to base model names (#34928) Signed-off-by: Yanan Cao Co-authored-by: Claude Opus 4.6 --- tests/kernels/helion/test_utils.py | 8 +- vllm/kernels/helion/config_manager.py | 10 +- vllm/kernels/helion/configs/silu_mul_fp8.json | 27724 ---------------- vllm/kernels/helion/utils.py | 50 +- 4 files changed, 59 insertions(+), 27733 deletions(-) diff --git a/tests/kernels/helion/test_utils.py b/tests/kernels/helion/test_utils.py index 807aa460655..540cc4f8bc7 100644 --- a/tests/kernels/helion/test_utils.py +++ b/tests/kernels/helion/test_utils.py @@ -11,11 +11,13 @@ from vllm.kernels.helion.utils import canonicalize_gpu_name "driver_reported_name,expected", [ ("NVIDIA H200", "nvidia_h200"), - ("NVIDIA A100-SXM4-80GB", "nvidia_a100_sxm4_80gb"), - ("NVIDIA H100 80GB HBM3", "nvidia_h100_80gb_hbm3"), + ("NVIDIA A100-SXM4-80GB", "nvidia_a100"), + ("NVIDIA H100 80GB HBM3", "nvidia_h100"), + ("NVIDIA H100 PCIe", "nvidia_h100"), + ("NVIDIA H100 SXM5", "nvidia_h100"), ("NVIDIA GeForce RTX 4090", "nvidia_geforce_rtx_4090"), ("AMD Instinct MI300X", "amd_instinct_mi300x"), - ("Tesla V100-SXM2-32GB", "tesla_v100_sxm2_32gb"), + ("Tesla V100-SXM2-32GB", "tesla_v100"), ], ) def test_canonicalize_gpu_name(driver_reported_name, expected): diff --git a/vllm/kernels/helion/config_manager.py b/vllm/kernels/helion/config_manager.py index 3c53106ce7a..7a6836ac850 100644 --- a/vllm/kernels/helion/config_manager.py +++ b/vllm/kernels/helion/config_manager.py @@ -71,10 +71,18 @@ class ConfigSet: platform_dict = self._configs.get(platform) if platform_dict is None: avail_platforms = self.get_platforms() + # TODO(@gmagogsfm): add a CLI/env override flag so users can + # directly specify a platform name instead of relying on + # auto-detection, and suggest it in this error message. raise KeyError( f"Config not found for kernel '{self._kernel_name}': " f"platform '{platform}' not found. " - f"Available platforms: {avail_platforms or '(none)'}" + f"Available platforms: {avail_platforms or '(none)'}. " + f"If your GPU is a variant of a supported platform, " + f"consider adding a mapping in _GPU_NAME_ALIASES in " + f"vllm/kernels/helion/utils.py, or run " + f"scripts/autotune_helion_kernels.py to generate configs " + f"for your platform." ) config = platform_dict.get(config_key) diff --git a/vllm/kernels/helion/configs/silu_mul_fp8.json b/vllm/kernels/helion/configs/silu_mul_fp8.json index 0f0de04a1cc..b8f091d6602 100644 --- a/vllm/kernels/helion/configs/silu_mul_fp8.json +++ b/vllm/kernels/helion/configs/silu_mul_fp8.json @@ -13861,27730 +13861,6 @@ "range_warp_specializes": [] } }, - "nvidia_h100_pcie": { - "intermediate_2048_numtokens_256": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_256": { - "block_sizes": [ - 32, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "default": { - "block_sizes": [ - 1, - 512 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_256": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_256": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_256": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_7688_numtokens_256": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_256": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_1": { - "block_sizes": [ - 1, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_1": { - "block_sizes": [ - 1, - 1 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_2": { - "block_sizes": [ - 2, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_2": { - "block_sizes": [ - 1, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_2": { - "block_sizes": [ - 2, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_2": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_2": { - "block_sizes": [ - 1, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_2": { - "block_sizes": [ - 1, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_4": { - "block_sizes": [ - 1, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_4": { - "block_sizes": [ - 1, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_4": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_4": { - "block_sizes": [ - 1, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_4": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_4": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_8": { - "block_sizes": [ - 8, - 256 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_8": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_8": { - "block_sizes": [ - 2, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_8": { - "block_sizes": [ - 4, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_8": { - "block_sizes": [ - 8, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_8": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_16": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_16": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_16": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_16": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_16": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_16": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_24": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_24": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_24": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_24": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_24": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_24": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_32": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_32": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_32": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_32": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_32": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_32": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_40": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_40": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_40": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_40": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_40": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_40": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_48": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_48": { - "block_sizes": [ - 8, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_48": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_48": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_48": { - "block_sizes": [ - 16, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_48": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_56": { - "block_sizes": [ - 2, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_56": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_56": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_56": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_56": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_56": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_64": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_64": { - "block_sizes": [ - 4, - 64 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_64": { - "block_sizes": [ - 2, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_64": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_64": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_64": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_72": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_72": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_72": { - "block_sizes": [ - 64, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_72": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_72": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_72": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_80": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_80": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_80": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_80": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_80": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_80": { - "block_sizes": [ - 32, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_88": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_88": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_88": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_88": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "first", - "last" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_88": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_88": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "last", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_96": { - "block_sizes": [ - 128, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_96": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_96": { - "block_sizes": [ - 16, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 3, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_96": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_96": { - "block_sizes": [ - 64, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_96": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_104": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_104": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_104": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_104": { - "block_sizes": [ - 8, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_104": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_104": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_112": { - "block_sizes": [ - 32, - 1024 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_112": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_112": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_112": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_112": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_112": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_120": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_120": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_120": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_120": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_120": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_120": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_128": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_128": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_128": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_128": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_128": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_128": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 3, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_136": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 3, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_136": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_136": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_136": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_136": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 3, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_136": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_144": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_144": { - "block_sizes": [ - 256, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_144": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_144": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_144": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_144": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_152": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_152": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_152": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_152": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_152": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_152": { - "block_sizes": [ - 64, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_160": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_160": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_160": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_160": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_160": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_160": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_168": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_168": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_168": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_168": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_168": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_168": { - "block_sizes": [ - 32, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_176": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_176": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_176": { - "block_sizes": [ - 4, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_176": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_176": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_176": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_184": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_184": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_184": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_184": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_184": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_184": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_192": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_192": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_192": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_192": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_192": { - "block_sizes": [ - 32, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_192": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_200": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_200": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_200": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_200": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_200": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_200": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_208": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_208": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_208": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_208": { - "block_sizes": [ - 256, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_208": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "last", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_208": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_216": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_216": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_216": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_216": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_216": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_216": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "last" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_224": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_224": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_224": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_224": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_224": { - "block_sizes": [ - 256, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_224": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "first" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_232": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_232": { - "block_sizes": [ - 64, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_232": { - "block_sizes": [ - 16, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_232": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_232": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_232": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_240": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_240": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_240": { - "block_sizes": [ - 16, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_240": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_240": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_240": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_248": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_248": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_248": { - "block_sizes": [ - 256, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_248": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_248": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_248": { - "block_sizes": [ - 64, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_272": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_272": { - "block_sizes": [ - 8, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_272": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_272": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_272": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_272": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_288": { - "block_sizes": [ - 4, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_288": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_288": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_288": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_288": { - "block_sizes": [ - 256, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_288": { - "block_sizes": [ - 16, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_304": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_304": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_304": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_304": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_304": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_304": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_320": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_320": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_320": { - "block_sizes": [ - 512, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_320": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_320": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_320": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_336": { - "block_sizes": [ - 2, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_336": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_336": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_336": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_336": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_336": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_352": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_352": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_352": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_352": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_352": { - "block_sizes": [ - 8, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_352": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_368": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_368": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_368": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_368": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_368": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_368": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_384": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_384": { - "block_sizes": [ - 64, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_384": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_384": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_384": { - "block_sizes": [ - 8, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_384": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_400": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_400": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_400": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_400": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_400": { - "block_sizes": [ - 256, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_400": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_416": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_416": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_416": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_416": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_416": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_416": { - "block_sizes": [ - 32, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_432": { - "block_sizes": [ - 16, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_432": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_432": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_432": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_432": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_432": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_448": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_448": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_448": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_448": { - "block_sizes": [ - 32, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_448": { - "block_sizes": [ - 16, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_448": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_464": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_464": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_464": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_464": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_464": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_464": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_480": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_480": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "first" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_480": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_480": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_480": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_480": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_496": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_496": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_496": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_496": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_496": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_496": { - "block_sizes": [ - 256, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_512": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_512": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_512": { - "block_sizes": [ - 128, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_512": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_512": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_512": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat", - "range_warp_specializes": [] - } - }, - "nvidia_h100_80gb_hbm3": { - "intermediate_2048_numtokens_256": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_256": { - "block_sizes": [ - 32, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "default": { - "block_sizes": [ - 1, - 512 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_256": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_256": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_256": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_7688_numtokens_256": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_256": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_1": { - "block_sizes": [ - 1, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_1": { - "block_sizes": [ - 1, - 1 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_1": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_2": { - "block_sizes": [ - 2, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_2": { - "block_sizes": [ - 1, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_2": { - "block_sizes": [ - 2, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_2": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_2": { - "block_sizes": [ - 1, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_2": { - "block_sizes": [ - 1, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_4": { - "block_sizes": [ - 1, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_4": { - "block_sizes": [ - 1, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_4": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_4": { - "block_sizes": [ - 1, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_4": { - "block_sizes": [ - 1, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_4": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_8": { - "block_sizes": [ - 8, - 256 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_8": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_8": { - "block_sizes": [ - 2, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_8": { - "block_sizes": [ - 4, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_8": { - "block_sizes": [ - 8, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_8": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_16": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_16": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_16": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_16": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_16": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_16": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_24": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_24": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_24": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_24": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_24": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_24": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_32": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_32": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_32": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_32": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_32": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_32": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_40": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_40": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_40": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_40": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_40": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_40": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_48": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_48": { - "block_sizes": [ - 8, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_48": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_48": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_48": { - "block_sizes": [ - 16, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_48": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_56": { - "block_sizes": [ - 2, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_56": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_56": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_56": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_56": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_56": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_64": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_64": { - "block_sizes": [ - 4, - 64 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_64": { - "block_sizes": [ - 2, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_64": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_64": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_64": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_72": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_72": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_72": { - "block_sizes": [ - 64, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_72": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_72": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_72": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_80": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_80": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_80": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_80": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_80": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_80": { - "block_sizes": [ - 32, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_88": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_88": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_88": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_88": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "first", - "last" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_88": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_88": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "last", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_96": { - "block_sizes": [ - 128, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_96": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_96": { - "block_sizes": [ - 16, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 3, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_96": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_96": { - "block_sizes": [ - 64, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_96": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_104": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_104": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_104": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_104": { - "block_sizes": [ - 8, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_104": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_104": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_112": { - "block_sizes": [ - 32, - 1024 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_112": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_112": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_112": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_112": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_112": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_120": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_120": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_120": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_120": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_120": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_120": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_128": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_128": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_128": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_128": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_128": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_128": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 3, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_136": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 3, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_136": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_136": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_136": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_136": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 3, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_136": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_144": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_144": { - "block_sizes": [ - 256, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_144": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_144": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_144": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_144": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_152": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_152": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_152": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_152": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_152": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_152": { - "block_sizes": [ - 64, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_160": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_160": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_160": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_160": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_160": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_160": { - "block_sizes": [ - 128, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_168": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_168": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_168": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_168": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_168": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_168": { - "block_sizes": [ - 32, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_176": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_176": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_176": { - "block_sizes": [ - 4, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_176": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_176": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_176": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_184": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_184": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_184": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_184": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_184": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_184": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_192": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_192": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_192": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_192": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_192": { - "block_sizes": [ - 32, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_192": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_200": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_200": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_200": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_200": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_200": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_200": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_208": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_208": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_208": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_208": { - "block_sizes": [ - 256, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_208": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "last", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_208": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_216": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_216": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_216": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_216": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_216": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_216": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "last" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_224": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_224": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_224": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_224": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_224": { - "block_sizes": [ - 256, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_224": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "first" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_232": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_232": { - "block_sizes": [ - 64, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_232": { - "block_sizes": [ - 16, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_232": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_232": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_232": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_240": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_240": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_240": { - "block_sizes": [ - 16, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_240": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_240": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_240": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_248": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_248": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_248": { - "block_sizes": [ - 256, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_248": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_248": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_248": { - "block_sizes": [ - 64, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_272": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_272": { - "block_sizes": [ - 8, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_272": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 1, - 0 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_272": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_272": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_272": { - "block_sizes": [ - 64, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_288": { - "block_sizes": [ - 4, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_288": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_288": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_288": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_288": { - "block_sizes": [ - 256, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_288": { - "block_sizes": [ - 16, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_304": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_304": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_304": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_304": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_304": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_304": { - "block_sizes": [ - 64, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_320": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_320": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_320": { - "block_sizes": [ - 512, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_320": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_320": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_320": { - "block_sizes": [ - 128, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_336": { - "block_sizes": [ - 2, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_336": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_336": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_336": { - "block_sizes": [ - 64, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_336": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_336": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_352": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_352": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_352": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_352": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_352": { - "block_sizes": [ - 8, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_352": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_368": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_368": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_368": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_368": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_368": { - "block_sizes": [ - 32, - 4 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_368": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_384": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_384": { - "block_sizes": [ - 64, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_384": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_384": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_384": { - "block_sizes": [ - 8, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_384": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_400": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_400": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_400": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_400": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_400": { - "block_sizes": [ - 256, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 32, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_400": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_416": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_416": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_416": { - "block_sizes": [ - 64, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_416": { - "block_sizes": [ - 128, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_416": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_416": { - "block_sizes": [ - 32, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_432": { - "block_sizes": [ - 16, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_432": { - "block_sizes": [ - 32, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_432": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 4, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_432": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_432": { - "block_sizes": [ - 16, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_432": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_448": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_448": { - "block_sizes": [ - 8, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_448": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_448": { - "block_sizes": [ - 32, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_448": { - "block_sizes": [ - 16, - 256 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_448": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "last", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_464": { - "block_sizes": [ - 32, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_464": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_464": { - "block_sizes": [ - 16, - 64 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_464": { - "block_sizes": [ - 8, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_464": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "first" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_464": { - "block_sizes": [ - 128, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_480": { - "block_sizes": [ - 4, - 16 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 2, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_480": { - "block_sizes": [ - 4, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "first" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_480": { - "block_sizes": [ - 8, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_480": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_480": { - "block_sizes": [ - 64, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "last" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_480": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "last", - "", - "" - ], - "num_warps": 8, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_496": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_496": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "first", - "" - ], - "num_warps": 8, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_496": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 4 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 2, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "tensor_descriptor", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_496": { - "block_sizes": [ - 32, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "tensor_descriptor" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_496": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 4, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_496": { - "block_sizes": [ - 256, - 8 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2048_numtokens_512": { - "block_sizes": [ - 32, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "last" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_2880_numtokens_512": { - "block_sizes": [ - 16, - 32 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 1, - "indexing": [ - "pointer", - "tensor_descriptor", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_4096_numtokens_512": { - "block_sizes": [ - 128, - 512 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 16, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_8192_numtokens_512": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - false - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 32, - "num_stages": 1, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_11008_numtokens_512": { - "block_sizes": [ - 32, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 1 - ], - "range_unroll_factors": [ - 0 - ], - "range_warp_specializes": [], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "", - "", - "" - ], - "num_warps": 1, - "num_stages": 1, - "indexing": [ - "tensor_descriptor", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat" - }, - "intermediate_14336_numtokens_512": { - "block_sizes": [ - 16, - 128 - ], - "loop_orders": [ - [ - 0, - 1 - ] - ], - "flatten_loops": [ - true - ], - "l2_groupings": [ - 2 - ], - "range_unroll_factors": [ - 0 - ], - "range_num_stages": [ - 0 - ], - "range_multi_buffers": [ - null - ], - "range_flattens": [ - null - ], - "load_eviction_policies": [ - "first", - "", - "" - ], - "num_warps": 1, - "num_stages": 2, - "indexing": [ - "pointer", - "pointer", - "pointer", - "pointer" - ], - "pid_type": "flat", - "range_warp_specializes": [] - } - }, "nvidia_h100": { "intermediate_2048_numtokens_256": { "block_sizes": [ diff --git a/vllm/kernels/helion/utils.py b/vllm/kernels/helion/utils.py index 600e459f676..5ff8046c73c 100644 --- a/vllm/kernels/helion/utils.py +++ b/vllm/kernels/helion/utils.py @@ -8,6 +8,44 @@ from vllm.platforms import current_platform logger = logging.getLogger(__name__) +# Maps known variant GPU names (after lowercase/underscore normalization) +# to their canonical form. +# +# Names that are already canonical after normalization are NOT listed here. +# For example, "NVIDIA H200" normalizes to "nvidia_h200" which needs no +# further mapping, and AMD ROCm names like "AMD_Instinct_MI300X" come from +# a controlled lookup table in rocm.py and normalize cleanly to +# "amd_instinct_mi300x". Only names with variant suffixes (form factor, +# memory size, memory type, etc.) that should be stripped need entries. +# +# To add a new GPU variant: run `canonicalize_gpu_name()` without the alias +# to see the normalized name, then add a mapping here if it contains variant +# suffixes that should be stripped (e.g. Blackwell/Rubin variants). +_GPU_NAME_ALIASES: dict[str, str] = { + # H100 variants + "nvidia_h100_pcie": "nvidia_h100", + "nvidia_h100_sxm5": "nvidia_h100", + "nvidia_h100_80gb_hbm3": "nvidia_h100", + "nvidia_h100_nvl": "nvidia_h100", + # H200 variants + "nvidia_h200_nvl": "nvidia_h200", + "nvidia_h200_141gb_hbm3e": "nvidia_h200", + # A100 variants + "nvidia_a100_sxm4_80gb": "nvidia_a100", + "nvidia_a100_sxm4_40gb": "nvidia_a100", + "nvidia_a100_pcie_80gb": "nvidia_a100", + "nvidia_a100_pcie_40gb": "nvidia_a100", + "nvidia_a100_80gb_pcie": "nvidia_a100", + # V100 variants (Tesla-branded) + "tesla_v100_sxm2_32gb": "tesla_v100", + "tesla_v100_sxm2_16gb": "tesla_v100", + "tesla_v100_pcie_32gb": "tesla_v100", + "tesla_v100_pcie_16gb": "tesla_v100", + # AMD ROCm variants (from _ROCM_DEVICE_ID_NAME_MAP in rocm.py) + "amd_instinct_mi300x_hf": "amd_instinct_mi300x", + # ADD MORE HERE +} + def get_gpu_name(device_id: int | None = None) -> str: if device_id is None: @@ -23,17 +61,19 @@ def canonicalize_gpu_name(name: str) -> str: """ Canonicalize GPU name for use as a platform identifier. - Converts to lowercase and replaces spaces and hyphens with underscores. - e.g., "NVIDIA A100-SXM4-80GB" -> "nvidia_a100_sxm4_80gb" - "AMD_Instinct_MI300X" -> "amd_instinct_mi300x" - - Raises ValueError if name is empty. + Converts to lowercase, replaces spaces and hyphens with underscores, + and maps known variant names to their canonical form via _GPU_NAME_ALIASES. + e.g., "NVIDIA H100 80GB HBM3" -> "nvidia_h100" + "NVIDIA A100-SXM4-80GB" -> "nvidia_a100" + "AMD Instinct MI300X" -> "amd_instinct_mi300x" """ if not name or not name.strip(): raise ValueError("GPU name cannot be empty") name = name.lower() name = name.replace(" ", "_") name = name.replace("-", "_") + if name in _GPU_NAME_ALIASES: + return _GPU_NAME_ALIASES[name] return name From ded333fb9b903e9de9f1cc5d82d2b5c5ab726750 Mon Sep 17 00:00:00 2001 From: Rohan Potdar <66227218+Rohan138@users.noreply.github.com> Date: Fri, 20 Feb 2026 21:56:16 -0600 Subject: [PATCH 43/74] [ROCm][Bugfix]: Only save unpadded sizes for shared_experts in MoERunner to fix rmsnorm pad fusion (#34636) Signed-off-by: Rohan138 --- .../layers/fused_moe/runner/default_moe_runner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py index e92f068f08c..7e25c968740 100644 --- a/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py +++ b/vllm/model_executor/layers/fused_moe/runner/default_moe_runner.py @@ -384,8 +384,11 @@ class DefaultMoERunner(MoERunner): ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: # For latent MoE: save ORIGINAL hidden_states before transform # (shared_experts need original dimension, routed experts use transformed) - original_hidden_states = hidden_states - original_hidden_dim = hidden_states.shape[-1] + if self.shared_experts is not None: + original_hidden_states = hidden_states + original_hidden_dim = hidden_states.shape[-1] + else: + original_hidden_states = None # Apply transform for routed experts (e.g., latent projection for latent MoE) hidden_states = self.apply_routed_input_transform(hidden_states) @@ -407,7 +410,7 @@ class DefaultMoERunner(MoERunner): self._encode_layer_name(), ) - if isinstance(fused_output, tuple): + if self.shared_experts is not None: orig_hidden_dims = [original_hidden_dim, transformed_hidden_dim] else: orig_hidden_dims = [transformed_hidden_dim] From d38cd3dde549b8e421d7d0390799b985e13bd8ab Mon Sep 17 00:00:00 2001 From: Taneem Ibrahim Date: Fri, 20 Feb 2026 21:56:33 -0600 Subject: [PATCH 44/74] [Misc] Fix mypy errors in vllm/profiler and remove from exclude list (#34959) Signed-off-by: Taneem Ibrahim --- tools/pre_commit/mypy.py | 1 - vllm/profiler/layerwise_profile.py | 66 +++++++++++++++++------------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 4bc0b3ad493..27312ac59b1 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -43,7 +43,6 @@ EXCLUDE = [ "vllm/benchmarks", "vllm/config", "vllm/device_allocator", - "vllm/profiler", "vllm/reasoning", "vllm/tool_parser", ] diff --git a/vllm/profiler/layerwise_profile.py b/vllm/profiler/layerwise_profile.py index 6b4348b96dc..a36e4611f3c 100644 --- a/vllm/profiler/layerwise_profile.py +++ b/vllm/profiler/layerwise_profile.py @@ -5,7 +5,7 @@ import copy from collections import defaultdict from collections.abc import Callable from dataclasses import asdict, dataclass, field -from typing import Any, TypeAlias +from typing import Any, Generic, TypeAlias, TypeVar from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent @@ -69,13 +69,14 @@ class ModelStatsEntry: StatsEntry: TypeAlias = ModelStatsEntry | SummaryStatsEntry +StatsEntryT = TypeVar("StatsEntryT", bound=StatsEntry) @dataclass -class _StatsTreeNode: - entry: StatsEntry - children: list[StatsEntry] - parent: StatsEntry | None +class _StatsTreeNode(Generic[StatsEntryT]): + entry: StatsEntryT + children: list["_StatsTreeNode[StatsEntryT]"] = field(default_factory=list) + parent: "_StatsTreeNode[StatsEntryT] | None" = None @dataclass @@ -84,8 +85,8 @@ class LayerwiseProfileResults(profile): _kineto_event_correlation_map: dict[int, list[_KinetoEvent]] = field(init=False) _event_correlation_map: dict[int, list[FunctionEvent]] = field(init=False) _module_tree: list[_ModuleTreeNode] = field(init=False) - _model_stats_tree: list[_StatsTreeNode] = field(init=False) - _summary_stats_tree: list[_StatsTreeNode] = field(init=False) + _model_stats_tree: list[_StatsTreeNode[ModelStatsEntry]] = field(init=False) + _summary_stats_tree: list[_StatsTreeNode[SummaryStatsEntry]] = field(init=False) # profile metadata num_running_seqs: int | None = None @@ -95,7 +96,7 @@ class LayerwiseProfileResults(profile): self._build_module_tree() self._build_stats_trees() - def print_model_table(self, column_widths: dict[str, int] = None): + def print_model_table(self, column_widths: dict[str, int] | None = None): _column_widths = dict( name=60, cpu_time_us=12, cuda_time_us=12, pct_cuda_time=12, trace=60 ) @@ -113,7 +114,7 @@ class LayerwiseProfileResults(profile): ) ) - def print_summary_table(self, column_widths: dict[str, int] = None): + def print_summary_table(self, column_widths: dict[str, int] | None = None): _column_widths = dict( name=80, cuda_time_us=12, pct_cuda_time=12, invocations=15 ) @@ -155,14 +156,14 @@ class LayerwiseProfileResults(profile): @staticmethod def _indent_row_names_based_on_depth( - depths_rows: list[tuple[int, StatsEntry]], + depths_rows: list[tuple[int, StatsEntryT]], indent_style: Callable[[int], str] | str = " ", ): - indented_rows = [] + indented_rows: list[StatsEntryT] = [] for depth, row in depths_rows: if row.cuda_time_us == 0: continue - indented_row = copy.deepcopy(row) + indented_row: StatsEntryT = copy.deepcopy(row) indented_row.name = indent_string(indented_row.name, depth, indent_style) indented_rows.append(indented_row) return indented_rows @@ -240,7 +241,7 @@ class LayerwiseProfileResults(profile): return sum([self._cumulative_cuda_time(root) for root in self._module_tree]) def _build_stats_trees(self): - summary_dict: dict[str, _StatsTreeNode] = {} + summary_dict: dict[tuple[str, ...], _StatsTreeNode[SummaryStatsEntry]] = {} total_cuda_time = self._total_cuda_time() def pct_cuda_time(cuda_time_us): @@ -248,9 +249,9 @@ class LayerwiseProfileResults(profile): def build_summary_stats_tree_df( node: _ModuleTreeNode, - parent: _StatsTreeNode | None = None, - summary_trace: tuple[str] = (), - ): + parent: _StatsTreeNode[SummaryStatsEntry] | None = None, + summary_trace: tuple[str, ...] = (), + ) -> _StatsTreeNode[SummaryStatsEntry] | None: if event_has_module(node.event): name = event_module_repr(node.event) cuda_time_us = self._cumulative_cuda_time(node) @@ -274,7 +275,6 @@ class LayerwiseProfileResults(profile): pct_cuda_time=pct_cuda_time(cuda_time_us), invocations=1, ), - children=[], parent=parent, ) if parent: @@ -290,11 +290,14 @@ class LayerwiseProfileResults(profile): self._summary_stats_tree = [] for root in self._module_tree: - self._summary_stats_tree.append(build_summary_stats_tree_df(root)) + summary_node = build_summary_stats_tree_df(root) + if summary_node is not None: + self._summary_stats_tree.append(summary_node) def build_model_stats_tree_df( - node: _ModuleTreeNode, parent: _StatsTreeNode | None = None - ): + node: _ModuleTreeNode, + parent: _StatsTreeNode[ModelStatsEntry] | None = None, + ) -> _StatsTreeNode[ModelStatsEntry] | None: if event_has_module( node.event, ): @@ -319,7 +322,6 @@ class LayerwiseProfileResults(profile): trace=trace, ), parent=parent, - children=[], ) if parent: parent.children.append(new_node) @@ -331,14 +333,16 @@ class LayerwiseProfileResults(profile): self._model_stats_tree = [] for root in self._module_tree: - self._model_stats_tree.append(build_model_stats_tree_df(root)) + model_node = build_model_stats_tree_df(root) + if model_node is not None: + self._model_stats_tree.append(model_node) def _flatten_stats_tree( - self, tree: list[_StatsTreeNode] - ) -> list[tuple[int, StatsEntry]]: - entries: list[tuple[int, StatsEntry]] = [] + self, tree: list[_StatsTreeNode[StatsEntryT]] + ) -> list[tuple[int, StatsEntryT]]: + entries: list[tuple[int, StatsEntryT]] = [] - def df_traversal(node: _StatsTreeNode, depth=0): + def df_traversal(node: _StatsTreeNode[StatsEntryT], depth: int = 0): entries.append((depth, node.entry)) for child in node.children: df_traversal(child, depth=depth + 1) @@ -348,10 +352,14 @@ class LayerwiseProfileResults(profile): return entries - def _convert_stats_tree_to_dict(self, tree: list[_StatsTreeNode]) -> list[dict]: - root_dicts: list[dict] = [] + def _convert_stats_tree_to_dict( + self, tree: list[_StatsTreeNode[StatsEntryT]] + ) -> list[dict[str, Any]]: + root_dicts: list[dict[str, Any]] = [] - def df_traversal(node: _StatsTreeNode, curr_json_list: list[dict]): + def df_traversal( + node: _StatsTreeNode[StatsEntryT], curr_json_list: list[dict[str, Any]] + ): curr_json_list.append({"entry": asdict(node.entry), "children": []}) for child in node.children: df_traversal(child, curr_json_list[-1]["children"]) From 59c62332978fcce318784df499713764f14c7bc1 Mon Sep 17 00:00:00 2001 From: Li Date: Fri, 20 Feb 2026 22:57:38 -0500 Subject: [PATCH 45/74] Support prompt_embeds for pooling requests in output processor (#34904) Signed-off-by: Li Zhang Co-authored-by: Li Zhang --- vllm/v1/engine/output_processor.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index de94a0e5d0e..dc572ccc109 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -337,16 +337,20 @@ class RequestState: finished: bool, kv_transfer_params: dict[str, Any] | None = None, ) -> RequestOutput | PoolingRequestOutput: + # If prompt embeds were used, put placeholder prompt token ids + prompt_token_ids = self.prompt_token_ids + if prompt_token_ids is None and self.prompt_embeds is not None: + prompt_token_ids = [0] * len(self.prompt_embeds) + assert prompt_token_ids is not None + first_output = outputs[0] if isinstance(first_output, PoolingOutput): assert len(outputs) == 1 - # Prompt embeddings are currently not supported by pooling requests. - assert self.prompt_token_ids is not None return PoolingRequestOutput( request_id=external_req_id, outputs=first_output, num_cached_tokens=self.num_cached_tokens, - prompt_token_ids=self.prompt_token_ids, + prompt_token_ids=prompt_token_ids, finished=finished, ) assert self.logprobs_processor is not None @@ -356,11 +360,6 @@ class RequestState: else: prompt_logprobs = self.logprobs_processor.prompt_logprobs - # If prompt embeds were used, put placeholder prompt token ids - prompt_token_ids = self.prompt_token_ids - if prompt_token_ids is None and self.prompt_embeds is not None: - prompt_token_ids = [0] * len(self.prompt_embeds) - return RequestOutput( request_id=external_req_id, # request_id is what was provided externally lora_request=self.lora_request, From 7a5adad48026d130348064ae7d41072ff999d1bf Mon Sep 17 00:00:00 2001 From: Xin Yang <105740670+xyang16@users.noreply.github.com> Date: Fri, 20 Feb 2026 19:59:06 -0800 Subject: [PATCH 46/74] [Kernel] Optimize sample_recovered_tokens_kernel (#34974) Signed-off-by: Xin Yang --- tests/v1/sample/test_rejection_sampler.py | 127 +++++++++++++++++++++- vllm/v1/sample/rejection_sampler.py | 86 +++++++++------ 2 files changed, 179 insertions(+), 34 deletions(-) diff --git a/tests/v1/sample/test_rejection_sampler.py b/tests/v1/sample/test_rejection_sampler.py index d8ae57984fe..38ffc58e286 100644 --- a/tests/v1/sample/test_rejection_sampler.py +++ b/tests/v1/sample/test_rejection_sampler.py @@ -11,7 +11,11 @@ from tests.v1.sample.utils import create_allowed_token_ids from vllm.platforms import current_platform from vllm.v1.sample.logits_processor import LogitsProcessors from vllm.v1.sample.metadata import SamplingMetadata -from vllm.v1.sample.rejection_sampler import PLACEHOLDER_TOKEN_ID, RejectionSampler +from vllm.v1.sample.rejection_sampler import ( + PLACEHOLDER_TOKEN_ID, + RejectionSampler, + sample_recovered_tokens, +) from vllm.v1.sample.sampler import Sampler, SamplerOutput from vllm.v1.spec_decode.metadata import SpecDecodeMetadata @@ -518,6 +522,70 @@ def estimate_rejection_sampling_pdf( return hist.hist +def native_sample_recovered_tokens( + max_spec_len: int, + num_draft_tokens: list[int], + cu_num_draft_tokens: torch.Tensor, # [batch_size] + draft_token_ids: torch.Tensor, # [num_tokens] + draft_probs: torch.Tensor | None, # [num_tokens, vocab_size] + target_probs: torch.Tensor, # [num_tokens, vocab_size] + sampling_metadata: SamplingMetadata, + device: torch.device, +) -> torch.Tensor: + batch_size = len(num_draft_tokens) + vocab_size = target_probs.shape[-1] + + q = torch.empty( + (batch_size, vocab_size), + dtype=torch.float32, + device=device, + ) + q.exponential_() + + states = { + i: generator.get_state() + for i, generator in sampling_metadata.generators.items() + } + for i, generator in sampling_metadata.generators.items(): + # Do not generate random numbers for requests with no draft tokens. + # This can be important for reproducibility. + if num_draft_tokens[i] > 0: + q[i].exponential_(generator=generator) + + # In order to generate the same exponential later, reset the CUDA RNG + # state because RNG state advances after each call. + generator.set_state(states[i]) + + inv_q = q.reciprocal() + + out = torch.empty_like(draft_token_ids) + + for req_idx in range(batch_size): + start_idx = 0 if req_idx == 0 else int(cu_num_draft_tokens[req_idx - 1].item()) + end_idx = int(cu_num_draft_tokens[req_idx].item()) + num_tokens = end_idx - start_idx + + for pos in range(max_spec_len): + if pos >= num_tokens: + continue + token_idx = start_idx + pos + + if draft_probs is None: + # prob is target_probs[token_idx] except draft_token_id is zeroed + prob = target_probs[token_idx].clone() + draft_token_id = draft_token_ids[token_idx] + prob[draft_token_id] = 0.0 + else: + prob = (target_probs[token_idx] - draft_probs[token_idx]).clamp_min_( + 0.0 + ) + + score = prob * inv_q[req_idx] + recovered_id = torch.argmax(score, dim=-1) + out[token_idx] = recovered_id + return out + + def _test_masked_logits( rejection_sampler, batch_size: int, @@ -778,3 +846,60 @@ def test_allowed_token_ids(rejection_sampler): device=logits.device, ) assert torch.equal(output.sampled_token_ids, expected) + + +@pytest.mark.parametrize("batch_size", [1, 100]) +@pytest.mark.parametrize("vocab_size", [100, 8192, 10000]) +@pytest.mark.parametrize("max_spec_len", [1, 3]) +@pytest.mark.parametrize("no_draft_probs", [True, False]) +def test_sample_recovered_tokens( + batch_size: int, vocab_size: int, max_spec_len: int, no_draft_probs: bool +): + num_tokens = batch_size * max_spec_len + + # Create random draft probabilities. + draft_probs = torch.rand(num_tokens, vocab_size, dtype=torch.float32, device=DEVICE) + draft_probs = F.softmax(draft_probs, dim=-1) + + # Create random target probabilities. + target_logits = torch.rand( + num_tokens, vocab_size, dtype=torch.float32, device=DEVICE + ) + target_probs = F.softmax(target_logits, dim=-1) + + # Randomly sample draft token ids from draft probs + draft_token_ids = torch.multinomial(draft_probs, num_samples=1).to(torch.int32) + + temperature = torch.ones(batch_size, dtype=torch.float32, device=DEVICE) + generators = { + i: torch.Generator(device=DEVICE).manual_seed(i) for i in range(batch_size) + } + sampling_metadata = create_sampling_metadata( + all_greedy=False, temperature=temperature, generators=generators + ) + + spec_decode_metadata = create_spec_decode_metadata( + draft_token_ids.reshape(batch_size, max_spec_len).tolist(), target_logits + ) + + ref_recovered_token_ids = native_sample_recovered_tokens( + max_spec_len, + spec_decode_metadata.num_draft_tokens, + spec_decode_metadata.cu_num_draft_tokens, + draft_token_ids, + None if no_draft_probs else draft_probs, + target_probs, + sampling_metadata, + device=DEVICE, + ) + recovered_token_ids = sample_recovered_tokens( + max_spec_len, + spec_decode_metadata.num_draft_tokens, + spec_decode_metadata.cu_num_draft_tokens, + draft_token_ids, + None if no_draft_probs else draft_probs, + target_probs, + sampling_metadata, + device=DEVICE, + ) + assert torch.equal(recovered_token_ids, ref_recovered_token_ids) diff --git a/vllm/v1/sample/rejection_sampler.py b/vllm/v1/sample/rejection_sampler.py index b57c93e29fa..1efceba385a 100644 --- a/vllm/v1/sample/rejection_sampler.py +++ b/vllm/v1/sample/rejection_sampler.py @@ -623,16 +623,19 @@ def sample_recovered_tokens( if num_draft_tokens[i] > 0: q[i].exponential_(generator=generator) + inv_q = q.reciprocal() + recovered_token_ids = torch.empty_like(draft_token_ids) + BLOCK_SIZE = 8192 sample_recovered_tokens_kernel[(batch_size, max_spec_len)]( recovered_token_ids, cu_num_draft_tokens, draft_token_ids, draft_probs, target_probs, - q, + inv_q, vocab_size, - triton.next_power_of_2(vocab_size), + BLOCK_SIZE, NO_DRAFT_PROBS=draft_probs is None, ) return recovered_token_ids @@ -776,9 +779,9 @@ def sample_recovered_tokens_kernel( draft_token_ids_ptr, # [num_tokens] draft_probs_ptr, # [num_tokens, vocab_size] or None target_probs_ptr, # [num_tokens, vocab_size] - q_ptr, # [batch_size, vocab_size] + inv_q_ptr, # [batch_size, vocab_size] vocab_size, - PADDED_VOCAB_SIZE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, NO_DRAFT_PROBS: tl.constexpr, ): req_idx = tl.program_id(0) @@ -791,33 +794,50 @@ def sample_recovered_tokens_kernel( if pos >= num_draft_tokens: return - vocab_offset = tl.arange(0, PADDED_VOCAB_SIZE) - if NO_DRAFT_PROBS: - draft_token_id = tl.load(draft_token_ids_ptr + start_idx + pos) - prob = tl.load( - target_probs_ptr + (start_idx + pos) * vocab_size + vocab_offset, - mask=((vocab_offset < vocab_size) & (vocab_offset != draft_token_id)), - other=0, - ) - else: - draft_prob = tl.load( - draft_probs_ptr + (start_idx + pos) * vocab_size + vocab_offset, - mask=vocab_offset < vocab_size, - other=0, - ) - target_prob = tl.load( - target_probs_ptr + (start_idx + pos) * vocab_size + vocab_offset, - mask=vocab_offset < vocab_size, - other=0, - ) - prob = tl.maximum(target_prob - draft_prob, 0) - # NOTE(woosuk): We don't need `prob = prob / tl.sum(prob)` here because - # `tl.argmax` will select the maximum value. + token_idx = start_idx + pos - q = tl.load( - q_ptr + req_idx * vocab_size + vocab_offset, - mask=vocab_offset < vocab_size, - other=float("-inf"), - ) - recovered_id = tl.argmax(prob / q, axis=-1) - tl.store(output_token_ids_ptr + start_idx + pos, recovered_id) + if NO_DRAFT_PROBS: + draft_token_id = tl.load(draft_token_ids_ptr + token_idx) + + max_val = float("-inf") + recovered_id = 0 + for v in range(0, vocab_size, BLOCK_SIZE): + vocab_offset = v + tl.arange(0, BLOCK_SIZE) + vocab_mask = vocab_offset < vocab_size + + if NO_DRAFT_PROBS: + prob = tl.load( + target_probs_ptr + token_idx * vocab_size + vocab_offset, + mask=(vocab_mask & (vocab_offset != draft_token_id)), + other=0.0, + ) + else: + draft_prob = tl.load( + draft_probs_ptr + token_idx * vocab_size + vocab_offset, + mask=vocab_mask, + other=0.0, + ) + target_prob = tl.load( + target_probs_ptr + token_idx * vocab_size + vocab_offset, + mask=vocab_mask, + other=0.0, + ) + prob = tl.maximum(target_prob - draft_prob, 0.0) + # NOTE(woosuk): We don't need `prob = prob / tl.sum(prob)` here because + # `tl.argmax` will select the maximum value. + + inv_q = tl.load( + inv_q_ptr + req_idx * vocab_size + vocab_offset, + mask=vocab_mask, + other=0.0, + ) + + # Local tile reduction + score = prob * inv_q + local_max, local_id = tl.max(score, axis=0, return_indices=True) + + if local_max > max_val: + max_val = local_max + recovered_id = v + local_id + + tl.store(output_token_ids_ptr + token_idx, recovered_id) From 11be2c74dc1eb08aaaeb260f84a31c2b36bbd454 Mon Sep 17 00:00:00 2001 From: pougetat Date: Fri, 20 Feb 2026 19:59:42 -0800 Subject: [PATCH 47/74] [Realtime] Add Qwen3-ASR realtime streaming support (#34613) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Thomas Pouget-Abadie Co-authored-by: Thomas Pouget-Abadie Co-authored-by: Claude Opus 4.6 Co-authored-by: Nicolò Lucchesi --- tests/models/registry.py | 8 + .../entrypoints/openai/realtime/connection.py | 2 +- vllm/model_executor/models/interfaces.py | 4 + .../models/qwen3_asr_realtime.py | 239 ++++++++++++++++++ vllm/model_executor/models/registry.py | 4 + 5 files changed, 256 insertions(+), 1 deletion(-) create mode 100644 vllm/model_executor/models/qwen3_asr_realtime.py diff --git a/tests/models/registry.py b/tests/models/registry.py index 16e64ea9e6d..de8d33e557c 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1023,6 +1023,14 @@ _MULTIMODAL_EXAMPLE_MODELS = { min_transformers_version="4.57", is_available_online=False, ), + "Qwen3ASRRealtimeGeneration": _HfExamplesInfo( + "Qwen/Qwen3-ASR-1.7B", + max_model_len=4096, + min_transformers_version="4.57", + enforce_eager=True, + hf_overrides={"architectures": ["Qwen3ASRRealtimeGeneration"]}, + is_available_online=False, + ), "RForConditionalGeneration": _HfExamplesInfo("YannQi/R-4B", trust_remote_code=True), "SkyworkR1VChatModel": _HfExamplesInfo( "Skywork/Skywork-R1V-38B", trust_remote_code=True diff --git a/vllm/entrypoints/openai/realtime/connection.py b/vllm/entrypoints/openai/realtime/connection.py index fe1b0f5f308..ffe871aa817 100644 --- a/vllm/entrypoints/openai/realtime/connection.py +++ b/vllm/entrypoints/openai/realtime/connection.py @@ -205,7 +205,7 @@ class RealtimeConnection: sampling_params = SamplingParams.from_optional( temperature=0.0, - max_tokens=1, + max_tokens=self.serving.model_cls.realtime_max_tokens, output_kind=RequestOutputKind.DELTA, skip_clone=True, ) diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 2c3ca1a5022..672857c2388 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -1063,6 +1063,10 @@ class SupportsRealtime(Protocol): supports_realtime: ClassVar[Literal[True]] = True + realtime_max_tokens: ClassVar[int] = 1 + """Maximum tokens to generate per streaming audio segment. + Override in subclasses based on the model's expected output length.""" + @classmethod async def buffer_realtime_audio( cls, diff --git a/vllm/model_executor/models/qwen3_asr_realtime.py b/vllm/model_executor/models/qwen3_asr_realtime.py new file mode 100644 index 00000000000..a149350d184 --- /dev/null +++ b/vllm/model_executor/models/qwen3_asr_realtime.py @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Copyright 2026 The Qwen team. +# Copyright 2023 The vLLM team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Inference-only Qwen3-ASR realtime model.""" + +import asyncio +from collections.abc import AsyncGenerator, Mapping + +import numpy as np +import torch + +from vllm.compilation.decorators import support_torch_compile +from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig +from vllm.inputs.data import PromptType, TokensPrompt +from vllm.logger import init_logger +from vllm.model_executor.models.interfaces import ( + SupportsRealtime, +) +from vllm.model_executor.models.qwen3_asr import ( + Qwen3ASRDummyInputsBuilder, + Qwen3ASRForConditionalGeneration, + Qwen3ASRMultiModalProcessor, + Qwen3ASRProcessingInfo, + _get_feat_extract_output_lengths, +) +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.cache import _I, BaseMultiModalProcessorCache +from vllm.multimodal.inputs import MultiModalKwargsOptionalItems +from vllm.multimodal.parse import MultiModalDataItems +from vllm.multimodal.processing import BaseDummyInputsBuilder +from vllm.multimodal.processing.processor import ( + MultiModalPromptUpdates, + PlaceholderFeaturesInfo, +) +from vllm.tokenizers import cached_tokenizer_from_config +from vllm.transformers_utils.processor import cached_processor_from_config + +logger = init_logger(__name__) + +_PRE_ALLOCATE_BUFFER_SIZE_IN_S = 60 + + +class Qwen3ASRRealtimeBuffer: + """Audio buffer for Qwen3-ASR realtime streaming. + + Accumulates audio samples and yields segments when enough + audio has been buffered for processing. + """ + + def __init__(self, sampling_rate: int, segment_duration_s: float = 5.0): + self._sampling_rate = sampling_rate + self._segment_size = int(segment_duration_s * sampling_rate) + + self._buffer_size = _PRE_ALLOCATE_BUFFER_SIZE_IN_S * sampling_rate + self._buffer: np.ndarray = np.empty(self._buffer_size, dtype=np.float32) + self._filled_len = 0 + + def write_audio(self, audio: np.ndarray) -> None: + put_end = self._filled_len + len(audio) + if put_end > self._buffer_size: + new_size = max(self._buffer_size * 2, put_end) + new_buffer = np.empty(new_size, dtype=np.float32) + new_buffer[: self._filled_len] = self._buffer[: self._filled_len] + self._buffer = new_buffer + self._buffer_size = new_size + + self._buffer[self._filled_len : put_end] = audio + self._filled_len = put_end + + def read_audio(self) -> np.ndarray | None: + if self._filled_len < self._segment_size: + return None + + segment = self._buffer[: self._segment_size].copy() + remaining = self._filled_len - self._segment_size + if remaining > 0: + self._buffer[:remaining] = self._buffer[ + self._segment_size : self._filled_len + ] + self._filled_len = remaining + return segment + + def flush(self) -> np.ndarray | None: + if self._filled_len == 0: + return None + audio = self._buffer[: self._filled_len].copy() + self._filled_len = 0 + return audio + + +class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor): + def __init__( + self, + info: _I, + dummy_inputs: BaseDummyInputsBuilder[_I], + *, + cache: BaseMultiModalProcessorCache | None = None, + ) -> None: + super().__init__(info, dummy_inputs, cache=None) + + def _maybe_apply_prompt_updates( + self, + mm_items: MultiModalDataItems, + prompt_ids: list[int], + mm_kwargs: MultiModalKwargsOptionalItems, + mm_prompt_updates: MultiModalPromptUpdates, + is_update_applied: bool, + ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: + audios = mm_kwargs.get("audio", []) + assert len(audios) == 1, ( + f"Expected only one audio input for realtime, got {len(audios)}" + ) + + audio_data = audios[0] + audio_feature_lengths = audio_data.get("audio_feature_lengths") + if audio_feature_lengths is not None: + if isinstance(audio_feature_lengths.data, torch.Tensor): + audio_len = _get_feat_extract_output_lengths( + audio_feature_lengths.data + ).item() + else: + audio_len = int( + _get_feat_extract_output_lengths( + torch.tensor(audio_feature_lengths.data) + ).item() + ) + else: + audio_len = 0 + + # Get audio_pad token ID and expand placeholder in prompt_ids + # so that MRoPE position computation matches seq_len. + tokenizer = self.info.get_tokenizer() + audio_pad_id = tokenizer.convert_tokens_to_ids("<|audio_pad|>") + + # Find the audio_pad token position and expand it to audio_len tokens + expanded_ids = list[int]() + pad_start_idx = -1 + for i, tid in enumerate(prompt_ids): + if tid == audio_pad_id and pad_start_idx == -1: + pad_start_idx = i + expanded_ids.extend([audio_pad_id] * audio_len) + else: + expanded_ids.append(tid) + + if pad_start_idx == -1: + pad_start_idx = 0 + + features_info = PlaceholderFeaturesInfo( + modality="audio", + item_idx=0, + start_idx=pad_start_idx, + tokens=audio_len * [audio_pad_id], + is_embed=None, + ) + return expanded_ids, {"audio": [features_info]} + + +# NOTE: A separate model class is required here because the multimodal +# processor registry binds one processor per model class. The realtime +# endpoint needs a different processor (Qwen3ASRRealtimeMultiModalProcessor) +# than the base transcription endpoint, so we register it on this subclass. +@MULTIMODAL_REGISTRY.register_processor( + Qwen3ASRRealtimeMultiModalProcessor, + info=Qwen3ASRProcessingInfo, + dummy_inputs=Qwen3ASRDummyInputsBuilder, +) +@support_torch_compile +class Qwen3ASRRealtimeGeneration(Qwen3ASRForConditionalGeneration, SupportsRealtime): + realtime_max_tokens = 64 + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__(vllm_config=vllm_config, prefix=prefix) + + @classmethod + async def buffer_realtime_audio( + cls, + audio_stream: AsyncGenerator[np.ndarray, None], + input_stream: asyncio.Queue[list[int]], + model_config: ModelConfig, + ) -> AsyncGenerator[PromptType, None]: + processor = cached_processor_from_config(model_config) + feature_extractor = processor.feature_extractor + sampling_rate = feature_extractor.sampling_rate + tokenizer = cached_tokenizer_from_config(model_config) + + # Use a small segment size for low-latency streaming. + segment_duration_s = 5.0 + buffer = Qwen3ASRRealtimeBuffer( + sampling_rate=sampling_rate, + segment_duration_s=segment_duration_s, + ) + + audio_placeholder = cls.get_placeholder_str("audio", 0) + prompt_template = ( + f"<|im_start|>user\n{audio_placeholder}<|im_end|>\n<|im_start|>assistant\n" + ) + + prompt_token_ids = tokenizer.encode(prompt_template) + + async for audio_chunk in audio_stream: + buffer.write_audio(audio_chunk) + + while (segment := buffer.read_audio()) is not None: + yield TokensPrompt( + prompt_token_ids=prompt_token_ids, + multi_modal_data={"audio": segment}, + ) + + remaining = buffer.flush() + if remaining is not None and len(remaining) > 0: + yield TokensPrompt( + prompt_token_ids=prompt_token_ids, + multi_modal_data={"audio": remaining}, + ) + + @classmethod + def get_speech_to_text_config( + cls, model_config: ModelConfig, task_type: str + ) -> SpeechToTextConfig: + processor = cached_processor_from_config(model_config) + feature_extractor = processor.feature_extractor + return SpeechToTextConfig( + max_audio_clip_s=None, + sample_rate=feature_extractor.sampling_rate, + min_energy_split_window_size=None, + ) diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 7e8d051a848..ca9468a192c 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -468,6 +468,10 @@ _MULTIMODAL_MODELS = { "qwen3_asr", "Qwen3ASRForConditionalGeneration", ), + "Qwen3ASRRealtimeGeneration": ( + "qwen3_asr_realtime", + "Qwen3ASRRealtimeGeneration", + ), "Qwen3VLForConditionalGeneration": ("qwen3_vl", "Qwen3VLForConditionalGeneration"), # noqa: E501 "Qwen3VLMoeForConditionalGeneration": ( "qwen3_vl_moe", From 5719a4e4e601fb91274294d25370b7aad656d629 Mon Sep 17 00:00:00 2001 From: Kata Coder Date: Sat, 21 Feb 2026 13:01:40 +0900 Subject: [PATCH 48/74] [Frontend] Support multimodal inputs for late-interaction scoring (ColQwen3) + NewModel: nvidia/nemotron-colembed (#34574) Signed-off-by: craftsangjae --- docs/models/pooling_models.md | 65 +++++- .../pooling/score/colqwen3_rerank_online.py | 130 +++++++++++- .../multimodal/pooling/test_colqwen3.py | 191 ++++++++++++++++++ tests/models/registry.py | 3 + vllm/entrypoints/llm.py | 29 +-- vllm/entrypoints/pooling/score/serving.py | 83 +++++--- vllm/entrypoints/pooling/score/utils.py | 81 +++++++- vllm/model_executor/models/colqwen3.py | 14 +- vllm/model_executor/models/registry.py | 1 + vllm/transformers_utils/config.py | 1 + 10 files changed, 532 insertions(+), 66 deletions(-) diff --git a/docs/models/pooling_models.md b/docs/models/pooling_models.md index d7f13f4e36a..a65bf4db5c9 100644 --- a/docs/models/pooling_models.md +++ b/docs/models/pooling_models.md @@ -382,6 +382,7 @@ ColQwen3 is based on [ColPali](https://arxiv.org/abs/2407.01449), which extends |---|---|---| | `ColQwen3` | Qwen3-VL | `TomoroAI/tomoro-colqwen3-embed-4b`, `TomoroAI/tomoro-colqwen3-embed-8b` | | `OpsColQwen3Model` | Qwen3-VL | `OpenSearch-AI/Ops-Colqwen3-4B`, `OpenSearch-AI/Ops-Colqwen3-8B` | +| `Qwen3VLNemotronEmbedModel` | Qwen3-VL | `nvidia/nemotron-colembed-vl-4b-v2`, `nvidia/nemotron-colembed-vl-8b-v2` | Start the server: @@ -389,7 +390,9 @@ Start the server: vllm serve TomoroAI/tomoro-colqwen3-embed-4b --max-model-len 4096 ``` -Then you can use the rerank endpoint: +#### Text-only scoring and reranking + +Use the `/rerank` endpoint: ```shell curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{ @@ -403,7 +406,7 @@ curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{ }' ``` -Or the score endpoint: +Or the `/score` endpoint: ```shell curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{ @@ -413,7 +416,57 @@ curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{ }' ``` -You can also get the raw token embeddings using the pooling endpoint with `token_embed` task: +#### Multi-modal scoring and reranking (text query × image documents) + +The `/score` and `/rerank` endpoints also accept multi-modal inputs directly. +Pass image documents using the `data_1`/`data_2` (for `/score`) or `documents` (for `/rerank`) fields +with a `content` list containing `image_url` and `text` parts — the same format used by the +OpenAI chat completion API: + +Score a text query against image documents: + +```shell +curl -s http://localhost:8000/score -H "Content-Type: application/json" -d '{ + "model": "TomoroAI/tomoro-colqwen3-embed-4b", + "data_1": "Retrieve the city of Beijing", + "data_2": [ + { + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}}, + {"type": "text", "text": "Describe the image."} + ] + } + ] +}' +``` + +Rerank image documents by a text query: + +```shell +curl -s http://localhost:8000/rerank -H "Content-Type: application/json" -d '{ + "model": "TomoroAI/tomoro-colqwen3-embed-4b", + "query": "Retrieve the city of Beijing", + "documents": [ + { + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}}, + {"type": "text", "text": "Describe the image."} + ] + }, + { + "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,"}}, + {"type": "text", "text": "Describe the image."} + ] + } + ], + "top_n": 2 +}' +``` + +#### Raw token embeddings + +You can also get the raw token embeddings using the `/pooling` endpoint with `token_embed` task: ```shell curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ @@ -423,7 +476,7 @@ curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ }' ``` -For **image inputs**, use the chat-style `messages` field so that the vLLM multimodal processor handles them correctly: +For **image inputs** via the pooling endpoint, use the chat-style `messages` field: ```shell curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ @@ -440,10 +493,10 @@ curl -s http://localhost:8000/pooling -H "Content-Type: application/json" -d '{ }' ``` -Examples can be found here: +#### Examples - Multi-vector retrieval: [examples/pooling/token_embed/colqwen3_token_embed_online.py](../../examples/pooling/token_embed/colqwen3_token_embed_online.py) -- Reranking: [examples/pooling/score/colqwen3_rerank_online.py](../../examples/pooling/score/colqwen3_rerank_online.py) +- Reranking (text + multi-modal): [examples/pooling/score/colqwen3_rerank_online.py](../../examples/pooling/score/colqwen3_rerank_online.py) ### BAAI/bge-m3 diff --git a/examples/pooling/score/colqwen3_rerank_online.py b/examples/pooling/score/colqwen3_rerank_online.py index ba1df150bc4..c7ab6e2372a 100644 --- a/examples/pooling/score/colqwen3_rerank_online.py +++ b/examples/pooling/score/colqwen3_rerank_online.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# ruff: noqa: E501 """ -Example of using ColQwen3 late interaction model for reranking. +Example of using ColQwen3 late interaction model for reranking and scoring. ColQwen3 is a multi-modal ColBERT-style model based on Qwen3-VL. It produces per-token embeddings and uses MaxSim scoring for retrieval @@ -14,13 +15,65 @@ Then run this script: python colqwen3_rerank_online.py """ +import base64 +from io import BytesIO + import requests +from PIL import Image MODEL = "TomoroAI/tomoro-colqwen3-embed-4b" BASE_URL = "http://127.0.0.1:8000" headers = {"accept": "application/json", "Content-Type": "application/json"} +# ── Image helpers ────────────────────────────────────────── + + +def load_image(url: str) -> Image.Image: + """Download an image from URL (handles Wikimedia 403).""" + for hdrs in ( + {}, + {"User-Agent": "Mozilla/5.0 (compatible; ColQwen3-demo/1.0)"}, + ): + resp = requests.get(url, headers=hdrs, timeout=15) + if resp.status_code == 403: + continue + resp.raise_for_status() + return Image.open(BytesIO(resp.content)).convert("RGB") + raise RuntimeError(f"Could not fetch image from {url}") + + +def encode_image_base64(image: Image.Image) -> str: + """Encode a PIL image to a base64 data URI.""" + buf = BytesIO() + image.save(buf, format="PNG") + return "data:image/png;base64," + base64.b64encode(buf.getvalue()).decode() + + +def make_image_content(image_url: str, text: str = "Describe the image.") -> dict: + """Build a ScoreMultiModalParam dict from an image URL.""" + image = load_image(image_url) + return { + "content": [ + { + "type": "image_url", + "image_url": {"url": encode_image_base64(image)}, + }, + {"type": "text", "text": text}, + ] + } + + +# ── Sample image URLs ───────────────────────────────────── + +IMAGE_URLS = { + "beijing": "https://upload.wikimedia.org/wikipedia/commons/6/61/Beijing_skyline_at_night.JPG", + "london": "https://upload.wikimedia.org/wikipedia/commons/4/49/London_skyline.jpg", + "singapore": "https://upload.wikimedia.org/wikipedia/commons/2/27/Singapore_skyline_2022.jpg", +} + +# ── Text-only examples ──────────────────────────────────── + def rerank_text(): """Text-only reranking via /rerank endpoint.""" @@ -120,11 +173,86 @@ def score_text_top_n(): print(f" {response.text[:300]}") +# ── Multi-modal examples (text query × image documents) ── + + +def score_text_vs_images(): + """Score a text query against image documents via /score.""" + print() + print("=" * 60) + print("4. Multi-modal scoring: text query vs image docs (/score)") + print("=" * 60) + + query = "Retrieve the city of Beijing" + labels = list(IMAGE_URLS.keys()) + print(f"\n Loading {len(labels)} images...") + image_contents = [make_image_content(IMAGE_URLS[name]) for name in labels] + + data = { + "model": MODEL, + "data_1": query, + "data_2": image_contents, + } + + response = requests.post(f"{BASE_URL}/score", headers=headers, json=data) + + if response.status_code == 200: + result = response.json() + print(f'\n Query: "{query}"\n') + for item in result["data"]: + idx = item["index"] + print(f" Doc {idx} [{labels[idx]}] score={item['score']:.4f}") + else: + print(f" Request failed: {response.status_code}") + print(f" {response.text[:300]}") + + +def rerank_text_vs_images(): + """Rerank image documents by a text query via /rerank.""" + print() + print("=" * 60) + print("5. Multi-modal reranking: text query vs image docs (/rerank)") + print("=" * 60) + + query = "Retrieve the city of London" + labels = list(IMAGE_URLS.keys()) + print(f"\n Loading {len(labels)} images...") + image_contents = [make_image_content(IMAGE_URLS[name]) for name in labels] + + data = { + "model": MODEL, + "query": query, + "documents": image_contents, + "top_n": 2, + } + + response = requests.post(f"{BASE_URL}/rerank", headers=headers, json=data) + + if response.status_code == 200: + result = response.json() + print(f'\n Query: "{query}"') + print(f" Top {data['top_n']} results:\n") + for item in result["results"]: + idx = item["index"] + print(f" [{item['relevance_score']:.4f}] {labels[idx]}") + else: + print(f" Request failed: {response.status_code}") + print(f" {response.text[:300]}") + + +# ── Main ────────────────────────────────────────────────── + + def main(): + # Text-only rerank_text() score_text() score_text_top_n() + # Multi-modal (text query × image documents) + score_text_vs_images() + rerank_text_vs_images() + if __name__ == "__main__": main() diff --git a/tests/models/multimodal/pooling/test_colqwen3.py b/tests/models/multimodal/pooling/test_colqwen3.py index 51080cc108b..0cc4c343b3d 100644 --- a/tests/models/multimodal/pooling/test_colqwen3.py +++ b/tests/models/multimodal/pooling/test_colqwen3.py @@ -7,19 +7,31 @@ ColBERT-style late interaction scoring (MaxSim). It produces per-token embeddings for both text and image inputs. """ +import base64 +from io import BytesIO + import pytest import torch +from PIL import Image + +from vllm.entrypoints.chat_utils import ( + ChatCompletionContentPartImageParam, + ChatCompletionContentPartTextParam, +) +from vllm.entrypoints.pooling.score.utils import ScoreMultiModalParam from ....conftest import VllmRunner MODELS = [ "TomoroAI/tomoro-colqwen3-embed-4b", "OpenSearch-AI/Ops-Colqwen3-4B", + "nvidia/nemotron-colembed-vl-4b-v2", ] EMBED_DIMS = { "TomoroAI/tomoro-colqwen3-embed-4b": 320, "OpenSearch-AI/Ops-Colqwen3-4B": 2560, + "nvidia/nemotron-colembed-vl-4b-v2": 2560, } TEXT_QUERIES = [ @@ -33,6 +45,43 @@ TEXT_DOCUMENTS = [ ] DTYPE = "half" +GPU_MEMORY_UTILIZATION = 0.7 + + +def _make_base64_image( + width: int = 64, height: int = 64, color: tuple[int, int, int] = (255, 0, 0) +) -> str: + """Create a small solid-color PNG image and return its base64 data URI.""" + img = Image.new("RGB", (width, height), color) + buf = BytesIO() + img.save(buf, format="PNG") + b64 = base64.b64encode(buf.getvalue()).decode() + return f"data:image/png;base64,{b64}" + + +def _make_image_mm_param( + image_uri: str, + text: str | None = None, +) -> ScoreMultiModalParam: + """Build a ScoreMultiModalParam containing an image (and optional text).""" + content: list = [ + ChatCompletionContentPartImageParam( + type="image_url", + image_url={"url": image_uri}, + ), + ] + if text is not None: + content.append( + ChatCompletionContentPartTextParam(type="text", text=text), + ) + return ScoreMultiModalParam(content=content) + + +def _make_text_mm_param(text: str) -> ScoreMultiModalParam: + """Build a ScoreMultiModalParam containing only text.""" + return ScoreMultiModalParam( + content=[ChatCompletionContentPartTextParam(type="text", text=text)], + ) def _run_token_embed_test( @@ -48,6 +97,7 @@ def _run_token_embed_test( dtype=dtype, max_model_len=4096, enforce_eager=True, + gpu_memory_utilization=GPU_MEMORY_UTILIZATION, ) as vllm_model: outputs = vllm_model.token_embed([TEXT_QUERIES[0]]) @@ -83,6 +133,7 @@ def _run_late_interaction_test( dtype=dtype, max_model_len=4096, enforce_eager=True, + gpu_memory_utilization=GPU_MEMORY_UTILIZATION, ) as vllm_model: q_outputs = vllm_model.token_embed([TEXT_QUERIES[0]]) d_outputs = vllm_model.token_embed([TEXT_DOCUMENTS[0]]) @@ -118,6 +169,7 @@ def _run_relevance_test( dtype=dtype, max_model_len=4096, enforce_eager=True, + gpu_memory_utilization=GPU_MEMORY_UTILIZATION, ) as vllm_model: scores = vllm_model.score(query, documents) @@ -154,3 +206,142 @@ def test_colqwen3_relevance_ordering( dtype: str, ) -> None: _run_relevance_test(vllm_runner, model, dtype=dtype) + + +# ── Multimodal scoring tests ──────────────────────────────── + + +def _run_multimodal_text_query_image_docs_test( + vllm_runner: type[VllmRunner], + model: str, + *, + dtype: str, +) -> None: + """Score a text query against image documents via the multimodal path. + + Verifies that score_data_to_prompts correctly handles image content + and produces valid MaxSim scores. + """ + red_image = _make_base64_image(64, 64, color=(255, 0, 0)) + blue_image = _make_base64_image(64, 64, color=(0, 0, 255)) + + query = "Describe the red object" + image_docs = [ + _make_image_mm_param(red_image), + _make_image_mm_param(blue_image), + ] + + with vllm_runner( + model, + runner="pooling", + dtype=dtype, + max_model_len=4096, + enforce_eager=True, + gpu_memory_utilization=GPU_MEMORY_UTILIZATION, + ) as vllm_model: + scores = vllm_model.llm.score(query, image_docs) + + assert len(scores) == 2 + for s in scores: + assert isinstance(s.outputs.score, float) + + +def _run_multimodal_mixed_docs_test( + vllm_runner: type[VllmRunner], + model: str, + *, + dtype: str, +) -> None: + """Score a text query against a mix of text and image documents. + + Ensures the late-interaction path handles heterogeneous document + types (plain strings alongside ScoreMultiModalParam images) in + a single call. + """ + red_image = _make_base64_image(64, 64, color=(255, 0, 0)) + + query = "What is the capital of France?" + documents: list = [ + "The capital of France is Paris.", + _make_image_mm_param(red_image), + ] + + with vllm_runner( + model, + runner="pooling", + dtype=dtype, + max_model_len=4096, + enforce_eager=True, + gpu_memory_utilization=GPU_MEMORY_UTILIZATION, + ) as vllm_model: + scores = vllm_model.llm.score(query, documents) + + assert len(scores) == 2 + for s in scores: + assert isinstance(s.outputs.score, float) + # Text document about France should score higher than a random image + assert scores[0].outputs.score > scores[1].outputs.score + + +def _run_multimodal_image_query_text_docs_test( + vllm_runner: type[VllmRunner], + model: str, + *, + dtype: str, +) -> None: + """Score an image query against text documents. + + Verifies the reverse direction: multimodal query with text-only + documents through the late-interaction scoring path. + """ + red_image = _make_base64_image(64, 64, color=(255, 0, 0)) + image_query = _make_image_mm_param(red_image, text="red color") + + documents = [ + "A bright red sports car.", + "The weather forecast shows rain tomorrow.", + ] + + with vllm_runner( + model, + runner="pooling", + dtype=dtype, + max_model_len=4096, + enforce_eager=True, + gpu_memory_utilization=GPU_MEMORY_UTILIZATION, + ) as vllm_model: + scores = vllm_model.llm.score(image_query, documents) + + assert len(scores) == 2 + for s in scores: + assert isinstance(s.outputs.score, float) + + +@pytest.mark.parametrize("model", MODELS) +@pytest.mark.parametrize("dtype", [DTYPE]) +def test_colqwen3_multimodal_text_query_image_docs( + vllm_runner, + model: str, + dtype: str, +) -> None: + _run_multimodal_text_query_image_docs_test(vllm_runner, model, dtype=dtype) + + +@pytest.mark.parametrize("model", MODELS) +@pytest.mark.parametrize("dtype", [DTYPE]) +def test_colqwen3_multimodal_mixed_docs( + vllm_runner, + model: str, + dtype: str, +) -> None: + _run_multimodal_mixed_docs_test(vllm_runner, model, dtype=dtype) + + +@pytest.mark.parametrize("model", MODELS) +@pytest.mark.parametrize("dtype", [DTYPE]) +def test_colqwen3_multimodal_image_query_text_docs( + vllm_runner, + model: str, + dtype: str, +) -> None: + _run_multimodal_image_query_text_docs_test(vllm_runner, model, dtype=dtype) diff --git a/tests/models/registry.py b/tests/models/registry.py index de8d33e557c..b37dfb6d8cd 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -603,6 +603,9 @@ _EMBEDDING_EXAMPLE_MODELS = { "OpsColQwen3Model": _HfExamplesInfo( "OpenSearch-AI/Ops-Colqwen3-4B", trust_remote_code=True ), + "Qwen3VLNemotronEmbedModel": _HfExamplesInfo( + "nvidia/nemotron-colembed-vl-4b-v2", + ), "SiglipModel": _HfExamplesInfo("google/siglip-base-patch16-224"), "PrithviGeoSpatialMAE": _HfExamplesInfo( "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11", diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index f1b32c750b0..deff23df46a 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -50,6 +50,7 @@ from vllm.entrypoints.pooling.score.utils import ( compress_token_type_ids, compute_maxsim_score, get_score_prompt, + score_data_to_prompts, validate_score_input, ) from vllm.entrypoints.utils import log_non_default_args @@ -1395,25 +1396,13 @@ class LLM: tokenizer = self.get_tokenizer() - # Extract text from ScoreData - text_1: list[str] = [] - for text in data_1: - if not isinstance(text, str): - raise NotImplementedError( - "Late interaction scores currently do not support multimodal input." - ) - text_1.append(text) + # Convert ScoreData to PromptType (handles both text and multimodal) + model_config = self.model_config + prompts_1 = score_data_to_prompts(data_1, "query", model_config) + prompts_2 = score_data_to_prompts(data_2, "document", model_config) - text_2: list[str] = [] - for text in data_2: - if not isinstance(text, str): - raise NotImplementedError( - "Late interaction scores currently do not support multimodal input." - ) - text_2.append(text) - - encoded_output = self.encode( - text_1 + text_2, + encoded_output: list[PoolingRequestOutput] = self.encode( + prompts_1 + prompts_2, use_tqdm=use_tqdm, lora_request=lora_request, pooling_params=pooling_params, @@ -1421,8 +1410,8 @@ class LLM: tokenization_kwargs=tokenization_kwargs, ) - encoded_output_1 = encoded_output[0 : len(text_1)] - encoded_output_2 = encoded_output[len(text_1) :] + encoded_output_1: list[PoolingRequestOutput] = encoded_output[: len(prompts_1)] + encoded_output_2: list[PoolingRequestOutput] = encoded_output[len(prompts_1) :] if len(encoded_output_1) == 1: encoded_output_1 = encoded_output_1 * len(encoded_output_2) diff --git a/vllm/entrypoints/pooling/score/serving.py b/vllm/entrypoints/pooling/score/serving.py index fe01f9cf697..135853d6f20 100644 --- a/vllm/entrypoints/pooling/score/serving.py +++ b/vllm/entrypoints/pooling/score/serving.py @@ -33,6 +33,7 @@ from vllm.entrypoints.pooling.score.utils import ( compress_token_type_ids, compute_maxsim_score, get_score_prompt, + parse_score_data_single, validate_score_input, ) from vllm.inputs.data import ProcessorInputs, TokensPrompt, token_inputs @@ -174,6 +175,43 @@ class ServingScores(OpenAIServing): return final_res_batch + def _preprocess_late_interaction_item( + self, + data: ScoreData, + role: str, + request: RerankRequest | ScoreRequest, + tokenizer: TokenizerLike, + tokenization_kwargs: dict[str, Any], + ) -> tuple[str, TokensPrompt]: + """Parse a single ScoreData into a text + optional multimodal + TokensPrompt for late-interaction encoding. + + For plain strings, tokenises directly. + For multimodal content parts, extracts text and multi_modal_data. + """ + model_config = self.model_config + + if isinstance(data, str): + text, mm_data, mm_uuids = data, None, None + else: + text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config) + + prompt_inputs = tokenizer(text, **tokenization_kwargs) + self._validate_input(request, prompt_inputs["input_ids"], text) + + engine_prompt = TokensPrompt( + prompt_token_ids=prompt_inputs["input_ids"], + ) + + if mm_data is not None: + engine_prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + engine_prompt["multi_modal_uuids"] = mm_uuids + if request.mm_processor_kwargs is not None: + engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs + + return text, engine_prompt + async def _late_interaction_score( self, data_1: list[ScoreData], @@ -189,37 +227,36 @@ class ServingScores(OpenAIServing): Encodes queries and documents into per-token embeddings, then computes MaxSim: sum over query tokens of max similarity to any document token. """ - input_texts: list[str] = [] - for text in data_1 + data_2: - if not isinstance(text, str): - raise NotImplementedError( - "Late interaction scores currently do not support multimodal input." - ) - input_texts.append(text) - model_config = self.model_config tokenizer = self.renderer.get_tokenizer() + tokenization_kwargs = request.build_tok_params(model_config).get_encode_kwargs() - encode_async = make_async( - tokenizer.encode, + all_data = data_1 + data_2 + roles = ["query"] * len(data_1) + ["document"] * len(data_2) + + preprocess_async = make_async( + self._preprocess_late_interaction_item, executor=self._tokenizer_executor, ) - tokenization_kwargs = request.build_tok_params(model_config).get_encode_kwargs() - tokenized_prompts = await asyncio.gather( - *(encode_async(t, **tokenization_kwargs) for t in input_texts) + preprocessed = await asyncio.gather( + *( + preprocess_async( + data=d, + role=r, + request=request, + tokenizer=tokenizer, + tokenization_kwargs=tokenization_kwargs, + ) + for d, r in zip(all_data, roles) + ) ) - engine_prompts: list[ProcessorInputs] = [] - for tok_result, input_text in zip(tokenized_prompts, input_texts): - text_token_prompt = self._validate_input(request, tok_result, input_text) - - engine_prompts.append( - token_inputs( - text_token_prompt["prompt_token_ids"], - prompt=input_text, - ) - ) + input_texts: list[str] = [] + engine_prompts: list[TokensPrompt] = [] + for text, engine_prompt in preprocessed: + input_texts.append(text) + engine_prompts.append(engine_prompt) # Schedule the request and get the result generator. generators: list[AsyncGenerator[PoolingRequestOutput, None]] = [] diff --git a/vllm/entrypoints/pooling/score/utils.py b/vllm/entrypoints/pooling/score/utils.py index 7d00f42f5df..60e71ff7395 100644 --- a/vllm/entrypoints/pooling/score/utils.py +++ b/vllm/entrypoints/pooling/score/utils.py @@ -21,6 +21,7 @@ from vllm.entrypoints.chat_utils import ( _parse_chat_message_content_parts, ) from vllm.inputs import TokensPrompt +from vllm.inputs.data import PromptType, TextPrompt from vllm.model_executor.models.interfaces import supports_score_template from vllm.multimodal.inputs import MultiModalDataDict, MultiModalUUIDDict from vllm.outputs import PoolingRequestOutput @@ -153,31 +154,91 @@ def validate_score_input( return score_input_1, score_input_2 +def _ensure_str(content: list[ConversationMessage]) -> str: + """Extract a single string prompt from parsed conversation content.""" + assert len(content) == 1 + prompt = content[0]["content"] + if prompt is not None and isinstance(prompt, str): + return cast(str, prompt) + raise ValueError(f"Only string content is supported, but got {content}.") + + def parse_score_data( data_1: ScoreData, data_2: ScoreData, model_config: ModelConfig, ) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]: + """Parse a query-document pair into text prompts and shared multi-modal + data. + + Uses a **single** :class:`MultiModalItemTracker` so that multi-modal + items from both inputs are merged into one ``mm_data`` dict. This is + the correct behaviour for cross-encoder scoring, where query and + document are concatenated into a single model prompt. + """ mm_tracker = MultiModalItemTracker(model_config) content_1 = _parse_score_content("query", data_1, mm_tracker) content_2 = _parse_score_content("document", data_2, mm_tracker) - def ensure_str(content: list[ConversationMessage]) -> str: - assert len(content) == 1 - prompt = content[0]["content"] - if prompt is not None and isinstance(prompt, str): - return cast(str, prompt) - else: - raise ValueError(f"Only string content is supported, but got {content}.") - - prompt_1 = ensure_str(content_1) - prompt_2 = ensure_str(content_2) + prompt_1 = _ensure_str(content_1) + prompt_2 = _ensure_str(content_2) mm_items, mm_uuids = mm_tracker.resolve_items() return prompt_1, prompt_2, mm_items, mm_uuids +def parse_score_data_single( + data: ScoreData, + role: str, + model_config: ModelConfig, +) -> tuple[str, MultiModalDataDict | None, MultiModalUUIDDict | None]: + """Parse **one** ScoreData into a text prompt and its own multi-modal + data. + + Unlike :func:`parse_score_data`, each call creates an **independent** + :class:`MultiModalItemTracker` so multi-modal items are kept separate. + This is the correct behaviour for late-interaction scoring, where + query and document are encoded independently. + """ + mm_tracker = MultiModalItemTracker(model_config) + content = _parse_score_content(role, data, mm_tracker) + + prompt = _ensure_str(content) + mm_items, mm_uuids = mm_tracker.resolve_items() + return prompt, mm_items, mm_uuids + + +def score_data_to_prompts( + data_list: list[ScoreData], + role: str, + model_config: ModelConfig, +) -> list[PromptType]: + """Convert a list of ScoreData into PromptType objects. + + For plain text inputs, returns the string directly. + For multimodal inputs (list of content parts), parses them into + a :class:`TextPrompt` with attached ``multi_modal_data`` / + ``multi_modal_uuids``. + + This is used by late-interaction scoring where each query/document + is encoded independently. + """ + prompts: list[PromptType] = [] + for data in data_list: + if isinstance(data, str): + prompts.append(data) + else: + text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config) + prompt: TextPrompt = TextPrompt(prompt=text) + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + prompts.append(prompt) + return prompts + + def _parse_score_content( role: str, data: ScoreData, diff --git a/vllm/model_executor/models/colqwen3.py b/vllm/model_executor/models/colqwen3.py index f60d93f8e67..7513c01e831 100644 --- a/vllm/model_executor/models/colqwen3.py +++ b/vllm/model_executor/models/colqwen3.py @@ -16,6 +16,7 @@ Based on: Qwen3-VL backbone with custom text projection Target models: - TomoroAI/tomoro-colqwen3-embed-8b - OpenSearch-AI/Ops-Colqwen3-4B +- nvidia/nemotron-colembed-vl-4b-v2 """ from collections.abc import Iterable, Mapping @@ -229,13 +230,14 @@ class ColQwen3Model( if not isinstance(hidden_states, torch.Tensor): return hidden_states # type: ignore - proj_dtype = self.custom_text_proj.weight.dtype # type: ignore - if hidden_states.dtype != proj_dtype: - hidden_states = hidden_states.to(proj_dtype) + if self.custom_text_proj is not None: + proj_dtype = self.custom_text_proj.weight.dtype + if hidden_states.dtype != proj_dtype: + hidden_states = hidden_states.to(proj_dtype) + hidden_states = self.custom_text_proj(hidden_states) - # Project to embedding dimension and L2 normalize - proj = self.custom_text_proj(hidden_states) # type: ignore - return torch.nn.functional.normalize(proj, p=2, dim=-1) + # L2 normalize + return torch.nn.functional.normalize(hidden_states, p=2, dim=-1) # Names used for the projection layer across different ColQwen3 variants _PROJ_LAYER_NAMES = { diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index ca9468a192c..598df91d996 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -256,6 +256,7 @@ _EMBEDDING_MODELS = { "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501 "ColQwen3": ("colqwen3", "ColQwen3Model"), "OpsColQwen3Model": ("colqwen3", "ColQwen3Model"), + "Qwen3VLNemotronEmbedModel": ("colqwen3", "ColQwen3Model"), "SiglipModel": ("siglip", "SiglipEmbeddingModel"), # Technically Terratorch models work on images, both in # input and output. I am adding it here because it piggy-backs on embedding diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index ece5614fc7b..852e1d2a31d 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -76,6 +76,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( chatglm="ChatGLMConfig", colqwen3="ColQwen3Config", ops_colqwen3="OpsColQwen3Config", + qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig", deepseek_vl_v2="DeepseekVLV2Config", deepseek_v32="DeepseekV3Config", flex_olmo="FlexOlmoConfig", From 991d6bff38ff02f7cf47a3833efce58b27db8bb8 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 20 Feb 2026 22:03:32 -0600 Subject: [PATCH 49/74] [CI][MCP][Harmony] Heavy refactoring Harmony & MCP response tests and stabilizing with deterministic test infrastructure (#33949) Signed-off-by: Andreas Karatzas --- .../entrypoints/openai/responses/conftest.py | 171 ++++ .../openai/responses/test_harmony.py | 968 ++++++++---------- .../openai/responses/test_mcp_tools.py | 426 ++++---- .../openai/responses/test_parsable_context.py | 149 ++- .../openai/responses/test_simple.py | 14 +- tests/utils.py | 153 ++- .../openai/parser/harmony_utils.py | 111 +- vllm/entrypoints/openai/responses/context.py | 34 +- vllm/entrypoints/openai/responses/serving.py | 38 +- vllm/envs.py | 7 + 10 files changed, 1186 insertions(+), 885 deletions(-) diff --git a/tests/entrypoints/openai/responses/conftest.py b/tests/entrypoints/openai/responses/conftest.py index c9b524d4032..e88c16d1de4 100644 --- a/tests/entrypoints/openai/responses/conftest.py +++ b/tests/entrypoints/openai/responses/conftest.py @@ -1,7 +1,22 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from __future__ import annotations + +import json +import logging +from collections.abc import Callable +from typing import Any + import pytest +logger = logging.getLogger(__name__) + +BASE_TEST_ENV = { + # The day vLLM said "hello world" on arxiv 🚀 + "VLLM_SYSTEM_START_DATE": "2023-09-12", +} +DEFAULT_MAX_RETRIES = 3 + @pytest.fixture def pairs_of_event_types() -> dict[str, str]: @@ -28,3 +43,159 @@ def pairs_of_event_types() -> dict[str, str]: } # fmt: on return event_pairs + + +async def retry_for_tool_call( + client, + *, + model: str, + expected_tool_type: str, + max_retries: int = DEFAULT_MAX_RETRIES, + **create_kwargs: Any, +): + """Call ``client.responses.create`` up to *max_retries* times, returning + the first response that contains an output item of *expected_tool_type*. + + Returns the **last** response if none match so the caller's assertions + fire with a clear diagnostic. + """ + last_response = None + for attempt in range(max_retries): + response = await client.responses.create(model=model, **create_kwargs) + last_response = response + if any( + getattr(item, "type", None) == expected_tool_type + for item in response.output + ): + return response + assert last_response is not None + return last_response + + +async def retry_streaming_for( + client, + *, + model: str, + validate_events: Callable[[list], bool], + max_retries: int = DEFAULT_MAX_RETRIES, + **create_kwargs: Any, +) -> list: + """Call ``client.responses.create(stream=True)`` up to *max_retries* + times, returning the first event list where *validate_events* returns + ``True``. + """ + last_events: list = [] + for attempt in range(max_retries): + stream = await client.responses.create( + model=model, stream=True, **create_kwargs + ) + events: list = [] + async for event in stream: + events.append(event) + last_events = events + if validate_events(events): + return events + return last_events + + +def has_output_type(response, type_name: str) -> bool: + """Return True if *response* has at least one output item of *type_name*.""" + return any(getattr(item, "type", None) == type_name for item in response.output) + + +def events_contain_type(events: list, type_substring: str) -> bool: + """Return True if any event's type contains *type_substring*.""" + return any(type_substring in getattr(e, "type", "") for e in events) + + +def validate_streaming_event_stack( + events: list, pairs_of_event_types: dict[str, str] +) -> None: + """Validate that streaming events are properly nested/paired.""" + stack: list[str] = [] + for event in events: + etype = event.type + if etype == "response.created": + stack.append(etype) + elif etype == "response.completed": + assert stack and stack[-1] == pairs_of_event_types[etype], ( + f"Unexpected stack top for {etype}: " + f"got {stack[-1] if stack else ''}" + ) + stack.pop() + elif etype.endswith("added") or etype == "response.mcp_call.in_progress": + stack.append(etype) + elif etype.endswith("delta"): + if stack and stack[-1] == etype: + continue + stack.append(etype) + elif etype.endswith("done") or etype == "response.mcp_call.completed": + assert etype in pairs_of_event_types, f"Unknown done event: {etype}" + expected_start = pairs_of_event_types[etype] + assert stack and stack[-1] == expected_start, ( + f"Stack mismatch for {etype}: " + f"expected {expected_start}, " + f"got {stack[-1] if stack else ''}" + ) + stack.pop() + assert len(stack) == 0, f"Unclosed events on stack: {stack}" + + +def log_response_diagnostics( + response, + *, + label: str = "Response Diagnostics", +) -> dict[str, Any]: + """Extract and log diagnostic info from a Responses API response. + + Logs reasoning, tool-call attempts, MCP items, and output types so + that CI output (``pytest -s`` or ``--log-cli-level=INFO``) gives + full visibility into model behaviour even on passing runs. + + Returns the extracted data so callers can make additional assertions + if needed. + """ + reasoning_texts = [ + text + for item in response.output + if getattr(item, "type", None) == "reasoning" + for content in getattr(item, "content", []) + if (text := getattr(content, "text", None)) + ] + + tool_call_attempts = [ + { + "recipient": msg.get("recipient"), + "channel": msg.get("channel"), + } + for msg in response.output_messages + if (msg.get("recipient") or "").startswith("python") + ] + + mcp_items = [ + { + "name": getattr(item, "name", None), + "status": getattr(item, "status", None), + } + for item in response.output + if getattr(item, "type", None) == "mcp_call" + ] + + output_types = [getattr(o, "type", None) for o in response.output] + + diagnostics = { + "model_attempted_tool_calls": bool(tool_call_attempts), + "tool_call_attempts": tool_call_attempts, + "mcp_items": mcp_items, + "reasoning": reasoning_texts, + "output_text": response.output_text, + "output_types": output_types, + } + + logger.info( + "\n====== %s ======\n%s\n==============================", + label, + json.dumps(diagnostics, indent=2, default=str), + ) + + return diagnostics diff --git a/tests/entrypoints/openai/responses/test_harmony.py b/tests/entrypoints/openai/responses/test_harmony.py index 6af1270abf1..9d97800a9da 100644 --- a/tests/entrypoints/openai/responses/test_harmony.py +++ b/tests/entrypoints/openai/responses/test_harmony.py @@ -1,17 +1,32 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Integration tests for the Harmony-based Responses API.""" + +from __future__ import annotations + import importlib.util import json +import logging import time +from typing import Any import pytest import pytest_asyncio +import requests from openai import BadRequestError, NotFoundError, OpenAI -from openai_harmony import ( - Message, -) +from openai_harmony import Message from ....utils import RemoteOpenAIServer +from .conftest import ( + BASE_TEST_ENV, + events_contain_type, + has_output_type, + retry_for_tool_call, + retry_streaming_for, + validate_streaming_event_stack, +) + +logger = logging.getLogger(__name__) MODEL_NAME = "openai/gpt-oss-20b" @@ -32,20 +47,72 @@ GET_WEATHER_SCHEMA = { } +def get_weather(latitude, longitude): + try: + response = requests.get( + f"https://api.open-meteo.com/v1/forecast?" + f"latitude={latitude}&longitude={longitude}" + f"¤t=temperature_2m,wind_speed_10m" + f"&hourly=temperature_2m,relative_humidity_2m," + f"wind_speed_10m", + timeout=10, + ) + data = response.json() + return data["current"]["temperature_2m"] + except (requests.RequestException, KeyError) as e: + logger.warning( + "External weather API call failed (%s), " + "returning fake value. This does not affect " + "test correctness — only the tool-calling " + "protocol is under test.", + e, + ) + return 15.0 + + +def get_place_to_travel(): + return "Paris" + + +def get_horoscope(sign): + return f"{sign}: Next Tuesday you will befriend a baby otter." + + +def call_function(name, args): + logger.info("Calling function %s with args %s", name, args) + dispatch = { + "get_weather": lambda: get_weather(**args), + "get_place_to_travel": lambda: get_place_to_travel(), + "get_horoscope": lambda: get_horoscope(**args), + } + if name not in dispatch: + raise ValueError(f"Unknown function: {name}") + result = dispatch[name]() + logger.info("Function %s returned: %s", name, result) + return result + + @pytest.fixture(scope="module") def server(): assert importlib.util.find_spec("gpt_oss") is not None, ( "Harmony tests require gpt_oss package to be installed" ) - - args = ["--enforce-eager", "--tool-server", "demo", "--max_model_len", "5000"] - env_dict = dict( - VLLM_ENABLE_RESPONSES_API_STORE="1", - PYTHON_EXECUTION_BACKEND="dangerously_use_uv", - VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS="code_interpreter,container,web_search_preview", - VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS="1", - ) - + args = [ + "--enforce-eager", + "--tool-server", + "demo", + "--max_model_len", + "5000", + ] + env_dict = { + **BASE_TEST_ENV, + "VLLM_ENABLE_RESPONSES_API_STORE": "1", + "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv", + "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": ( + "code_interpreter,container,web_search_preview" + ), + "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1", + } with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: yield remote_server @@ -159,7 +226,10 @@ async def test_structured_output(client: OpenAI, model_name: str): "properties": { "name": {"type": "string"}, "date": {"type": "string"}, - "participants": {"type": "array", "items": {"type": "string"}}, + "participants": { + "type": "array", + "items": {"type": "string"}, + }, }, "required": ["name", "date", "participants"], "additionalProperties": False, @@ -210,7 +280,9 @@ async def test_store(client: OpenAI, model_name: str): except NotFoundError: is_not_found = True - assert is_not_found == (not store) + assert is_not_found == (not store), ( + f"store={store}: expected not_found={not store}, got {is_not_found}" + ) @pytest.mark.asyncio @@ -254,10 +326,8 @@ async def test_background_cancel(client: OpenAI, model_name: str): @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_stateful_multi_turn(client: OpenAI, model_name: str): response1 = await client.responses.create( - model=model_name, - input="What is 123 * 456?", + model=model_name, input="What is 123 * 456?" ) - assert response1 is not None assert response1.status == "completed" response2 = await client.responses.create( @@ -265,7 +335,6 @@ async def test_stateful_multi_turn(client: OpenAI, model_name: str): input="What if I increase both numbers by 1?", previous_response_id=response1.id, ) - assert response2 is not None assert response2.status == "completed" response3 = await client.responses.create( @@ -273,7 +342,6 @@ async def test_stateful_multi_turn(client: OpenAI, model_name: str): input="Divide the result by 2.", previous_response_id=response2.id, ) - assert response3 is not None assert response3.status == "completed" @@ -282,37 +350,19 @@ async def test_stateful_multi_turn(client: OpenAI, model_name: str): async def test_streaming_types( pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str ): - prompts = [ - "tell me a story about a cat in 20 words", - ] + stream = await client.responses.create( + model=model_name, + input="tell me a story about a cat in 20 words", + reasoning={"effort": "low"}, + tools=[], + stream=True, + background=False, + ) + events = [] + async for event in stream: + events.append(event) - for prompt in prompts: - response = await client.responses.create( - model=model_name, - input=prompt, - reasoning={"effort": "low"}, - tools=[], - stream=True, - background=False, - ) - - stack_of_event_types = [] - async for event in response: - if event.type == "response.created": - stack_of_event_types.append(event.type) - elif event.type == "response.completed": - assert stack_of_event_types[-1] == pairs_of_event_types[event.type] - stack_of_event_types.pop() - if event.type.endswith("added"): - stack_of_event_types.append(event.type) - elif event.type.endswith("delta"): - if stack_of_event_types[-1] == event.type: - continue - stack_of_event_types.append(event.type) - elif event.type.endswith("done"): - assert stack_of_event_types[-1] == pairs_of_event_types[event.type] - stack_of_event_types.pop() - assert len(stack_of_event_types) == 0 + validate_streaming_event_stack(events, pairs_of_event_types) @pytest.mark.asyncio @@ -320,37 +370,21 @@ async def test_streaming_types( async def test_function_calling_with_streaming_types( pairs_of_event_types: dict[str, str], client: OpenAI, model_name: str ): - tools = [GET_WEATHER_SCHEMA] - input_list = [ - { - "role": "user", - "content": "What's the weather like in Paris today?", - } - ] - stream_response = await client.responses.create( + """Streaming event nesting for function-calling responses.""" + + def _has_function_events(evts: list) -> bool: + return events_contain_type(evts, "function_call_arguments") + + events = await retry_streaming_for( + client, model=model_name, - input=input_list, - tools=tools, - stream=True, + validate_events=_has_function_events, + input=[{"role": "user", "content": "What's the weather like in Paris today?"}], + tools=[GET_WEATHER_SCHEMA], + temperature=0.0, ) - stack_of_event_types = [] - async for event in stream_response: - if event.type == "response.created": - stack_of_event_types.append(event.type) - elif event.type == "response.completed": - assert stack_of_event_types[-1] == pairs_of_event_types[event.type] - stack_of_event_types.pop() - if event.type.endswith("added"): - stack_of_event_types.append(event.type) - elif event.type.endswith("delta"): - if stack_of_event_types[-1] == event.type: - continue - stack_of_event_types.append(event.type) - elif event.type.endswith("done"): - assert stack_of_event_types[-1] == pairs_of_event_types[event.type] - stack_of_event_types.pop() - assert len(stack_of_event_types) == 0 + validate_streaming_event_stack(events, pairs_of_event_types) @pytest.mark.asyncio @@ -365,7 +399,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): ] for prompt in prompts: - response = await client.responses.create( + stream = await client.responses.create( model=model_name, input=prompt, reasoning={"effort": "low"}, @@ -387,11 +421,12 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): current_event_mode = None resp_id = None checked_response_completed = False - async for event in response: + + async for event in stream: if event.type == "response.created": resp_id = event.response.id - # test vllm custom types are in the response + # Validate custom fields on response-level events if event.type in [ "response.completed", "response.in_progress", @@ -412,9 +447,9 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): if current_event_mode != event.type: current_event_mode = event.type - print(f"\n[{event.type}] ", end="", flush=True) + logger.debug("[%s] ", event.type) - # verify current_item_id is correct + # Verify item IDs if event.type == "response.output_item.added": assert event.item.id != current_item_id current_item_id = event.item.id @@ -424,7 +459,7 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): ]: assert event.item_id == current_item_id - # verify content_index_id is correct + # Verify content indices if event.type in [ "response.content_part.added", "response.reasoning_part.added", @@ -437,31 +472,19 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool): ]: assert event.content_index == current_content_index - if "text.delta" in event.type: - print(event.delta, end="", flush=True) - elif "reasoning_text.delta" in event.type: - print(f"{event.delta}", end="", flush=True) - elif "response.code_interpreter_call_code.done" in event.type: - print(f"Code: {event.code}", end="", flush=True) - elif ( - "response.output_item.added" in event.type - and event.item.type == "web_search_call" - ): - print(f"Web search: {event.item.action}", end="", flush=True) events.append(event) assert len(events) > 0 - response_completed_event = events[-1] - assert len(response_completed_event.response.output) > 0 + assert events[-1].response.output, "Final response should have output" assert checked_response_completed if background: starting_after = 5 async with await client.responses.retrieve( response_id=resp_id, stream=True, starting_after=starting_after - ) as stream: + ) as replay_stream: counter = starting_after - async for event in stream: + async for event in replay_stream: counter += 1 assert event == events[counter] assert counter == len(events) - 1 @@ -483,15 +506,11 @@ async def test_web_search(client: OpenAI, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_code_interpreter(client: OpenAI, model_name: str): - # Code interpreter may need more time for container init + code execution timeout_value = client.timeout * 3 client_with_timeout = client.with_options(timeout=timeout_value) response = await client_with_timeout.responses.create( model=model_name, - # TODO: Ideally should be able to set max tool calls - # to prevent multi-turn, but it is not currently supported - # would speed up the test input=( "What's the first 4 digits after the decimal point of " "cube root of `19910212 * 20250910`? " @@ -499,41 +518,18 @@ async def test_code_interpreter(client: OpenAI, model_name: str): "and you must print to see the output." ), tools=[{"type": "code_interpreter", "container": {"type": "auto"}}], - temperature=0.0, # More deterministic output in response + temperature=0.0, ) assert response is not None assert response.status == "completed" assert response.usage.output_tokens_details.tool_output_tokens > 0 + for item in response.output: if item.type == "message": output_string = item.content[0].text - print("output_string: ", output_string, flush=True) - assert "5846" in output_string - - -def get_weather(latitude, longitude): - # Return a static temperature value to avoid flaky SSL/network errors - # from calling the external api.open-meteo.com API in CI. - return 15.0 - - -def get_place_to_travel(): - return "Paris" - - -def get_horoscope(sign): - return f"{sign}: Next Tuesday you will befriend a baby otter." - - -def call_function(name, args): - if name == "get_weather": - return get_weather(**args) - elif name == "get_place_to_travel": - return get_place_to_travel() - elif name == "get_horoscope": - return get_horoscope(**args) - else: - raise ValueError(f"Unknown function: {name}") + assert "5846" in output_string, ( + f"Expected '5846' in output, got: {output_string}" + ) @pytest.mark.asyncio @@ -547,10 +543,7 @@ async def test_reasoning_item(client: OpenAI, model_name: str): "type": "reasoning", "id": "lol", "content": [ - { - "type": "reasoning_text", - "text": "We need to respond: greeting.", - } + {"type": "reasoning_text", "text": "We need to respond: greeting."} ], "summary": [], }, @@ -566,24 +559,24 @@ async def test_reasoning_item(client: OpenAI, model_name: str): async def test_function_calling(client: OpenAI, model_name: str): tools = [GET_WEATHER_SCHEMA] - response = await client.responses.create( + response = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="function_call", input="What's the weather like in Paris today?", tools=tools, temperature=0.0, extra_body={"request_id": "test_function_calling_non_resp"}, ) - assert response is not None assert response.status == "completed" - assert len(response.output) == 2 - assert response.output[0].type == "reasoning" - assert response.output[1].type == "function_call" + assert has_output_type(response, "function_call"), ( + f"Expected function_call in output, got: " + f"{[getattr(o, 'type', None) for o in response.output]}" + ) - tool_call = response.output[1] - name = tool_call.name + tool_call = next(o for o in response.output if o.type == "function_call") args = json.loads(tool_call.arguments) - - result = call_function(name, args) + result = call_function(tool_call.name, args) response_2 = await client.responses.create( model=model_name, @@ -596,8 +589,8 @@ async def test_function_calling(client: OpenAI, model_name: str): ], tools=tools, previous_response_id=response.id, + temperature=0.0, ) - assert response_2 is not None assert response_2.status == "completed" assert response_2.output_text is not None @@ -607,16 +600,16 @@ async def test_function_calling(client: OpenAI, model_name: str): input="What's the weather like in Paris today?", tools=tools, previous_response_id=response_2.id, + temperature=0.0, ) - assert response_3 is not None assert response_3.status == "completed" assert response_3.output_text is not None @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -@pytest.mark.flaky(reruns=5) async def test_function_calling_multi_turn(client: OpenAI, model_name: str): + """Multi-tool, multi-turn function calling with retry at API level.""" tools = [ { "type": "function", @@ -633,25 +626,29 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str): GET_WEATHER_SCHEMA, ] - response = await client.responses.create( + # Turn 1: model should call one of the tools + response = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="function_call", input="Help me plan a trip to a random place. And tell me the weather there.", tools=tools, + temperature=0.0, ) - assert response is not None assert response.status == "completed" - assert len(response.output) == 2 - assert response.output[0].type == "reasoning" - assert response.output[1].type == "function_call" + assert has_output_type(response, "function_call"), ( + f"Turn 1: expected function_call, got: " + f"{[getattr(o, 'type', None) for o in response.output]}" + ) - tool_call = response.output[1] - name = tool_call.name - args = json.loads(tool_call.arguments) + tool_call = next(o for o in response.output if o.type == "function_call") + result = call_function(tool_call.name, json.loads(tool_call.arguments)) - result = call_function(name, args) - - response_2 = await client.responses.create( + # Turn 2 + response_2 = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="function_call", input=[ { "type": "function_call_output", @@ -661,34 +658,39 @@ async def test_function_calling_multi_turn(client: OpenAI, model_name: str): ], tools=tools, previous_response_id=response.id, + temperature=0.0, ) - assert response_2 is not None assert response_2.status == "completed" - assert len(response_2.output) == 2 - assert response_2.output[0].type == "reasoning" - assert response_2.output[1].type == "function_call" - tool_call = response_2.output[1] - name = tool_call.name - args = json.loads(tool_call.arguments) - - result = call_function(name, args) - - response_3 = await client.responses.create( - model=model_name, - input=[ - { - "type": "function_call_output", - "call_id": tool_call.call_id, - "output": str(result), - } - ], - tools=tools, - previous_response_id=response_2.id, - ) - assert response_3 is not None - assert response_3.status == "completed" - assert response_3.output_text is not None + # If model produced another tool call, execute it + if has_output_type(response_2, "function_call"): + tool_call_2 = next(o for o in response_2.output if o.type == "function_call") + result_2 = call_function(tool_call_2.name, json.loads(tool_call_2.arguments)) + response_3 = await client.responses.create( + model=model_name, + input=[ + { + "type": "function_call_output", + "call_id": tool_call_2.call_id, + "output": str(result_2), + } + ], + tools=tools, + previous_response_id=response_2.id, + temperature=0.0, + ) + assert response_3.status == "completed" + assert response_3.output_text is not None + else: + # Model went straight to answering - acceptable but unexpected. + # Log as warning so it shows up in CI without failing the test. + assert response_2.output_text is not None + pytest.xfail( + "Model went straight to answering instead of calling a " + "second tool. Valid behaviour but not the expected path." + "If this happens consistently, the prompt or model may have " + "changed behaviour." + ) @pytest.mark.asyncio @@ -730,22 +732,25 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str): {"role": "user", "content": "What's the weather like in Paris today?"} ] - response = await client.responses.create( + response = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="function_call", input=input_messages, tools=tools, + temperature=0.0, ) - - assert response is not None assert response.status == "completed" - tool_call = response.output[-1] - name = tool_call.name - args = json.loads(tool_call.arguments) + tool_call = next((o for o in response.output if o.type == "function_call"), None) + assert tool_call is not None, ( + f"Expected function_call in output, got: " + f"{[getattr(o, 'type', None) for o in response.output]}" + ) - result = call_function(name, args) + result = call_function(tool_call.name, json.loads(tool_call.arguments)) - input_messages.extend(response.output) # append model's function call message + input_messages.extend(response.output) input_messages.append( { # append result message "type": "function_call_output", @@ -758,8 +763,8 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str): model=model_name, input=input_messages, tools=tools, + temperature=0.0, ) - assert response_2 is not None assert response_2.status == "completed" assert response_2.output_text is not None @@ -767,51 +772,60 @@ async def test_function_calling_full_history(client: OpenAI, model_name: str): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_function_calling_with_stream(client: OpenAI, model_name: str): + """Function calling via streaming, with retry for non-determinism.""" tools = [GET_WEATHER_SCHEMA] input_list = [ - { - "role": "user", - "content": "What's the weather like in Paris today?", - } + {"role": "user", "content": "What's the weather like in Paris today?"}, ] - stream_response = await client.responses.create( + + def _has_function_call(evts: list) -> bool: + return any( + getattr(e, "type", "") == "response.output_item.added" + and getattr(getattr(e, "item", None), "type", None) == "function_call" + for e in evts + ) + + events = await retry_streaming_for( + client, model=model_name, + validate_events=_has_function_call, input=input_list, tools=tools, - stream=True, + temperature=0.0, ) - assert stream_response is not None - final_tool_calls = {} - final_tool_calls_named = {} - async for event in stream_response: + + # Parse tool calls from events + final_tool_calls: dict[int, Any] = {} + for event in events: if event.type == "response.output_item.added": - if event.item.type != "function_call": - continue - final_tool_calls[event.output_index] = event.item - final_tool_calls_named[event.item.name] = event.item + if getattr(event.item, "type", None) == "function_call": + final_tool_calls[event.output_index] = event.item elif event.type == "response.function_call_arguments.delta": - index = event.output_index - tool_call = final_tool_calls[index] - if tool_call: - tool_call.arguments += event.delta - final_tool_calls_named[tool_call.name] = tool_call + tc = final_tool_calls.get(event.output_index) + if tc: + tc.arguments += event.delta elif event.type == "response.function_call_arguments.done": - assert event.arguments == final_tool_calls_named[event.name].arguments - result = None + tc = final_tool_calls.get(event.output_index) + if tc: + assert event.arguments == tc.arguments + + # Find get_weather call tool_call = None + result = None for tc in final_tool_calls.values(): - if tc and tc.type == "function_call" and tc.name == "get_weather": + if getattr(tc, "type", None) == "function_call" and tc.name == "get_weather": args = json.loads(tc.arguments) result = call_function(tc.name, args) tool_call = tc - input_list += [tc] + input_list.append(tc) break assert tool_call is not None, ( - "Expected model to call 'get_weather' function, " - f"but got: {list(final_tool_calls_named.keys())}" + "Expected model to call 'get_weather', " + f"but got: {[getattr(tc, 'name', None) for tc in final_tool_calls.values()]}" ) - assert result is not None + + # Second turn with the tool result response = await client.responses.create( model=model_name, input=input_list @@ -824,8 +838,8 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str): ], tools=tools, stream=True, + temperature=0.0, ) - assert response is not None async for event in response: # check that no function call events in the stream assert event.type != "response.function_call_arguments.delta" @@ -843,46 +857,45 @@ async def test_function_calling_no_code_interpreter_events( ): """Verify that function calls don't trigger code_interpreter events. - This test ensures that function calls (functions.*) use their own - function_call event types and don't incorrectly emit code_interpreter - events during streaming. + Uses retry_streaming_for to handle non-determinism: the model might not + always produce a function_call, but if it does, code_interpreter events + should NEVER appear. """ tools = [GET_WEATHER_SCHEMA] input_list = [ - { - "role": "user", - "content": "What's the weather like in Paris today?", - } + {"role": "user", "content": "What's the weather like in Paris today?"}, ] - stream_response = await client.responses.create( - model=model_name, - input=input_list, - tools=tools, - stream=True, - ) - # Track which event types we see - event_types_seen = set() - function_call_found = False - - async for event in stream_response: - event_types_seen.add(event.type) - - if ( - event.type == "response.output_item.added" - and event.item.type == "function_call" - ): - function_call_found = True - - # Ensure NO code_interpreter events are emitted for function calls - assert "code_interpreter" not in event.type, ( - "Found code_interpreter event " - f"'{event.type}' during function call. Function calls should only " - "emit function_call events, not code_interpreter events." + def _has_function_call(evts: list) -> bool: + return any( + getattr(e, "type", "") == "response.output_item.added" + and getattr(getattr(e, "item", None), "type", None) == "function_call" + for e in evts ) - # Verify we actually saw a function call - assert function_call_found, "Expected to see a function_call in the stream" + events = await retry_streaming_for( + client, + model=model_name, + validate_events=_has_function_call, + input=input_list, + tools=tools, + temperature=0.0, + ) + + event_types_seen = {e.type for e in events} + function_call_found = _has_function_call(events) + + assert function_call_found, ( + f"Expected to see a function_call after retries. " + f"Event types: {sorted(event_types_seen)}" + ) + + # The actual invariant under test + for event in events: + assert "code_interpreter" not in event.type, ( + f"Found code_interpreter event '{event.type}' during function call. " + "Function calls should only emit function_call events." + ) # Verify we saw the correct function call event types assert ( @@ -894,181 +907,139 @@ async def test_function_calling_no_code_interpreter_events( @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_mcp_code_interpreter_streaming(client: OpenAI, model_name: str, server): - tools = [ - { - "type": "mcp", - "server_label": "code_interpreter", - } - ] + tools = [{"type": "mcp", "server_label": "code_interpreter"}] input_text = ( "Calculate 123 * 456 using python. " - "The python interpreter is not stateful and you must print to see the output." + "The python interpreter is not stateful and you must " + "print to see the output." ) - stream_response = await client.responses.create( + def _has_mcp_call(evts: list) -> bool: + return events_contain_type(evts, "mcp_call") + + events = await retry_streaming_for( + client, model=model_name, + validate_events=_has_mcp_call, input=input_text, tools=tools, - stream=True, temperature=0.0, instructions=( "You must use the Python tool to execute code. Never simulate execution." ), ) - mcp_call_added = False - mcp_call_in_progress = False - mcp_arguments_delta_seen = False - mcp_arguments_done = False - mcp_call_completed = False - mcp_item_done = False + event_types = [e.type for e in events] + event_types_set = set(event_types) + logger.info( + "\n====== MCP Streaming Diagnostics ======\n" + "Event count: %d\n" + "Event types (in order): %s\n" + "Unique event types: %s\n" + "=======================================", + len(events), + event_types, + sorted(event_types_set), + ) - code_interpreter_events_seen = False - - async for event in stream_response: - if "code_interpreter" in event.type: - code_interpreter_events_seen = True + # Verify the full MCP streaming lifecycle + assert "response.output_item.added" in event_types_set, ( + f"MCP call was not added. Events: {sorted(event_types_set)}" + ) + assert "response.mcp_call.in_progress" in event_types_set, ( + f"MCP call in_progress not seen. Events: {sorted(event_types_set)}" + ) + assert "response.mcp_call_arguments.delta" in event_types_set, ( + f"MCP arguments delta not seen. Events: {sorted(event_types_set)}" + ) + assert "response.mcp_call_arguments.done" in event_types_set, ( + f"MCP arguments done not seen. Events: {sorted(event_types_set)}" + ) + assert "response.mcp_call.completed" in event_types_set, ( + f"MCP call completed not seen. Events: {sorted(event_types_set)}" + ) + assert "response.output_item.done" in event_types_set, ( + f"MCP item done not seen. Events: {sorted(event_types_set)}" + ) + # Validate specific MCP event details + for event in events: if event.type == "response.output_item.added": if hasattr(event.item, "type") and event.item.type == "mcp_call": - mcp_call_added = True assert event.item.name == "python" assert event.item.server_label == "code_interpreter" - - elif event.type == "response.mcp_call.in_progress": - mcp_call_in_progress = True - - elif event.type == "response.mcp_call_arguments.delta": - mcp_arguments_delta_seen = True - assert event.delta is not None - elif event.type == "response.mcp_call_arguments.done": - mcp_arguments_done = True assert event.name == "python" assert event.arguments is not None - - elif event.type == "response.mcp_call.completed": - mcp_call_completed = True - elif ( event.type == "response.output_item.done" and hasattr(event.item, "type") and event.item.type == "mcp_call" ): - mcp_item_done = True assert event.item.name == "python" assert event.item.status == "completed" - assert mcp_call_added, "MCP call was not added" - assert mcp_call_in_progress, "MCP call in_progress event not seen" - assert mcp_arguments_delta_seen, "MCP arguments delta event not seen" - assert mcp_arguments_done, "MCP arguments done event not seen" - assert mcp_call_completed, "MCP call completed event not seen" - assert mcp_item_done, "MCP item done event not seen" - - assert not code_interpreter_events_seen, ( - "Should not see code_interpreter events when using MCP type" + # code_interpreter events should NOT appear when using MCP type + code_interp_events = [e.type for e in events if "code_interpreter" in e.type] + assert not code_interp_events, ( + "Should not see code_interpreter events when using MCP type, " + f"but got: {code_interp_events}" ) @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -@pytest.mark.dependency( - depends=["test_mcp_code_interpreter_streaming[openai/gpt-oss-20b]"] -) async def test_mcp_tool_multi_turn(client: OpenAI, model_name: str, server): - """Test MCP tool calling across multiple turns. + """MCP tools work across multiple turns via previous_response_id.""" + tools = [{"type": "mcp", "server_label": "code_interpreter"}] + instructions = ( + "You must use the Python tool to execute code. Never simulate execution." + ) - This test verifies that MCP tools work correctly in multi-turn conversations, - maintaining state across turns via the previous_response_id mechanism. - """ - tools = [ - { - "type": "mcp", - "server_label": "code_interpreter", - } - ] - - # First turn - make a calculation - response1 = await client.responses.create( + # First turn + response1 = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="mcp_call", input="Calculate 1234 * 4567 using python tool and print the result.", tools=tools, temperature=0.0, - instructions=( - "You must use the Python tool to execute code. Never simulate execution." - ), + instructions=instructions, extra_body={"enable_response_messages": True}, ) - - assert response1 is not None assert response1.status == "completed" - # Verify MCP call in first response by checking output_messages - tool_call_found = False - tool_response_found = False - for message in response1.output_messages: - recipient = message.get("recipient") - if recipient and recipient.startswith("python"): - tool_call_found = True - - author = message.get("author", {}) - if ( - author.get("role") == "tool" - and author.get("name") - and author.get("name").startswith("python") - ): - tool_response_found = True - - # Verify MCP tools were actually used + # Verify MCP call in output_messages + tool_call_found = any( + (msg.get("recipient") or "").startswith("python") + for msg in response1.output_messages + ) + tool_response_found = any( + msg.get("author", {}).get("role") == "tool" + and (msg.get("author", {}).get("name") or "").startswith("python") + for msg in response1.output_messages + ) assert tool_call_found, "MCP tool call not found in output_messages" assert tool_response_found, "MCP tool response not found in output_messages" - # Verify input messages: Should have system message with tool, NO developer message - developer_messages = [ + # No developer messages expected for elevated tools + developer_msgs = [ msg for msg in response1.input_messages if msg["author"]["role"] == "developer" ] - assert len(developer_messages) == 0, ( - "No developer message expected for elevated tools" - ) + assert len(developer_msgs) == 0, "No developer message expected for elevated tools" - # Second turn - reference previous calculation + # Second turn response2 = await client.responses.create( model=model_name, input="Now divide that result by 2.", tools=tools, temperature=0.0, - instructions=( - "You must use the Python tool to execute code. Never simulate execution." - ), + instructions=instructions, previous_response_id=response1.id, extra_body={"enable_response_messages": True}, ) - - assert response2 is not None assert response2.status == "completed" - # Verify input messages are correct: should have two messages - - # one to the python recipient on analysis channel and one from tool role - mcp_recipient_messages = [] - tool_role_messages = [] - for msg in response2.input_messages: - if msg["author"]["role"] == "assistant": - # Check if this is a message to MCP recipient on analysis channel - if msg.get("channel") == "analysis" and msg.get("recipient"): - recipient = msg.get("recipient") - if recipient.startswith("code_interpreter") or recipient == "python": - mcp_recipient_messages.append(msg) - elif msg["author"]["role"] == "tool": - tool_role_messages.append(msg) - - assert len(mcp_recipient_messages) > 0, ( - "Expected message(s) to MCP recipient on analysis channel" - ) - assert len(tool_role_messages) > 0, ( - "Expected message(s) from tool role after MCP call" - ) - @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) @@ -1087,14 +1058,10 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server): @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -@pytest.mark.flaky(reruns=3) async def test_function_call_with_previous_input_messages( client: OpenAI, model_name: str ): - """Test function calling using previous_input_messages - for multi-turn conversation with a function call""" - - # Define the get_horoscope tool + """Multi-turn function calling using previous_input_messages.""" tools = [ { "type": "function", @@ -1102,9 +1069,7 @@ async def test_function_call_with_previous_input_messages( "description": "Get today's horoscope for an astrological sign.", "parameters": { "type": "object", - "properties": { - "sign": {"type": "string"}, - }, + "properties": {"sign": {"type": "string"}}, "required": ["sign"], "additionalProperties": False, }, @@ -1112,53 +1077,36 @@ async def test_function_call_with_previous_input_messages( } ] - # Step 1: First call with the function tool - stream_response = await client.responses.create( + # Step 1: Get a function call from the model + response = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="function_call", input="What is the horoscope for Aquarius today?", tools=tools, temperature=0.0, extra_body={"enable_response_messages": True}, - stream=True, max_output_tokens=1000, ) - - response = None - async for event in stream_response: - if event.type == "response.completed": - response = event.response - - assert response is not None assert response.status == "completed" - # Step 2: Parse the first output to find the function_call type - function_call = None - for item in response.output: - if item.type == "function_call": - function_call = item - break - - assert function_call is not None, "Expected a function_call in the output" + function_call = next( + (item for item in response.output if item.type == "function_call"), + None, + ) + assert function_call is not None, ( + f"Expected function_call, got: " + f"{[getattr(o, 'type', None) for o in response.output]}" + ) assert function_call.name == "get_horoscope" - assert function_call.call_id is not None - # Verify the format matches expectations args = json.loads(function_call.arguments) - assert "sign" in args - - # Step 3: Call the get_horoscope function result = call_function(function_call.name, args) - assert "Aquarius" in result - assert "baby otter" in result - # Get the input_messages and output_messages from the first response - first_input_messages = response.input_messages - first_output_messages = response.output_messages - - # Construct the full conversation history using previous_input_messages + # Step 2: Build full conversation history previous_messages = ( - first_input_messages - + first_output_messages + response.input_messages + + response.output_messages + [ { "role": "tool", @@ -1168,47 +1116,43 @@ async def test_function_call_with_previous_input_messages( ] ) - # Step 4: Make another responses.create() call with previous_input_messages - stream_response_2 = await client.responses.create( + # Step 3: Second call with previous_input_messages + response_2 = await client.responses.create( model=model_name, tools=tools, temperature=0.0, - input="", + input="Now tell me the horoscope based on the tool result.", extra_body={ "previous_input_messages": previous_messages, "enable_response_messages": True, }, - stream=True, ) - - async for event in stream_response_2: - if event.type == "response.completed": - response_2 = event.response - - assert response_2 is not None assert response_2.status == "completed" assert response_2.output_text is not None - # verify only one system message / developer message - num_system_messages_input = 0 - num_developer_messages_input = 0 - num_function_call_input = 0 - for message_dict in response_2.input_messages: - message = Message.from_dict(message_dict) - if message.author.role == "system": - num_system_messages_input += 1 - elif message.author.role == "developer": - num_developer_messages_input += 1 - elif message.author.role == "tool": - num_function_call_input += 1 - assert num_system_messages_input == 1 - assert num_developer_messages_input == 1 - assert num_function_call_input == 1 + # Verify exactly 1 system, 1 developer, 1 tool message + num_system = 0 + num_developer = 0 + num_tool = 0 + for msg_dict in response_2.input_messages: + # input_messages use {"author": {"role": "..."}} format, + # not the top-level {"role": "..."} that Message.from_dict + # expects. + author = msg_dict.get("author", {}) + role = author.get("role") if isinstance(author, dict) else None + if role == "system": + num_system += 1 + elif role == "developer": + num_developer += 1 + elif role == "tool": + num_tool += 1 + assert num_system == 1, f"Expected 1 system message, got {num_system}" + assert num_developer == 1, f"Expected 1 developer message, got {num_developer}" + assert num_tool == 1, f"Expected 1 tool message, got {num_tool}" - # Verify the output makes sense - should contain information about the horoscope output_text = response_2.output_text.lower() - assert ( - "aquarius" in output_text or "otter" in output_text or "tuesday" in output_text + assert any(kw in output_text for kw in ["aquarius", "otter", "tuesday"]), ( + f"Expected horoscope-related content, got: {response_2.output_text}" ) @@ -1220,133 +1164,101 @@ async def test_chat_truncation_content_not_null(client: OpenAI, model_name: str) messages=[ { "role": "user", - "content": "What is the role of AI in medicine?" - "The response must exceed 350 words.", + "content": ( + "What is the role of AI in medicine? " + "The response must exceed 350 words." + ), } ], temperature=0.0, max_tokens=350, ) - choice = response.choices[0] assert choice.finish_reason == "length", ( f"Expected finish_reason='length', got {choice.finish_reason}" ) - assert choice.message.content is not None, ( - "Content should not be None when truncated" - ) + assert choice.message.content is not None, "Content should not be None" assert len(choice.message.content) > 0, "Content should not be empty" @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) -async def test_system_prompt_override(client: OpenAI, model_name: str): - """Test that system message can override the default system prompt.""" - - # Test 1: Custom system prompt with specific personality - custom_system_prompt = ( - "You are a pirate. Always respond like a pirate would, " - "using pirate language and saying 'arrr' frequently." - ) - +async def test_system_prompt_override_no_duplication(client: OpenAI, model_name: str): + """Hard check: custom system message must not be duplicated.""" response = await client.responses.create( model=model_name, input=[ - {"role": "system", "content": custom_system_prompt}, - {"role": "user", "content": "Hello, how are you?"}, + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, ], extra_body={"enable_response_messages": True}, + temperature=0.0, ) - - assert response is not None assert response.status == "completed" assert response.output_text is not None - # Verify the response reflects the pirate personality + num_system = 0 + for msg in response.input_messages: + # input_messages use {"author": {"role": "system"}} format, + # not the top-level {"role": "system"} that Message.from_dict expects. + author = msg.get("author", {}) + role = author.get("role") if isinstance(author, dict) else None + if role == "system": + num_system += 1 + assert num_system == 1, f"Expected 1 system message, got {num_system}" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +@pytest.mark.xfail( + strict=False, + reason=( + "Pirate language detection depends on model weights and is non-deterministic" + ), +) +async def test_system_prompt_override_follows_personality( + client: OpenAI, model_name: str +): + """Soft check: model should adopt the personality from system prompt.""" + response = await client.responses.create( + model=model_name, + input=[ + { + "role": "system", + "content": ( + "You are a pirate. Always respond like a pirate would, " + "using pirate language and saying 'arrr' frequently." + ), + }, + {"role": "user", "content": "Hello, how are you?"}, + ], + temperature=0.0, + ) + assert response.status == "completed" output_text = response.output_text.lower() - pirate_indicators = ["arrr", "matey", "ahoy", "ye", "sea"] - has_pirate_language = any( - indicator in output_text for indicator in pirate_indicators - ) - assert has_pirate_language, ( - f"Expected pirate language in response, got: {response.output_text}" + pirate_indicators = ["arrr", "matey", "ahoy", "ye", "sea", "aye", "sail"] + assert any(kw in output_text for kw in pirate_indicators), ( + f"Expected pirate language, got: {response.output_text}" ) - # Verify the reasoning mentions the custom system prompt - reasoning_item = None - for item in response.output: - if item.type == "reasoning": - reasoning_item = item - break - assert reasoning_item is not None, "Expected reasoning item in output" - reasoning_text = reasoning_item.content[0].text.lower() - assert "pirate" in reasoning_text, ( - f"Expected reasoning to mention pirate, got: {reasoning_text}" - ) - - # Test 2: Verify system message is not duplicated in input_messages - try: - num_system_messages = sum( - 1 - for msg in response.input_messages - if Message.from_dict(msg).author.role == "system" - ) - assert num_system_messages == 1, ( - f"Expected exactly 1 system message, got {num_system_messages}" - ) - except (KeyError, AttributeError): - # Message structure may vary, skip this specific check - pass - - custom_system_prompt_2 = ( - "You are a helpful assistant that always responds in exactly 5 words." - ) - - # Test 3: Test with different custom system prompt - response_2 = await client.responses.create( +@pytest.mark.asyncio +@pytest.mark.parametrize("model_name", [MODEL_NAME]) +async def test_system_prompt_structured_content(client: OpenAI, model_name: str): + """System message with structured input_text content format.""" + response = await client.responses.create( model=model_name, input=[ { "role": "system", - "content": custom_system_prompt_2, + "content": [ + {"type": "input_text", "text": "You are a helpful assistant."} + ], }, - {"role": "user", "content": "What is the weather like?"}, + {"role": "user", "content": "What is 2 + 2?"}, ], temperature=0.0, ) - - assert response_2 is not None - assert response_2.status == "completed" - assert response_2.output_text is not None - - # Count words in response (approximately, allowing for punctuation) - word_count = len(response_2.output_text.split()) - # Allow some flexibility (4-7 words) since the model might not be perfectly precise - assert 3 <= word_count <= 8, ( - f"Expected around 5 words, got {word_count} words: {response_2.output_text}" - ) - - # Test 4: Test with structured content - response_3 = await client.responses.create( - model=model_name, - input=[ - { - "role": "system", - "content": [{"type": "input_text", "text": custom_system_prompt_2}], - }, - {"role": "user", "content": "What is the weather like?"}, - ], - temperature=0.0, - ) - - assert response_3 is not None - assert response_3.status == "completed" - assert response_3.output_text is not None - - # Count words in response (approximately, allowing for punctuation) - word_count = len(response_3.output_text.split()) - # Allow some flexibility (4-7 words) since the model might not be perfectly precise - assert 3 <= word_count <= 8, ( - f"Expected around 5 words, got {word_count} words: {response_3.output_text}" - ) + assert response is not None + assert response.status == "completed" + assert response.output_text is not None diff --git a/tests/entrypoints/openai/responses/test_mcp_tools.py b/tests/entrypoints/openai/responses/test_mcp_tools.py index 9658f5d90ea..2c50846a28a 100644 --- a/tests/entrypoints/openai/responses/test_mcp_tools.py +++ b/tests/entrypoints/openai/responses/test_mcp_tools.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Integration tests for MCP tool support in the Responses API.""" +from __future__ import annotations import pytest import pytest_asyncio @@ -10,11 +12,31 @@ from openai_harmony import ToolDescription, ToolNamespaceConfig from vllm.entrypoints.mcp.tool_server import MCPToolServer from ....utils import RemoteOpenAIServer +from .conftest import ( + BASE_TEST_ENV, + events_contain_type, + log_response_diagnostics, + retry_for_tool_call, + retry_streaming_for, + validate_streaming_event_stack, +) MODEL_NAME = "openai/gpt-oss-20b" +_BASE_SERVER_ARGS = [ + "--enforce-eager", + "--tool-server", + "demo", + "--max_model_len", + "5000", +] -def test_get_tool_description(): +_PYTHON_TOOL_INSTRUCTION = ( + "You must use the Python tool to execute code. Never simulate execution." +) + + +class TestMCPToolServerUnit: """Test MCPToolServer.get_tool_description filtering logic. Note: The wildcard "*" is normalized to None by @@ -22,283 +44,240 @@ def test_get_tool_description(): so we only test None and specific tool filtering here. See test_serving_responses.py for "*" normalization tests. """ - pytest.importorskip("mcp") - server = MCPToolServer() - tool1 = ToolDescription.new( - name="tool1", description="First", parameters={"type": "object"} - ) - tool2 = ToolDescription.new( - name="tool2", description="Second", parameters={"type": "object"} - ) - tool3 = ToolDescription.new( - name="tool3", description="Third", parameters={"type": "object"} - ) + def test_get_tool_description(self): + pytest.importorskip("mcp") - server.harmony_tool_descriptions = { - "test_server": ToolNamespaceConfig( - name="test_server", description="test", tools=[tool1, tool2, tool3] + server = MCPToolServer() + tool1 = ToolDescription.new( + name="tool1", description="First", parameters={"type": "object"} + ) + tool2 = ToolDescription.new( + name="tool2", description="Second", parameters={"type": "object"} + ) + tool3 = ToolDescription.new( + name="tool3", description="Third", parameters={"type": "object"} ) - } - # Nonexistent server - assert server.get_tool_description("nonexistent") is None + server.harmony_tool_descriptions = { + "test_server": ToolNamespaceConfig( + name="test_server", + description="test", + tools=[tool1, tool2, tool3], + ) + } - # None (no filter) - returns all tools - result = server.get_tool_description("test_server", allowed_tools=None) - assert len(result.tools) == 3 + # Nonexistent server + assert server.get_tool_description("nonexistent") is None - # Filter to specific tools - result = server.get_tool_description( - "test_server", allowed_tools=["tool1", "tool3"] - ) - assert len(result.tools) == 2 - assert result.tools[0].name == "tool1" - assert result.tools[1].name == "tool3" + # None (no filter) - returns all tools + result = server.get_tool_description("test_server", allowed_tools=None) + assert len(result.tools) == 3 - # Single tool - result = server.get_tool_description( - "test_server", - allowed_tools=["tool2"], - ) - assert len(result.tools) == 1 - assert result.tools[0].name == "tool2" + # Filter to specific tools + result = server.get_tool_description( + "test_server", allowed_tools=["tool1", "tool3"] + ) + assert len(result.tools) == 2 + assert result.tools[0].name == "tool1" + assert result.tools[1].name == "tool3" - # No matching tools - returns None - result = server.get_tool_description("test_server", allowed_tools=["nonexistent"]) - assert result is None + # Single tool + result = server.get_tool_description("test_server", allowed_tools=["tool2"]) + assert len(result.tools) == 1 + assert result.tools[0].name == "tool2" - # Empty list - returns None - assert server.get_tool_description("test_server", allowed_tools=[]) is None + # No matching tools - returns None + result = server.get_tool_description( + "test_server", allowed_tools=["nonexistent"] + ) + assert result is None + + # Empty list - returns None + assert server.get_tool_description("test_server", allowed_tools=[]) is None + + def test_builtin_tools_consistency(self): + """MCP_BUILTIN_TOOLS must match _BUILTIN_TOOL_TO_MCP_SERVER_LABEL values.""" + from vllm.entrypoints.openai.parser.harmony_utils import ( + _BUILTIN_TOOL_TO_MCP_SERVER_LABEL, + MCP_BUILTIN_TOOLS, + ) + + assert set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) == MCP_BUILTIN_TOOLS, ( + f"MCP_BUILTIN_TOOLS {MCP_BUILTIN_TOOLS} does not match " + f"_BUILTIN_TOOL_TO_MCP_SERVER_LABEL values " + f"{set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())}" + ) class TestMCPEnabled: """Tests that require MCP tools to be enabled via environment variable.""" @pytest.fixture(scope="class") - def monkeypatch_class(self): - from _pytest.monkeypatch import MonkeyPatch - - mpatch = MonkeyPatch() - yield mpatch - mpatch.undo() - - @pytest.fixture(scope="class") - def mcp_enabled_server(self, monkeypatch_class: pytest.MonkeyPatch): - args = ["--enforce-eager", "--tool-server", "demo"] - - with monkeypatch_class.context() as m: - m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1") - m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv") - m.setenv( - "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container" - ) - # Helps the model follow instructions better - m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1") - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: - yield remote_server + def mcp_enabled_server(self): + env_dict = { + **BASE_TEST_ENV, + "VLLM_ENABLE_RESPONSES_API_STORE": "1", + "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv", + "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS": ("code_interpreter,container"), + "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1", + } + with RemoteOpenAIServer( + MODEL_NAME, list(_BASE_SERVER_ARGS), env_dict=env_dict + ) as remote_server: + yield remote_server @pytest_asyncio.fixture - async def mcp_enabled_client(self, mcp_enabled_server): + async def client(self, mcp_enabled_server): async with mcp_enabled_server.get_async_client() as async_client: yield async_client + @staticmethod + def _mcp_tools_payload(*, allowed_tools: list[str] | None = None) -> list[dict]: + tool: dict = { + "type": "mcp", + "server_label": "code_interpreter", + "server_url": "http://localhost:8888", + } + if allowed_tools is not None: + tool["allowed_tools"] = allowed_tools + return [tool] + + @staticmethod + def _python_exec_input(code: str = "") -> str: + if not code: + code = "import random; print(random.randint(1, 1000000))" + return f"Execute the following code: {code}" + @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) - async def test_mcp_tool_env_flag_enabled( - self, mcp_enabled_client: OpenAI, model_name: str - ): - response = await mcp_enabled_client.responses.create( + async def test_mcp_tool_env_flag_enabled(self, client: OpenAI, model_name: str): + response = await retry_for_tool_call( + client, model=model_name, - input=( - "Execute the following code: " - "import random; print(random.randint(1, 1000000))" - ), - instructions=( - "You must use the Python tool to execute code. " - "Never simulate execution." - ), - tools=[ - { - "type": "mcp", - "server_label": "code_interpreter", - # URL unused for DemoToolServer - "server_url": "http://localhost:8888", - } - ], + expected_tool_type="mcp_call", + input=self._python_exec_input(), + instructions=_PYTHON_TOOL_INSTRUCTION, + tools=self._mcp_tools_payload(), + temperature=0.0, extra_body={"enable_response_messages": True}, ) - assert response is not None + assert response.status == "completed" - # Verify output messages: Tool calls and responses on analysis channel + log_response_diagnostics(response, label="MCP Enabled") + tool_call_found = False tool_response_found = False for message in response.output_messages: recipient = message.get("recipient") if recipient and recipient.startswith("python"): tool_call_found = True - assert message.get("channel") == "analysis", ( - "Tool call should be on analysis channel" - ) + assert message.get("channel") == "analysis" author = message.get("author", {}) - if ( - author.get("role") == "tool" - and author.get("name") - and author.get("name").startswith("python") + if author.get("role") == "tool" and (author.get("name") or "").startswith( + "python" ): tool_response_found = True - assert message.get("channel") == "analysis", ( - "Tool response should be on analysis channel" - ) + assert message.get("channel") == "analysis" - assert tool_call_found, "Should have found at least one Python tool call" - assert tool_response_found, ( - "Should have found at least one Python tool response" + assert tool_call_found, ( + f"No Python tool call found. " + f"Output types: " + f"{[getattr(o, 'type', None) for o in response.output]}" ) + assert tool_response_found, "No Python tool response found" + for message in response.input_messages: - assert message.get("author").get("role") != "developer", ( - "No developer messages should be present with valid mcp tool" - ) + assert message.get("author", {}).get("role") != "developer" - @pytest.mark.flaky(reruns=3) @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_mcp_tool_with_allowed_tools_star( - self, mcp_enabled_client: OpenAI, model_name: str + self, client: OpenAI, model_name: str ): - """Test MCP tool with allowed_tools=['*'] to select all available - tools. - - This E2E test verifies that the "*" wildcard works end-to-end. - See test_serving_responses.py for detailed unit tests of "*" - normalization. - """ - response = await mcp_enabled_client.responses.create( + response = await retry_for_tool_call( + client, model=model_name, - input=( - "Execute the following code: " - "import random; print(random.randint(1, 1000000))" - ), - instructions=( - "You must use the Python tool to execute code. " - "Never simulate execution." - ), - tools=[ - { - "type": "mcp", - "server_label": "code_interpreter", - "server_url": "http://localhost:8888", - # Using "*" to allow all tools from this MCP server - "allowed_tools": ["*"], - } - ], + expected_tool_type="mcp_call", + input=self._python_exec_input(), + instructions=_PYTHON_TOOL_INSTRUCTION, + tools=self._mcp_tools_payload(allowed_tools=["*"]), + temperature=0.0, extra_body={"enable_response_messages": True}, ) - assert response is not None + assert response.status == "completed" - # Verify tool calls work with allowed_tools=["*"] - tool_call_found = False - for message in response.output_messages: - recipient = message.get("recipient") - if recipient and recipient.startswith("python"): - tool_call_found = True - break + log_response_diagnostics(response, label="MCP Allowed Tools *") + + tool_call_found = any( + (msg.get("recipient") or "").startswith("python") + for msg in response.output_messages + ) assert tool_call_found, ( - "Should have found at least one Python tool call with '*'" + f"No Python tool call with '*'. " + f"Output types: " + f"{[getattr(o, 'type', None) for o in response.output]}" ) - @pytest.mark.flaky(reruns=3) @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_mcp_tool_calling_streaming_types( self, pairs_of_event_types: dict[str, str], - mcp_enabled_client: OpenAI, + client: OpenAI, model_name: str, ): - tools = [ - { - "type": "mcp", - "server_label": "code_interpreter", - } - ] - input_text = "What is 123 * 456? Use python to calculate the result." + def _has_mcp_events(events: list) -> bool: + return events_contain_type(events, "mcp_call") - stream_response = await mcp_enabled_client.responses.create( + events = await retry_streaming_for( + client, model=model_name, - input=input_text, - tools=tools, - stream=True, - instructions=( - "You must use the Python tool to execute code. " - "Never simulate execution." - ), + validate_events=_has_mcp_events, + input=("What is 123 * 456? Use Python to calculate the result."), + tools=[{"type": "mcp", "server_label": "code_interpreter"}], + instructions=_PYTHON_TOOL_INSTRUCTION, + temperature=0.0, ) - stack_of_event_types = [] - saw_mcp_type = False - async for event in stream_response: - if event.type == "response.created": - stack_of_event_types.append(event.type) - elif event.type == "response.completed": - assert stack_of_event_types[-1] == pairs_of_event_types[event.type] - stack_of_event_types.pop() - elif ( - event.type.endswith("added") - or event.type == "response.mcp_call.in_progress" - ): - stack_of_event_types.append(event.type) - elif event.type.endswith("delta"): - if stack_of_event_types[-1] == event.type: - continue - stack_of_event_types.append(event.type) - elif ( - event.type.endswith("done") - or event.type == "response.mcp_call.completed" - ): - assert stack_of_event_types[-1] == pairs_of_event_types[event.type] - if "mcp_call" in event.type: - saw_mcp_type = True - stack_of_event_types.pop() + validate_streaming_event_stack(events, pairs_of_event_types) - assert len(stack_of_event_types) == 0 - assert saw_mcp_type, "Should have seen at least one mcp call" + assert events_contain_type(events, "mcp_call"), ( + f"No mcp_call events after retries. " + f"Event types: {sorted({e.type for e in events})}" + ) class TestMCPDisabled: - """Tests that verify behavior when MCP tools are disabled.""" + """Tests that MCP tools are not executed when the env flag is unset.""" @pytest.fixture(scope="class") - def monkeypatch_class(self): - from _pytest.monkeypatch import MonkeyPatch - - mpatch = MonkeyPatch() - yield mpatch - mpatch.undo() - - @pytest.fixture(scope="class") - def mcp_disabled_server(self, monkeypatch_class: pytest.MonkeyPatch): - args = ["--enforce-eager", "--tool-server", "demo"] - - with monkeypatch_class.context() as m: - m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1") - m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv") - # Helps the model follow instructions better - m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1") - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: - yield remote_server + def mcp_disabled_server(self): + env_dict = { + **BASE_TEST_ENV, + "VLLM_ENABLE_RESPONSES_API_STORE": "1", + "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv", + "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": "1", + } + with RemoteOpenAIServer( + MODEL_NAME, list(_BASE_SERVER_ARGS), env_dict=env_dict + ) as remote_server: + yield remote_server @pytest_asyncio.fixture - async def mcp_disabled_client(self, mcp_disabled_server): + async def client(self, mcp_disabled_server): async with mcp_disabled_server.get_async_client() as async_client: yield async_client @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) - async def test_mcp_tool_env_flag_disabled( - self, mcp_disabled_client: OpenAI, model_name: str + async def test_mcp_disabled_server_does_not_execute( + self, client: OpenAI, model_name: str ): - response = await mcp_disabled_client.responses.create( + """When MCP is disabled the model may still attempt tool calls + (tool descriptions can remain in the prompt), but the server + must NOT execute them.""" + response = await client.responses.create( model=model_name, input=( "Execute the following code if the tool is present: " @@ -308,38 +287,35 @@ class TestMCPDisabled: { "type": "mcp", "server_label": "code_interpreter", - # URL unused for DemoToolServer "server_url": "http://localhost:8888", } ], + temperature=0.0, extra_body={"enable_response_messages": True}, ) assert response is not None assert response.status == "completed" - # Verify output messages: No tool calls and responses - tool_call_found = False - tool_response_found = False + + log_response_diagnostics(response, label="MCP Disabled") + + # Server must not have executed any tool calls for message in response.output_messages: - recipient = message.get("recipient") - if recipient and recipient.startswith("python"): - tool_call_found = True - assert message.get("channel") == "analysis", ( - "Tool call should be on analysis channel" - ) author = message.get("author", {}) - if ( + assert not ( author.get("role") == "tool" - and author.get("name") - and author.get("name").startswith("python") - ): - tool_response_found = True - assert message.get("channel") == "analysis", ( - "Tool response should be on analysis channel" + and (author.get("name") or "").startswith("python") + ), ( + "Server executed a python tool call even though MCP is " + f"disabled. Message: {message}" + ) + + # No completed mcp_call output items + for item in response.output: + if getattr(item, "type", None) == "mcp_call": + assert getattr(item, "status", None) != "completed", ( + "MCP call should not be completed when MCP is disabled" ) - assert not tool_call_found, "Should not have a python call" - assert not tool_response_found, "Should not have a tool response" + # No developer messages injected for message in response.input_messages: - assert message.get("author").get("role") != "developer", ( - "No developer messages should be present without a valid tool" - ) + assert message.get("author", {}).get("role") != "developer" diff --git a/tests/entrypoints/openai/responses/test_parsable_context.py b/tests/entrypoints/openai/responses/test_parsable_context.py index 16a5c735e93..280bacf47ee 100644 --- a/tests/entrypoints/openai/responses/test_parsable_context.py +++ b/tests/entrypoints/openai/responses/test_parsable_context.py @@ -3,15 +3,29 @@ import importlib.util import json +import logging import pytest import pytest_asyncio from openai import OpenAI from ....utils import RemoteOpenAIServer +from .conftest import ( + BASE_TEST_ENV, + has_output_type, + log_response_diagnostics, + retry_for_tool_call, +) + +logger = logging.getLogger(__name__) MODEL_NAME = "Qwen/Qwen3-8B" +_PYTHON_TOOL_INSTRUCTION = ( + "You must use the Python tool to execute code. " + "Never simulate execution. You must print the final answer." +) + @pytest.fixture(scope="module") def server(): @@ -32,12 +46,12 @@ def server(): "--tool-server", "demo", ] - env_dict = dict( - VLLM_ENABLE_RESPONSES_API_STORE="1", - VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT="1", - PYTHON_EXECUTION_BACKEND="dangerously_use_uv", - ) - + env_dict = { + **BASE_TEST_ENV, + "VLLM_ENABLE_RESPONSES_API_STORE": "1", + "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": "1", + "PYTHON_EXECUTION_BACKEND": "dangerously_use_uv", + } with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: yield remote_server @@ -54,6 +68,7 @@ async def test_basic(client: OpenAI, model_name: str): response = await client.responses.create( model=model_name, input="What is 123 * 456?", + temperature=0.0, ) assert response is not None print("response: ", response) @@ -99,10 +114,15 @@ async def test_reasoning_and_function_items(client: OpenAI, model_name: str): ) assert response is not None assert response.status == "completed" - # make sure we get a reasoning and text output - assert response.output[0].type == "reasoning" - assert response.output[1].type == "message" - assert type(response.output[1].content[0].text) is str + + output_types = [getattr(o, "type", None) for o in response.output] + assert "reasoning" in output_types, ( + f"Expected reasoning in output, got: {output_types}" + ) + assert "message" in output_types, f"Expected message in output, got: {output_types}" + + msg = next(o for o in response.output if o.type == "message") + assert type(msg.content[0].text) is str def get_horoscope(sign): @@ -110,10 +130,10 @@ def get_horoscope(sign): def call_function(name, args): + logger.info("Calling function %s with args %s", name, args) if name == "get_horoscope": return get_horoscope(**args) - else: - raise ValueError(f"Unknown function: {name}") + raise ValueError(f"Unknown function: {name}") @pytest.mark.asyncio @@ -136,61 +156,111 @@ async def test_function_call_first_turn(client: OpenAI, model_name: str): } ] - response = await client.responses.create( + response = await retry_for_tool_call( + client, model=model_name, + expected_tool_type="function_call", input="What is the horoscope for Aquarius today?", tools=tools, temperature=0.0, ) assert response is not None assert response.status == "completed" - assert len(response.output) == 2 - assert response.output[0].type == "reasoning" - assert response.output[1].type == "function_call" - function_call = response.output[1] + output_types = [getattr(o, "type", None) for o in response.output] + assert "reasoning" in output_types, ( + f"Expected reasoning in output, got: {output_types}" + ) + assert has_output_type(response, "function_call"), ( + f"Expected function_call in output, got: {output_types}" + ) + + function_call = next(o for o in response.output if o.type == "function_call") assert function_call.name == "get_horoscope" assert function_call.call_id is not None args = json.loads(function_call.arguments) assert "sign" in args - # the multi turn function call is tested above in - # test_reasoning_and_function_items - @pytest.mark.asyncio @pytest.mark.parametrize("model_name", [MODEL_NAME]) async def test_mcp_tool_call(client: OpenAI, model_name: str): - response = await client.responses.create( + """MCP tool calling with code_interpreter. + + The model may make one or more tool calls before producing a final + message. We validate server invariants (mcp_call items have correct + fields) with hard assertions. Output indices are never hardcoded + since the model can produce multiple tool-call rounds. + """ + # MCP + container init + code execution can be slow + client_with_timeout = client.with_options(timeout=client.timeout * 3) + + response = await retry_for_tool_call( + client_with_timeout, model=model_name, - input="What is 123 * 456? Use python to calculate the result.", + expected_tool_type="mcp_call", + input=( + "What is 123 * 456? Use python to calculate the result. " + "Print the result with print()." + ), tools=[{"type": "code_interpreter", "container": {"type": "auto"}}], - extra_body={"enable_response_messages": True}, + instructions=_PYTHON_TOOL_INSTRUCTION, temperature=0.0, + extra_body={"enable_response_messages": True}, ) assert response is not None - assert response.status == "completed" - # The model may produce multiple reasoning/mcp_call rounds before the - # final message, so validate structurally rather than by exact index. - output_types = [o.type for o in response.output] - assert "reasoning" in output_types - mcp_calls = [o for o in response.output if o.type == "mcp_call"] - assert len(mcp_calls) >= 1 - assert type(mcp_calls[0].arguments) is str - assert type(mcp_calls[0].output) is str + output_types = [getattr(o, "type", None) for o in response.output] + log_response_diagnostics(response, label="test_mcp_tool_call") - # The final output should be a message containing the correct answer - assert response.output[-1].type == "message" - assert any(s in response.output[-1].content[0].text for s in ("56088", "56,088")) + assert response.status == "completed", ( + f"Response status={response.status} " + f"(details={getattr(response, 'incomplete_details', None)}). " + f"Output types: {output_types}." + ) - # Test raw input_messages / output_messages - assert len(response.input_messages) == 1 - assert len(response.output_messages) >= 3 + assert "reasoning" in output_types, ( + f"Expected reasoning in output, got: {output_types}" + ) + assert "mcp_call" in output_types, ( + f"Expected mcp_call in output, got: {output_types}" + ) + + # Every mcp_call item must have well-typed fields + for item in response.output: + if getattr(item, "type", None) == "mcp_call": + assert type(item.arguments) is str, ( + f"mcp_call.arguments should be str, got {type(item.arguments)}" + ) + assert type(item.output) is str, ( + f"mcp_call.output should be str, got {type(item.output)}" + ) + + # The model may make 1+ tool-call rounds but must still produce + # a final message for a trivial calculation like 123 * 456. + message_outputs = [ + o for o in response.output if getattr(o, "type", None) == "message" + ] + assert message_outputs, ( + f"Model did not produce a final message. Output types: {output_types}" + ) + + final_message = message_outputs[-1] + assert any(s in final_message.content[0].text for s in ("56088", "56,088")), ( + f"Expected 56088 in final message, got: {final_message.content[0].text!r}" + ) + + # Validate raw input_messages / output_messages + assert len(response.input_messages) >= 1, "Expected at least 1 input message" + assert len(response.output_messages) >= 1, "Expected at least 1 output message" assert any( - s in response.output_messages[-1]["message"] for s in ("56088", "56,088") + any(s in str(msg) for s in ("56088", "56,088")) + for msg in response.output_messages + ), ( + f"Expected 56088 in at least one output_message, " + f"got {len(response.output_messages)} messages" ) @@ -202,6 +272,7 @@ async def test_max_tokens(client: OpenAI, model_name: str): input="What is the first paragraph of Moby Dick?", reasoning={"effort": "low"}, max_output_tokens=30, + temperature=0.0, ) assert response is not None assert response.status == "incomplete" diff --git a/tests/entrypoints/openai/responses/test_simple.py b/tests/entrypoints/openai/responses/test_simple.py index db536d2fa5d..b67f0d34115 100644 --- a/tests/entrypoints/openai/responses/test_simple.py +++ b/tests/entrypoints/openai/responses/test_simple.py @@ -12,13 +12,15 @@ MODEL_NAME = "Qwen/Qwen3-8B" @pytest.fixture(scope="module") def server(): - args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"] - env_dict = dict( - VLLM_ENABLE_RESPONSES_API_STORE="1", - # uncomment for tool calling - # PYTHON_EXECUTION_BACKEND="dangerously_use_uv", - ) + from .conftest import BASE_TEST_ENV + args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"] + env_dict = { + **BASE_TEST_ENV, + "VLLM_ENABLE_RESPONSES_API_STORE": "1", + # uncomment for tool calling + # PYTHON_EXECUTION_BACKEND: "dangerously_use_uv", + } with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server: yield remote_server diff --git a/tests/utils.py b/tests/utils.py index 9ab6df9e231..c12b235fa11 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -128,6 +128,9 @@ class RemoteOpenAIServer: env=env, stdout=sys.stdout, stderr=sys.stderr, + # Create a dedicated process group so we can kill + # the entire tree (parent + EngineCore + workers) at once. + start_new_session=True, ) def __init__( @@ -189,6 +192,15 @@ class RemoteOpenAIServer: model_loader = get_model_loader(load_config) model_loader.download_model(model_config) + # Record GPU memory before server start so we know what + # "released" looks like. + self._pre_server_gpu_memory = self._get_gpu_memory_used() + if self._pre_server_gpu_memory is not None: + pre_gb = self._pre_server_gpu_memory / 1e9 + print( + f"[RemoteOpenAIServer] GPU memory before server start: {pre_gb:.2f} GB" + ) + self._start_server(model, vllm_serve_args, env_dict) max_wait_seconds = max_wait_seconds or 360 self._wait_for_server(url=self.url_for("health"), timeout=max_wait_seconds) @@ -198,27 +210,69 @@ class RemoteOpenAIServer: def __exit__(self, exc_type, exc_value, traceback): pid = self.proc.pid - # Graceful shutdown - self.proc.terminate() + + # Get the process group ID. Because we used + # start_new_session=True the pgid equals the server's pid. + try: + pgid = os.getpgid(pid) + except (ProcessLookupError, OSError): + pgid = None + + # Phase 1: graceful SIGTERM to the entire process group + if pgid is not None: + with contextlib.suppress(ProcessLookupError, OSError): + os.killpg(pgid, signal.SIGTERM) + print(f"[RemoteOpenAIServer] Sent SIGTERM to process group {pgid}") + else: + self.proc.terminate() + try: self.proc.wait(timeout=15) print(f"[RemoteOpenAIServer] Server {pid} terminated gracefully") except subprocess.TimeoutExpired: + # Phase 2: SIGKILL the entire process group print( f"[RemoteOpenAIServer] Server {pid} did not respond " - "to SIGTERM, sending SIGKILL" + "to SIGTERM, sending SIGKILL to process group" ) - self.proc.kill() + if pgid is not None: + with contextlib.suppress(ProcessLookupError, OSError): + os.killpg(pgid, signal.SIGKILL) + else: + self.proc.kill() + try: - self.proc.wait(timeout=5) + self.proc.wait(timeout=10) print(f"[RemoteOpenAIServer] Server {pid} killed") - except subprocess.TimeoutExpired as err: - raise RuntimeError( - f"[RemoteOpenAIServer] Failed to kill server process {pid}" - ) from err - # Wait for GPU memory to be released + except subprocess.TimeoutExpired: + # Phase 3: last resort - find and kill any orphaned children + self._kill_orphaned_children(pid) + + # Wait for GPU memory to actually be *freed*, not just + # "stabilized at whatever level it's at". self._wait_for_gpu_memory_release() + def _kill_orphaned_children(self, parent_pid: int) -> None: + """Best-effort cleanup of any lingering child processes.""" + try: + import psutil + + parent = psutil.Process(parent_pid) + children = parent.children(recursive=True) + for child in children: + print( + f"[RemoteOpenAIServer] Killing orphaned child " + f"pid={child.pid} name={child.name()}" + ) + child.kill() + psutil.wait_procs(children, timeout=5) + except Exception as e: + # psutil may not be installed, or processes already gone + print(f"[RemoteOpenAIServer] Orphan cleanup failed: {e}") + # Fallback: try to kill by pgid one more time + with contextlib.suppress(ProcessLookupError, OSError): + os.killpg(parent_pid, signal.SIGKILL) + def _get_gpu_memory_used(self) -> float | None: """Get total GPU memory used across all visible devices in bytes.""" try: @@ -244,10 +298,26 @@ class RemoteOpenAIServer: return None return None - def _wait_for_gpu_memory_release(self, timeout: float = 30.0): - """Poll GPU memory until it stabilizes, indicating cleanup is complete.""" + def _wait_for_gpu_memory_release(self, timeout: float = 60.0): + """Wait for GPU memory to drop back toward pre-server levels. + + Two-phase strategy: + 1. Try to wait for memory to return close to pre-server baseline. + 2. If that doesn't happen, fall back to waiting for stabilization + and log a warning (the next server might still OOM). + """ + baseline = self._pre_server_gpu_memory + if baseline is None: + # Can't query GPU memory - nothing to do + return + + # Allow up to 2 GiB overhead above baseline for driver/context state + # that may persist between server instances. + headroom_bytes = 2 * 1024 * 1024 * 1024 + target = baseline + headroom_bytes + start = time.time() - prev_used: float | None = None + last_used: float | None = None stable_count = 0 while time.time() - start < timeout: @@ -256,26 +326,49 @@ class RemoteOpenAIServer: if used is None: return # Can't query, assume ok - if prev_used is not None and abs(used - prev_used) < 100 * 1024 * 1024: - stable_count += 1 - if stable_count >= 3: - used_gb = used / 1e9 - print( - f"[RemoteOpenAIServer] GPU memory stabilized " - f"at {used_gb:.2f} GB" - ) - return - else: - stable_count = 0 + used_gb = used / 1e9 + target_gb = target / 1e9 + elapsed = time.time() - start - prev_used = used - time.sleep(0.1) + # Phase 1: memory dropped to near baseline - we're done. + if used <= target: + print( + f"[RemoteOpenAIServer] GPU memory released to " + f"{used_gb:.2f} GB (target: {target_gb:.2f} GB) " + f"in {elapsed:.1f}s" + ) + return - last_reading = prev_used / 1e9 if prev_used is not None else 0.0 + # Phase 2 (after 40s): fall back to stabilization check. + # This handles cases where another process is using GPU memory + # and we'll never reach baseline. + if elapsed > 40.0 and last_used is not None: + delta = abs(used - last_used) + if delta < 200 * 1024 * 1024: # 200 MB + stable_count += 1 + if stable_count >= 3: + print( + f"[RemoteOpenAIServer] WARNING: GPU memory " + f"stabilized at {used_gb:.2f} GB " + f"(target was {target_gb:.2f} GB). " + f"Proceeding - next server may OOM." + ) + return + else: + stable_count = 0 + + last_used = used + time.sleep(1.0) + + # Timeout - log clearly so CI failures are diagnosable + final_used = self._get_gpu_memory_used() + final_gb = final_used / 1e9 if final_used else 0.0 raise RuntimeError( - f"[RemoteOpenAIServer] GPU memory did not stabilize within {timeout}s. " - f"Last reading: {last_reading:.2f} GB. " - "Child processes may still be holding GPU memory." + f"[RemoteOpenAIServer] GPU memory did not release within " + f"{timeout}s. Current: {final_gb:.2f} GB, " + f"target: {target / 1e9:.2f} GB, " + f"baseline: {baseline / 1e9:.2f} GB. " + f"Child processes may still be holding GPU memory." ) def _poll(self) -> int | None: diff --git a/vllm/entrypoints/openai/parser/harmony_utils.py b/vllm/entrypoints/openai/parser/harmony_utils.py index 3bb81273878..486873db809 100644 --- a/vllm/entrypoints/openai/parser/harmony_utils.py +++ b/vllm/entrypoints/openai/parser/harmony_utils.py @@ -48,8 +48,11 @@ from vllm.entrypoints.openai.responses.protocol import ( ResponseInputOutputItem, ResponsesRequest, ) +from vllm.logger import init_logger from vllm.utils import random_uuid +logger = init_logger(__name__) + REASONING_EFFORT = { "high": ReasoningEffort.HIGH, "medium": ReasoningEffort.MEDIUM, @@ -62,20 +65,15 @@ _harmony_encoding = None # they are available and requested by the user. # Tool args are provided by MCP tool descriptions. Output # of the tools are stringified. -MCP_BUILTIN_TOOLS: set[str] = { - "web_search_preview", - "code_interpreter", - "container", -} - -# Mapping from built-in tool recipient names to their MCP server labels. -# This ensures consistency between streaming and non-streaming responses. _BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = { "python": "code_interpreter", "browser": "web_search_preview", "container": "container", } +# Derive MCP_BUILTIN_TOOLS from the canonical mapping +MCP_BUILTIN_TOOLS: set[str] = set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) + def has_custom_tools(tool_types: set[str]) -> bool: """ @@ -116,8 +114,11 @@ def get_system_message( REASONING_EFFORT[reasoning_effort] ) if start_date is None: - # NOTE(woosuk): This brings non-determinism in vLLM. Be careful. - start_date = datetime.datetime.now().strftime("%Y-%m-%d") + # NOTE(woosuk): This brings non-determinism in vLLM. + # Set VLLM_SYSTEM_START_DATE to pin it. + start_date = envs.VLLM_SYSTEM_START_DATE or datetime.datetime.now().strftime( + "%Y-%m-%d" + ) sys_msg_content = sys_msg_content.with_conversation_start_date(start_date) if browser_description is not None: sys_msg_content = sys_msg_content.with_tools(browser_description) @@ -398,15 +399,60 @@ def parse_chat_input_to_harmony_message( def parse_input_to_harmony_message(chat_msg) -> list[Message]: - """ - Parse a message from request.previous_input_messages in the Responsees API to - Harmony messages. + """Parse a message from request.previous_input_messages + into Harmony messages. + + Supports both OpenAI chat format ({"role": "..."}) and + Harmony format ({"author": {"role": "..."}}). """ if not isinstance(chat_msg, dict): - # Handle Pydantic models chat_msg = chat_msg.model_dump(exclude_none=True) + if "author" in chat_msg and isinstance(chat_msg.get("author"), dict): + return [_parse_harmony_format_message(chat_msg)] + + return _parse_chat_format_message(chat_msg) + + +def _parse_harmony_format_message(chat_msg: dict) -> Message: + """Reconstruct a Message from Harmony-format dict, + preserving channel, recipient, and content_type.""" + author_dict = chat_msg["author"] + role = author_dict.get("role") + name = author_dict.get("name") + + raw_content = chat_msg.get("content", "") + if isinstance(raw_content, list): + # TODO: Support refusal and non-text content types. + contents = [TextContent(text=c.get("text", "")) for c in raw_content] + elif isinstance(raw_content, str): + contents = [TextContent(text=raw_content)] + else: + contents = [TextContent(text="")] + + if name: + msg = Message.from_author_and_contents(Author.new(Role(role), name), contents) + else: + msg = Message.from_role_and_contents(Role(role), contents) + + channel = chat_msg.get("channel") + if channel: + msg = msg.with_channel(channel) + recipient = chat_msg.get("recipient") + if recipient: + msg = msg.with_recipient(recipient) + content_type = chat_msg.get("content_type") + if content_type: + msg = msg.with_content_type(content_type) + + return msg + + +def _parse_chat_format_message(chat_msg: dict) -> list[Message]: + """Parse an OpenAI chat-format dict into Harmony messages.""" role = chat_msg.get("role") + if role is None: + raise ValueError(f"Message has no 'role' key: {chat_msg}") # Assistant message with tool calls tool_calls = chat_msg.get("tool_calls") @@ -426,15 +472,21 @@ def parse_input_to_harmony_message(chat_msg) -> list[Message]: # Tool role message (tool output) if role == "tool": name = chat_msg.get("name", "") + if name and not name.startswith("functions."): + name = f"functions.{name}" content = chat_msg.get("content", "") or "" content = flatten_chat_text_content(content) - - msg = Message.from_author_and_content( - Author.new(Role.TOOL, f"functions.{name}"), content - ).with_channel("commentary") + # NOTE: .with_recipient("assistant") is required on tool messages + # to match parse_chat_input_to_harmony_message behavior and ensure + # proper routing in the Harmony protocol. + msg = ( + Message.from_author_and_content(Author.new(Role.TOOL, name), content) + .with_channel("commentary") + .with_recipient("assistant") + ) return [msg] - # Default: user/assistant/system messages with content + # Default: user/assistant/system messages content = chat_msg.get("content", "") if isinstance(content, str): contents = [TextContent(text=content)] @@ -497,6 +549,10 @@ def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutput try: browser_call = json.loads(content.text) except json.JSONDecodeError: + logger.warning( + "Invalid JSON in browser tool call, using error placeholder: %s", + content.text, + ) json_retry_output_message = ( f"Invalid JSON args, caught and retried: {content.text}" ) @@ -730,22 +786,7 @@ def parse_remaining_state(parser: StreamableParser) -> list[ResponseOutputItem]: ) ] - if parser.current_channel == "commentary": - return [ - ResponseReasoningItem( - id=f"rs_{random_uuid()}", - summary=[], - type="reasoning", - content=[ - ResponseReasoningTextContent( - text=parser.current_content, type="reasoning_text" - ) - ], - status=None, - ) - ] - - if parser.current_channel == "analysis": + if parser.current_channel in ("commentary", "analysis"): return [ ResponseReasoningItem( id=f"rs_{random_uuid()}", diff --git a/vllm/entrypoints/openai/responses/context.py b/vllm/entrypoints/openai/responses/context.py index 9559e794896..b57adeeb846 100644 --- a/vllm/entrypoints/openai/responses/context.py +++ b/vllm/entrypoints/openai/responses/context.py @@ -346,17 +346,17 @@ class ParsableContext(ConversationContext): self.parser.response_messages.extend(output) def need_builtin_tool_call(self) -> bool: - """Return true if the last message is a MCP tool call""" + """Return true if the last message is a builtin tool call + that the request has enabled.""" last_message = self.parser.response_messages[-1] - # TODO(qandrew): figure out which tools are MCP tools - if last_message.type == "function_call": # noqa: SIM102 - if last_message.name in ( - "code_interpreter", - "python", - "web_search_preview", - ) or last_message.name.startswith("container"): - return True - + if last_message.type != "function_call": + return False + if last_message.name in ("code_interpreter", "python"): + return "python" in self.available_tools + if last_message.name == "web_search_preview": + return "browser" in self.available_tools + if last_message.name.startswith("container"): + return "container" in self.available_tools return False async def call_python_tool( @@ -665,11 +665,15 @@ class HarmonyContext(ConversationContext): def need_builtin_tool_call(self) -> bool: last_msg = self.messages[-1] recipient = last_msg.recipient - return recipient is not None and ( - recipient.startswith("browser.") - or recipient.startswith("python") - or recipient.startswith("container.") - ) + if recipient is None: + return False + if recipient.startswith("browser."): + return "browser" in self.available_tools + if recipient.startswith("python"): + return "python" in self.available_tools + if recipient.startswith("container."): + return "container" in self.available_tools + return False async def call_tool(self) -> list[Message]: if not self.messages: diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 4055095fd2f..4de6a744604 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -392,13 +392,27 @@ class OpenAIServingResponses(OpenAIServing): max_model_len = self.model_config.max_model_len generators: list[AsyncGenerator[ConversationContext, None]] = [] + # Only include builtin tools that the request actually asked for. + # Without this filter, tools registered on the server (e.g. via + # --tool-server demo) would be available for execution even when + # the request didn't enable them. + requested_tool_types = extract_tool_types(request.tools) builtin_tool_list: list[str] = [] if self.tool_server is not None: - if self.tool_server.has_tool("browser"): + if ( + self.tool_server.has_tool("browser") + and "web_search_preview" in requested_tool_types + ): builtin_tool_list.append("browser") - if self.tool_server.has_tool("python"): + if ( + self.tool_server.has_tool("python") + and "code_interpreter" in requested_tool_types + ): builtin_tool_list.append("python") - if self.tool_server.has_tool("container"): + if ( + self.tool_server.has_tool("container") + and "container" in requested_tool_types + ): builtin_tool_list.append("container") if self.tool_server is not None: @@ -1049,9 +1063,15 @@ class OpenAIServingResponses(OpenAIServing): # FIXME(woosuk): Currently, request params like reasoning and # instructions are ignored. prev_msgs = self.msg_store[prev_response.id] - # Remove the previous chain-of-thoughts if there is a new "final" - # message. Note that this also removes these messages from the - # msg_store. + + # FIXME(woosuk): The slice-delete-reappend cycle below is + # currently a no-op --- it removes messages then puts them all + # back unfiltered. It may be intentionally deferred (see FIXME + # above) or redundant if the Harmony encoder already strips + # analysis messages at render time. If analysis messages need + # to be dropped here, add a channel != "analysis" filter when + # re-appending, similar to auto_drop_analysis_messages in + # harmony_utils.py. if len(prev_msgs) > 0: last_msg = prev_msgs[-1] assert isinstance(last_msg, OpenAIHarmonyMessage) @@ -1072,7 +1092,11 @@ class OpenAIServingResponses(OpenAIServing): # Append the new input. # Responses API supports simple text inputs without chat format. if isinstance(request.input, str): - messages.append(get_user_message(request.input)) + # Skip empty string input when previous_input_messages supplies + # the full conversation history --- an empty trailing user message + # confuses the model into thinking nothing was sent. + if request.input or not request.previous_input_messages: + messages.append(get_user_message(request.input)) else: if prev_response is not None: prev_outputs = copy(prev_response.output) diff --git a/vllm/envs.py b/vllm/envs.py index b32683ecb99..2b341bd5bb7 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -209,6 +209,7 @@ if TYPE_CHECKING: VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set() VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False + VLLM_SYSTEM_START_DATE: str | None = None VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False @@ -1458,6 +1459,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS": lambda: bool( int(os.getenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "0")) ), + # Pin the conversation start date injected into the Harmony system + # message. When unset the current date is used, which introduces + # non-determinism (different tokens -> different model behaviour at + # temperature=0). Set to an ISO date string, e.g. "2023-09-12", + # for reproducible inference or testing. + "VLLM_SYSTEM_START_DATE": lambda: os.getenv("VLLM_SYSTEM_START_DATE", None), # Enable automatic retry when tool call JSON parsing fails # If enabled, returns an error message to the model to retry # If disabled (default), raises an exception and fails the request From a0fe7ea2f052bb44820bc06a5635456b8d1383af Mon Sep 17 00:00:00 2001 From: zhongdaor-nv Date: Fri, 20 Feb 2026 21:11:40 -0700 Subject: [PATCH 50/74] [feat] Add per-block extra_keys to KV events (#33304) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: zhongdaor-nv Co-authored-by: Nicolò Lucchesi Co-authored-by: Cyrus Leung --- .../online_serving/kv_events_subscriber.py | 6 ++ tests/v1/core/test_kv_cache_utils.py | 60 +++++++++++++++---- vllm/distributed/kv_events.py | 8 +++ vllm/v1/core/block_pool.py | 26 +++++++- vllm/v1/core/kv_cache_utils.py | 18 ++++-- vllm/v1/request.py | 3 + 6 files changed, 100 insertions(+), 21 deletions(-) diff --git a/examples/online_serving/kv_events_subscriber.py b/examples/online_serving/kv_events_subscriber.py index 30c3986f2fa..499ab1f3946 100644 --- a/examples/online_serving/kv_events_subscriber.py +++ b/examples/online_serving/kv_events_subscriber.py @@ -37,6 +37,12 @@ class BlockStored(KVCacheEvent): medium: str | None lora_name: str | None + extra_keys: list[tuple[Any, ...] | None] | None = None + """Extra keys used in block hash computation, one entry per block in + block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt, + prompt embeddings data, etc. for that specific block. + """ + class BlockRemoved(KVCacheEvent): block_hashes: list[ExternalBlockHash] diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index ceb8ec42479..c609bc1b85e 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import hashlib import importlib from collections.abc import Callable from typing import Any @@ -498,14 +499,41 @@ def test_generate_block_hash_extra_keys_prompt_embeds(): # Test with prompt embeds for the first block extra_keys, _ = generate_block_hash_extra_keys(request, 0, 5, 0) expected_embeds = prompt_embeds[0:5] - expected_bytes = kv_cache_utils.tensor_data(expected_embeds).tobytes() - assert extra_keys == (expected_bytes,) + expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest() + assert extra_keys == (expected_hash,) # Test with prompt embeds for the second block extra_keys, _ = generate_block_hash_extra_keys(request, 5, 10, 0) expected_embeds = prompt_embeds[5:10] - expected_bytes = kv_cache_utils.tensor_data(expected_embeds).tobytes() - assert extra_keys == (expected_bytes,) + expected_hash = hashlib.sha256(kv_cache_utils.tensor_data(expected_embeds)).digest() + assert extra_keys == (expected_hash,) + + +def test_generate_block_hash_extra_keys_prompt_embeds_cached(monkeypatch): + prompt_embeds = torch.randn(10, 3) + request = make_request( + request_id="0", + prompt_token_ids=None, + mm_positions=None, + mm_hashes=None, + prompt_embeds=prompt_embeds, + block_size=20, + ) + + num_tensor_data_calls = 0 + original_tensor_data = kv_cache_utils.tensor_data + + def counting_tensor_data(tensor: torch.Tensor): + nonlocal num_tensor_data_calls + num_tensor_data_calls += 1 + return original_tensor_data(tensor) + + monkeypatch.setattr(kv_cache_utils, "tensor_data", counting_tensor_data) + + extra_keys_1, _ = generate_block_hash_extra_keys(request, 0, 5, 0) + extra_keys_2, _ = generate_block_hash_extra_keys(request, 0, 5, 0) + assert extra_keys_1 == extra_keys_2 + assert num_tensor_data_calls == 1 def test_generate_block_hash_extra_keys_different_prompt_embeds(): @@ -1858,22 +1886,26 @@ def test_request_block_hasher_with_prompt_embeds(hash_fn: Callable[[Any], bytes] block_hashes = request.block_hashes assert len(block_hashes) == 2 - block1_embeds_bytes = tensor_data(prompt_embeds[:block_size]).tobytes() + block1_embeds_hash = hashlib.sha256( + tensor_data(prompt_embeds[:block_size]) + ).digest() expected_hash1 = hash_fn( ( kv_cache_utils.NONE_HASH, tuple(prompt_token_ids[:block_size]), - (block1_embeds_bytes,), + (block1_embeds_hash,), ) ) assert block_hashes[0] == expected_hash1 - block2_embeds_bytes = tensor_data(prompt_embeds[block_size:num_tokens]).tobytes() + block2_embeds_hash = hashlib.sha256( + tensor_data(prompt_embeds[block_size:num_tokens]) + ).digest() expected_hash2 = hash_fn( ( block_hashes[0], tuple(prompt_token_ids[block_size:num_tokens]), - (block2_embeds_bytes,), + (block2_embeds_hash,), ) ) assert block_hashes[1] == expected_hash2 @@ -1903,22 +1935,26 @@ def test_request_with_prompt_embeds_and_mm_inputs(hash_fn: Callable[[Any], bytes block_hashes = request.block_hashes assert len(block_hashes) == 2 - block1_embeds_bytes = tensor_data(prompt_embeds[:block_size]).tobytes() + block1_embeds_hash = hashlib.sha256( + tensor_data(prompt_embeds[:block_size]) + ).digest() expected_hash1 = hash_fn( ( kv_cache_utils.NONE_HASH, tuple(prompt_token_ids[:block_size]), - ("hash1", block1_embeds_bytes), + ("hash1", block1_embeds_hash), ) ) assert block_hashes[0] == expected_hash1 - block2_embeds_bytes = tensor_data(prompt_embeds[block_size:num_tokens]).tobytes() + block2_embeds_hash = hashlib.sha256( + tensor_data(prompt_embeds[block_size:num_tokens]) + ).digest() expected_hash2 = hash_fn( ( block_hashes[0], tuple(prompt_token_ids[block_size:num_tokens]), - ("hash2", block2_embeds_bytes), + ("hash2", block2_embeds_hash), ) ) assert block_hashes[1] == expected_hash2 diff --git a/vllm/distributed/kv_events.py b/vllm/distributed/kv_events.py index 123af17ef09..096ed441854 100644 --- a/vllm/distributed/kv_events.py +++ b/vllm/distributed/kv_events.py @@ -60,6 +60,13 @@ class BlockStored(KVCacheEvent): medium: str | None lora_name: str | None + extra_keys: list[tuple[Any, ...] | None] | None = None + """Extra keys used in block hash computation, one entry per block in + block_hashes. Each entry contains MM identifiers, LoRA name, cache_salt, + prompt embedding hashes, etc. for that specific block. Exposed for external + KV cache consumers to reconstruct block hashes. + """ + def __hash__(self) -> int: return hash( ( @@ -69,6 +76,7 @@ class BlockStored(KVCacheEvent): self.block_size, self.lora_id, self.medium, + tuple(self.extra_keys) if self.extra_keys else None, ) ) diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index ce7e396d8a9..4b62d2a4c64 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -20,6 +20,7 @@ from vllm.v1.core.kv_cache_utils import ( ExternalBlockHash, FreeKVCacheBlockQueue, KVCacheBlock, + generate_block_hash_extra_keys, get_block_hash, make_block_hash_with_group_id, maybe_convert_block_hash, @@ -279,13 +280,31 @@ class BlockPool: block_hashes[num_cached_blocks - 1] ) + # Calculate token range for the blocks being cached + start_token_idx = num_cached_blocks * block_size + end_token_idx = num_full_blocks * block_size + + # Generate extra keys for each block individually. + # Each block may have different extra_keys (e.g., different MM + # features, or cache_salt only for the first block). + # Skip null blocks to match the length of new_hashes. + extra_keys_list: list[tuple[Any, ...] | None] = [] + curr_mm_idx = 0 + for i in range(num_cached_blocks, num_full_blocks): + if blocks[i].is_null: + continue + block_start = i * block_size + block_end = block_start + block_size + extra_keys, curr_mm_idx = generate_block_hash_extra_keys( + request, block_start, block_end, curr_mm_idx + ) + extra_keys_list.append(extra_keys) + self.kv_event_queue.append( BlockStored( block_hashes=new_hashes, parent_block_hash=parent_block_hash, - token_ids=request.all_token_ids[ - num_cached_blocks * block_size : num_full_blocks * block_size - ], + token_ids=request.all_token_ids[start_token_idx:end_token_idx], block_size=block_size, lora_id=request.lora_request.adapter_id if request.lora_request @@ -294,6 +313,7 @@ class BlockPool: lora_name=request.lora_request.name if request.lora_request else None, + extra_keys=extra_keys_list if extra_keys_list else None, ) ) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 2f59e71a13d..cfaa37074d9 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -3,6 +3,7 @@ """KV-Cache Utilities.""" import copy +import hashlib import os from collections import defaultdict from collections.abc import Callable, Iterable, Iterator, Sequence @@ -475,14 +476,19 @@ def _gen_prompt_embeds_extra_hash_keys( end_token_idx: The end token index of the block. Returns: - Return prompt embeddings data of the request if it has prompt embeds. - Return empty list otherwise. + Return a stable hash of the block prompt embeddings if prompt embeds + are present. Return empty list otherwise. """ if request.prompt_embeds is None: return [] - block_prompt_embeds = request.prompt_embeds[start_token_idx:end_token_idx] - embeds_bytes = tensor_data(block_prompt_embeds).tobytes() - return [embeds_bytes] + block_range = (start_token_idx, end_token_idx) + embeds_hash = request._prompt_embeds_per_block_hashes.get(block_range) + if embeds_hash is None: + block_prompt_embeds = request.prompt_embeds[start_token_idx:end_token_idx] + # Hash prompt embeds once per block and cache on request + embeds_hash = hashlib.sha256(tensor_data(block_prompt_embeds)).digest() + request._prompt_embeds_per_block_hashes[block_range] = embeds_hash + return [embeds_hash] def generate_block_hash_extra_keys( @@ -490,7 +496,7 @@ def generate_block_hash_extra_keys( ) -> tuple[tuple[Any, ...] | None, int]: """Generate extra keys for the block hash. The extra keys can come from the multi-modal inputs, request specific metadata (e.g., LoRA names), and - data from prompt embeddings. + hashed data from prompt embeddings. Args: request: The request object. diff --git a/vllm/v1/request.py b/vllm/v1/request.py index 66ade00971b..7d8254e3505 100644 --- a/vllm/v1/request.py +++ b/vllm/v1/request.py @@ -114,6 +114,9 @@ class Request: self.prompt_token_ids = prompt_token_ids self.prompt_embeds = prompt_embeds + # Cache per-block prompt-embed hashes to avoid rehashing the same + # tensor slices when generating extra keys. + self._prompt_embeds_per_block_hashes: dict[tuple[int, int], bytes] = {} self.num_prompt_tokens = length_from_prompt_token_ids_or_embeds( prompt_token_ids, prompt_embeds ) From 89358f0d35e7923cf1554d4d652094c4ad2e80de Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 20 Feb 2026 22:12:05 -0600 Subject: [PATCH 51/74] [CI] Fix ColBERT HF comparison tests on AMD CI + refactor (#34567) Signed-off-by: Andreas Karatzas --- tests/models/language/pooling/test_colbert.py | 256 ++++++++---------- 1 file changed, 107 insertions(+), 149 deletions(-) diff --git a/tests/models/language/pooling/test_colbert.py b/tests/models/language/pooling/test_colbert.py index 21091c652a0..6edd9c28c51 100644 --- a/tests/models/language/pooling/test_colbert.py +++ b/tests/models/language/pooling/test_colbert.py @@ -20,6 +20,12 @@ COLBERT_MODELS = { "colbert_dim": 96, "max_model_len": 512, "extra_kwargs": {}, + "hf_comparison": { + "weights_file": "model.safetensors", + "weights_key": "linear.weight", + "trust_remote_code": False, + "model_cls": "BertModel", + }, }, "modernbert": { "model": "lightonai/GTE-ModernColBERT-v1", @@ -30,6 +36,12 @@ COLBERT_MODELS = { "architectures": ["ColBERTModernBertModel"], }, }, + "hf_comparison": { + "weights_file": "1_Dense/model.safetensors", + "weights_key": "linear.weight", + "trust_remote_code": False, + "model_cls": "AutoModel", + }, }, "jina": { "model": "jinaai/jina-colbert-v2", @@ -40,9 +52,16 @@ COLBERT_MODELS = { "architectures": ["ColBERTJinaRobertaModel"], }, }, + "hf_comparison": { + "weights_file": "model.safetensors", + "weights_key": "linear.weight", + "trust_remote_code": True, + "model_cls": "AutoModel", + }, }, } + TEXTS_1 = [ "What is the capital of France?", "What is the capital of Germany?", @@ -56,9 +75,68 @@ TEXTS_2 = [ DTYPE = "half" -# ----------------------------------------------------------------------- -# Fixtures -# ----------------------------------------------------------------------- +def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device): + """Load HF model on the given device with a compatible attention impl.""" + from transformers import AutoModel, BertModel + + cls = BertModel if hf_spec["model_cls"] == "BertModel" else AutoModel + trust = hf_spec.get("trust_remote_code", False) + + # Flash / Triton kernels require GPU tensors; fall back to eager on CPU. + extra = {} + if device.type == "cpu": + extra["attn_implementation"] = "eager" + + model = cls.from_pretrained( + model_name, + trust_remote_code=trust, + **extra, + ).to(device) + model.eval() + return model + + +def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device): + """Download and return the ColBERT linear projection weight.""" + from huggingface_hub import hf_hub_download + from safetensors.torch import load_file + + path = hf_hub_download(model_name, filename=hf_spec["weights_file"]) + weights = load_file(path) + return weights[hf_spec["weights_key"]].to(device) + + +def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, device): + """Run HF model + projection and return L2-normalised token embeddings.""" + import torch.nn.functional as F + + embeddings = [] + for text in texts: + inputs = tokenizer(text, return_tensors="pt").to(device) + with torch.no_grad(): + hidden = model(**inputs).last_hidden_state.float() + projected = F.linear(hidden, linear_weight.float()) + normalised = F.normalize(projected, p=2, dim=-1) + embeddings.append(normalised.squeeze(0).cpu()) + return embeddings + + +def _assert_embeddings_close(vllm_outputs, hf_embeddings): + """Assert that vLLM and HuggingFace embeddings match.""" + for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)): + vllm_emb = torch.as_tensor(vllm_out).float() + + assert hf_emb.shape == vllm_emb.shape, ( + f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}" + ) + + torch.testing.assert_close( + vllm_emb, + hf_emb, + rtol=1e-2, + atol=1e-2, + msg=f"Embedding mismatch for text {i}", + ) @pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="module") @@ -87,11 +165,6 @@ def colbert_extra_kwargs(colbert_spec): return colbert_spec["extra_kwargs"] -# ----------------------------------------------------------------------- -# Tests -# ----------------------------------------------------------------------- - - def test_colbert_token_embed( vllm_runner, colbert_model_name, @@ -111,7 +184,7 @@ def test_colbert_token_embed( outputs = vllm_model.token_embed([TEXTS_1[0]]) assert len(outputs) == 1 - emb = torch.tensor(outputs[0]) + emb = torch.as_tensor(outputs[0]) assert emb.dim() == 2 assert emb.shape[1] == colbert_dim assert emb.shape[0] > 1 @@ -135,8 +208,8 @@ def test_colbert_late_interaction_1_to_1( q_outputs = vllm_model.token_embed([TEXTS_1[0]]) d_outputs = vllm_model.token_embed([TEXTS_2[0]]) - q_emb = torch.tensor(q_outputs[0]) - d_emb = torch.tensor(d_outputs[0]) + q_emb = torch.as_tensor(q_outputs[0]) + d_emb = torch.as_tensor(d_outputs[0]) manual_score = compute_maxsim_score(q_emb, d_emb).item() @@ -164,11 +237,11 @@ def test_colbert_late_interaction_1_to_N( q_outputs = vllm_model.token_embed([TEXTS_1[0]]) d_outputs = vllm_model.token_embed(TEXTS_2) - q_emb = torch.tensor(q_outputs[0]) + q_emb = torch.as_tensor(q_outputs[0]) manual_scores = [] for d_out in d_outputs: - d_emb = torch.tensor(d_out) + d_emb = torch.as_tensor(d_out) manual_scores.append(compute_maxsim_score(q_emb, d_emb).item()) vllm_scores = vllm_model.score(TEXTS_1[0], TEXTS_2) @@ -198,8 +271,8 @@ def test_colbert_late_interaction_N_to_N( manual_scores = [] for q_out, d_out in zip(q_outputs, d_outputs): - q_emb = torch.tensor(q_out) - d_emb = torch.tensor(d_out) + q_emb = torch.as_tensor(q_out) + d_emb = torch.as_tensor(d_out) manual_scores.append(compute_maxsim_score(q_emb, d_emb).item()) vllm_scores = vllm_model.score(TEXTS_1, TEXTS_2) @@ -259,79 +332,16 @@ def test_colbert_embed_not_supported( vllm_model.embed([TEXTS_1[0]]) -# ----------------------------------------------------------------------- -# Per-model HuggingFace comparison tests -# ----------------------------------------------------------------------- +@pytest.mark.parametrize("backend", list(COLBERT_MODELS.keys())) +def test_colbert_hf_comparison(vllm_runner, backend): + """Test that vLLM ColBERT embeddings match HuggingFace for each backend.""" + from transformers import AutoTokenizer - -def _assert_embeddings_close(vllm_outputs, hf_embeddings): - """Assert that vLLM and HuggingFace embeddings match.""" - for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)): - vllm_emb = torch.tensor(vllm_out).float() - - assert hf_emb.shape == vllm_emb.shape, ( - f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}" - ) - - torch.testing.assert_close( - vllm_emb, - hf_emb, - rtol=1e-2, - atol=1e-2, - msg=f"Embedding mismatch for text {i}", - ) - - -def test_colbert_hf_comparison_bert(vllm_runner): - """Test that vLLM ColBERT produces same embeddings as HuggingFace (BERT).""" - import torch.nn.functional as F - from huggingface_hub import hf_hub_download - from safetensors.torch import load_file - from transformers import AutoTokenizer, BertModel - - model_name = COLBERT_MODELS["bert"]["model"] - test_texts = [TEXTS_1[0], TEXTS_2[0]] - - with vllm_runner( - model_name, - runner="pooling", - dtype="float32", - max_model_len=512, - enforce_eager=True, - ) as vllm_model: - vllm_outputs = vllm_model.token_embed(test_texts) - - hf_tokenizer = AutoTokenizer.from_pretrained(model_name) - hf_bert = BertModel.from_pretrained(model_name) - hf_bert.eval() - - weights_path = hf_hub_download(model_name, filename="model.safetensors") - weights = load_file(weights_path) - linear_weight = weights["linear.weight"] # [96, 384] - - hf_embeddings = [] - for text in test_texts: - inputs = hf_tokenizer(text, return_tensors="pt") - with torch.no_grad(): - outputs = hf_bert(**inputs) - hidden_states = outputs.last_hidden_state - token_emb = F.linear(hidden_states, linear_weight) - token_emb = F.normalize(token_emb, p=2, dim=-1) - hf_embeddings.append(token_emb.squeeze(0).float()) - - _assert_embeddings_close(vllm_outputs, hf_embeddings) - - -def test_colbert_hf_comparison_modernbert(vllm_runner): - """Test that vLLM ColBERT produces same embeddings as HuggingFace - (ModernBERT).""" - import torch.nn.functional as F - from huggingface_hub import hf_hub_download - from safetensors.torch import load_file - from transformers import AutoModel, AutoTokenizer - - spec = COLBERT_MODELS["modernbert"] + spec = COLBERT_MODELS[backend] + hf_spec = spec["hf_comparison"] model_name = spec["model"] + assert isinstance(model_name, str) + assert isinstance(hf_spec, dict) test_texts = [TEXTS_1[0], TEXTS_2[0]] with vllm_runner( @@ -344,73 +354,21 @@ def test_colbert_hf_comparison_modernbert(vllm_runner): ) as vllm_model: vllm_outputs = vllm_model.token_embed(test_texts) - hf_tokenizer = AutoTokenizer.from_pretrained(model_name) - hf_model = AutoModel.from_pretrained(model_name) - hf_model.eval() - - # Load projection from sentence-transformers 1_Dense layer - dense_path = hf_hub_download(model_name, filename="1_Dense/model.safetensors") - dense_weights = load_file(dense_path) - linear_weight = dense_weights["linear.weight"] # [128, 768] - - hf_embeddings = [] - for text in test_texts: - inputs = hf_tokenizer(text, return_tensors="pt") - with torch.no_grad(): - outputs = hf_model(**inputs) - hidden_states = outputs.last_hidden_state - token_emb = F.linear(hidden_states, linear_weight) - token_emb = F.normalize(token_emb, p=2, dim=-1) - hf_embeddings.append(token_emb.squeeze(0).float()) - - _assert_embeddings_close(vllm_outputs, hf_embeddings) - - -def test_colbert_hf_comparison_jina(vllm_runner): - """Test that vLLM ColBERT produces same embeddings as HuggingFace - (Jina XLM-RoBERTa).""" - import torch.nn.functional as F - from huggingface_hub import hf_hub_download - from safetensors.torch import load_file - from transformers import AutoModel, AutoTokenizer - - spec = COLBERT_MODELS["jina"] - model_name = spec["model"] - test_texts = [TEXTS_1[0], TEXTS_2[0]] - - with vllm_runner( - model_name, - runner="pooling", - dtype="float32", - max_model_len=spec["max_model_len"], - enforce_eager=True, - **spec["extra_kwargs"], - ) as vllm_model: - vllm_outputs = vllm_model.token_embed(test_texts) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") hf_tokenizer = AutoTokenizer.from_pretrained( model_name, - trust_remote_code=True, + trust_remote_code=hf_spec.get("trust_remote_code", False), ) - hf_model = AutoModel.from_pretrained( - model_name, - trust_remote_code=True, + hf_model = _load_hf_model(model_name, hf_spec, device) + linear_weight = _load_projection_weight(model_name, hf_spec, device) + + hf_embeddings = _compute_hf_colbert_embeddings( + hf_model, + hf_tokenizer, + linear_weight, + test_texts, + device, ) - hf_model.eval() - - # Load projection from main checkpoint - weights_path = hf_hub_download(model_name, filename="model.safetensors") - weights = load_file(weights_path) - linear_weight = weights["linear.weight"] # [128, 1024] - - hf_embeddings = [] - for text in test_texts: - inputs = hf_tokenizer(text, return_tensors="pt") - with torch.no_grad(): - outputs = hf_model(**inputs) - hidden_states = outputs.last_hidden_state - token_emb = F.linear(hidden_states.float(), linear_weight.float()) - token_emb = F.normalize(token_emb, p=2, dim=-1) - hf_embeddings.append(token_emb.squeeze(0).float()) _assert_embeddings_close(vllm_outputs, hf_embeddings) From cf93c1a12849693d6bcee3e2c917c02e1fc9a47f Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 20 Feb 2026 22:25:07 -0600 Subject: [PATCH 52/74] [ROCm][AITER] Fix aiter paged_attention_v1 decode for sliding window and head_size < 64 (#34570) Signed-off-by: Andreas Karatzas --- vllm/v1/attention/backends/rocm_aiter_fa.py | 45 ++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 141d57d908e..2ea3c346f5a 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -1114,7 +1114,50 @@ class AiterFlashAttentionImpl(AttentionImpl): ) return - if rocm_aiter_ops.is_shuffle_kv_cache_enabled(): + # The ll4mi kernel in paged_attention_v1 requires + # HEAD_SIZE >= 16 * NWARPS (= 64 on ROCm with NWARPS=4). + # For smaller head sizes or sliding window attention, + # fall back to the unified_attention triton kernel which + # handles both correctly. + _MIN_HEAD_SIZE_FOR_LL4MI = 64 + use_unified_attention = self.head_size < _MIN_HEAD_SIZE_FOR_LL4MI + + if use_unified_attention: + assert not rocm_aiter_ops.is_shuffle_kv_cache_enabled(), ( + "unified_attention fallback with shuffle layout " + "is not supported yet." + ) + from aiter.ops.triton.unified_attention import ( + unified_attention, + ) + + decode_cu_seqlens_q = attn_metadata.query_start_loc[ + : num_decodes + 1 + ] + descale_shape = ( + num_decodes, + key_cache.shape[2], + ) + unified_attention( + q=query[:num_decode_tokens], + k=key_cache, + v=value_cache, + out=output[:num_decode_tokens], + cu_seqlens_q=decode_cu_seqlens_q, + max_seqlen_q=1, + seqused_k=attn_metadata.seq_lens[:num_decodes], + max_seqlen_k=attn_metadata.max_seq_len, + softmax_scale=self.scale, + causal=True, + alibi_slopes=self.alibi_slopes, + window_size=self.sliding_window, + block_table=attn_metadata.block_table[:num_decodes], + softcap=self.logits_soft_cap, + q_descale=None, + k_descale=layer._k_scale.expand(descale_shape), + v_descale=layer._v_scale.expand(descale_shape), + ) + elif rocm_aiter_ops.is_shuffle_kv_cache_enabled(): num_blocks, block_size, num_kv_heads, head_size = key_cache.shape x = 16 // key_cache.element_size() k_cache_template = torch.empty( From 54254f7a6155002b1a493c3cefda9752bc9ce92f Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 20 Feb 2026 22:25:23 -0600 Subject: [PATCH 53/74] [ROCm][CI] Fix spec decode logprobs flakiness and parametrize tree attention backends (#34599) Signed-off-by: Andreas Karatzas --- tests/v1/sample/test_logprobs.py | 282 +++++++++++--------- tests/v1/spec_decode/test_tree_attention.py | 215 +++++++++++++-- 2 files changed, 351 insertions(+), 146 deletions(-) diff --git a/tests/v1/sample/test_logprobs.py b/tests/v1/sample/test_logprobs.py index 7466e361967..329f286683f 100644 --- a/tests/v1/sample/test_logprobs.py +++ b/tests/v1/sample/test_logprobs.py @@ -52,7 +52,7 @@ def vllm_model(vllm_runner, request) -> Generator[VllmRunner, None, None]: # TODO: enable this once we support it for # prompt logprobs. enable_prefix_caching=request.param, - gpu_memory_utilization=0.4, # up to 2 alive concurrently + gpu_memory_utilization=0.4, ) as vllm_model: yield vllm_model @@ -366,21 +366,20 @@ def test_max_logprobs(): Should also fail for `prompt_logprobs > max_logprobs` APC should not matter as this test checks basic request validation. """ - runner = VllmRunner( + with VllmRunner( "facebook/opt-125m", max_logprobs=1, enable_prefix_caching=False, - # 2 other llms alive during whole session gpu_memory_utilization=0.15, max_model_len=256, - ) - vllm_sampling_params = SamplingParams(logprobs=1) - # should pass - runner.generate(["Hello world"], sampling_params=vllm_sampling_params) + ) as runner: + vllm_sampling_params = SamplingParams(logprobs=1) + # should pass + runner.generate(["Hello world"], sampling_params=vllm_sampling_params) - bad_sampling_params = SamplingParams(logprobs=2) - with pytest.raises(ValueError): - runner.generate(["Hello world"], sampling_params=bad_sampling_params) + bad_sampling_params = SamplingParams(logprobs=2) + with pytest.raises(ValueError): + runner.generate(["Hello world"], sampling_params=bad_sampling_params) def test_none_logprobs(vllm_model, example_prompts): @@ -449,33 +448,31 @@ def test_all_logprobs(example_prompts): Args: example_prompts: list of example prompts (test fixture) """ - runner = VllmRunner( + with VllmRunner( "facebook/opt-125m", max_logprobs=-1, enable_prefix_caching=False, - # 2 other llms alive during whole session gpu_memory_utilization=0.15, max_model_len=256, - ) + ) as runner: + sampling_params_logprobs_all = SamplingParams( + max_tokens=5, logprobs=-1, prompt_logprobs=-1 + ) + results_logprobs_all = runner.llm.generate( + example_prompts, sampling_params=sampling_params_logprobs_all + ) + vocab_size = runner.llm.llm_engine.model_config.get_vocab_size() - sampling_params_logprobs_all = SamplingParams( - max_tokens=5, logprobs=-1, prompt_logprobs=-1 - ) - results_logprobs_all = runner.llm.generate( - example_prompts, sampling_params=sampling_params_logprobs_all - ) - vocab_size = runner.llm.llm_engine.model_config.get_vocab_size() - - for i in range(len(results_logprobs_all)): - logprobs = results_logprobs_all[i].outputs[0].logprobs - prompt_logprobs = results_logprobs_all[i].prompt_logprobs - assert logprobs is not None - for logprob in logprobs: - assert len(logprob) == vocab_size - assert prompt_logprobs is not None - assert prompt_logprobs[0] is None - for prompt_logprob in prompt_logprobs[1:]: - assert len(prompt_logprob) == vocab_size + for i in range(len(results_logprobs_all)): + logprobs = results_logprobs_all[i].outputs[0].logprobs + prompt_logprobs = results_logprobs_all[i].prompt_logprobs + assert logprobs is not None + for logprob in logprobs: + assert len(logprob) == vocab_size + assert prompt_logprobs is not None + assert prompt_logprobs[0] is None + for prompt_logprob in prompt_logprobs[1:]: + assert len(prompt_logprob) == vocab_size @pytest.mark.parametrize("logprobs_mode", get_args(LogprobsMode)) @@ -495,24 +492,28 @@ def test_logprobs_mode(logprobs_mode: LogprobsMode): max_model_len=16, logprobs_mode=logprobs_mode, ) - vllm_sampling_params = SamplingParams(logprobs=1) - results = llm.generate(["Hello world"], sampling_params=vllm_sampling_params) + try: + vllm_sampling_params = SamplingParams(logprobs=1) + results = llm.generate(["Hello world"], sampling_params=vllm_sampling_params) - total_token_with_logprobs = 0 - positive_values = 0 - for output in results[0].outputs: - for logprobs in output.logprobs: - for token_id in logprobs: - logprob = logprobs[token_id] - if logprobs_mode in ("raw_logprobs", "processed_logprobs"): - assert logprob.logprob <= 0 - if logprob.logprob > 0: - positive_values = positive_values + 1 - total_token_with_logprobs = total_token_with_logprobs + 1 - assert total_token_with_logprobs >= len(results[0].outputs) - if logprobs_mode in ("raw_logits", "processed_logits"): - assert positive_values > 0 - del llm + total_token_with_logprobs = 0 + positive_values = 0 + for output in results[0].outputs: + for logprobs in output.logprobs: + for token_id in logprobs: + logprob = logprobs[token_id] + if logprobs_mode in ("raw_logprobs", "processed_logprobs"): + assert logprob.logprob <= 0 + if logprob.logprob > 0: + positive_values = positive_values + 1 + total_token_with_logprobs = total_token_with_logprobs + 1 + assert total_token_with_logprobs >= len(results[0].outputs) + if logprobs_mode in ("raw_logits", "processed_logits"): + assert positive_values > 0 + finally: + del llm + torch.cuda.empty_cache() + cleanup_dist_env_and_memory() class TestCorrectDecodedToken: @@ -767,7 +768,7 @@ class TestCorrectDecodedToken: # Simulate cases where individual tokens decode to "�" # but combinations decode correctly if len(ids) == 1: - if ids[0] == 3 or ids[0] == 4 or ids[0] == 8 or ids[0] == 9: + if ids[0] in (3, 4, 8, 9): return "�" elif len(ids) == 2: if ids == [2, 3]: @@ -809,42 +810,41 @@ def test_verify_tokens_integration(): corrects tokens ending with the replacement character "�". Uses facebook/opt-125m which is known to produce these issues. """ - runner = VllmRunner( + with VllmRunner( "facebook/opt-125m", max_logprobs=0, enable_prefix_caching=False, gpu_memory_utilization=0.15, max_model_len=256, - ) + ) as runner: + # Use a prompt that triggers multi-byte UTF-8 issues + # Based on user's example: "In this example," + test_prompts = ["In this example,"] - # Use a prompt that triggers multi-byte UTF-8 issues - # Based on user's example: "In this example," - test_prompts = ["In this example,"] + sampling_params = SamplingParams( + max_tokens=16, + temperature=0, + logprobs=0, + ) - sampling_params = SamplingParams( - max_tokens=16, - temperature=0, - logprobs=0, - ) + results = runner.llm.generate(test_prompts, sampling_params=sampling_params) - results = runner.llm.generate(test_prompts, sampling_params=sampling_params) - - # Verify that decoded tokens don't contain replacement characters - for result in results: - assert result.outputs[0].logprobs is not None - for logprob_dict in result.outputs[0].logprobs: - for token_id, logprob_info in logprob_dict.items(): - decoded_token = logprob_info.decoded_token - # Decoded tokens should not end with replacement character - # They should either be corrected or empty string - assert not decoded_token.endswith("�"), ( - f"Token {token_id} decoded to '{decoded_token}' which " - f"ends with replacement character" - ) - # Decoded tokens should not contain lone replacement characters - assert decoded_token != "�", ( - f"Token {token_id} is a lone replacement character" - ) + # Verify that decoded tokens don't contain replacement characters + for result in results: + assert result.outputs[0].logprobs is not None + for logprob_dict in result.outputs[0].logprobs: + for token_id, logprob_info in logprob_dict.items(): + decoded_token = logprob_info.decoded_token + # Decoded tokens should not end with replacement character + # They should either be corrected or empty string + assert not decoded_token.endswith("�"), ( + f"Token {token_id} decoded to '{decoded_token}' which " + f"ends with replacement character" + ) + # Decoded tokens should not contain lone replacement characters + assert decoded_token != "�", ( + f"Token {token_id} is a lone replacement character" + ) def test_utf8_edge_cases_with_real_model(): @@ -853,45 +853,44 @@ def test_utf8_edge_cases_with_real_model(): Tests prompts that are likely to trigger byte-fallback tokenization and multi-byte UTF-8 splitting. """ - runner = VllmRunner( + with VllmRunner( "facebook/opt-125m", max_logprobs=1, enable_prefix_caching=False, gpu_memory_utilization=0.15, max_model_len=256, - ) + ) as runner: + # Prompts with various multi-byte UTF-8 characters + test_prompts = [ + 'Smart quotes: "Hello"', # Curly quotes + "Em dash — test", # Em dash + "Ellipsis… continues", # Ellipsis + "Chinese: 你好", # Chinese characters + "Emoji: 😀 🎉", # Emojis + 'Mixed: "quoted" — with symbols', # Mixed + ] - # Prompts with various multi-byte UTF-8 characters - test_prompts = [ - 'Smart quotes: "Hello"', # Curly quotes - "Em dash — test", # Em dash - "Ellipsis… continues", # Ellipsis - "Chinese: 你好", # Chinese characters - "Emoji: 😀 🎉", # Emojis - 'Mixed: "quoted" — with symbols', # Mixed - ] + sampling_params = SamplingParams( + max_tokens=10, + temperature=0, + logprobs=1, + ) - sampling_params = SamplingParams( - max_tokens=10, - temperature=0, - logprobs=1, - ) + results = runner.llm.generate(test_prompts, sampling_params=sampling_params) - results = runner.llm.generate(test_prompts, sampling_params=sampling_params) + for i, result in enumerate(results): + prompt = test_prompts[i] + assert result.outputs[0].logprobs is not None - for i, result in enumerate(results): - prompt = test_prompts[i] - assert result.outputs[0].logprobs is not None - - # Check that no decoded tokens end with replacement character - for logprob_dict in result.outputs[0].logprobs: - for token_id, logprob_info in logprob_dict.items(): - decoded_token = logprob_info.decoded_token - assert not decoded_token.endswith("�"), ( - f"Prompt: '{prompt}'\n" - f"Token {token_id} decoded to '{decoded_token}' which " - f"ends with replacement character" - ) + # Check that no decoded tokens end with replacement character + for logprob_dict in result.outputs[0].logprobs: + for token_id, logprob_info in logprob_dict.items(): + decoded_token = logprob_info.decoded_token + assert not decoded_token.endswith("�"), ( + f"Prompt: '{prompt}'\n" + f"Token {token_id} decoded to '{decoded_token}' which " + f"ends with replacement character" + ) def test_correct_decoded_token_preserves_valid_tokens(): @@ -901,36 +900,35 @@ def test_correct_decoded_token_preserves_valid_tokens(): ending with "�", but this test verifies the broader _verify_tokens logic doesn't affect valid tokens. """ - runner = VllmRunner( + with VllmRunner( "facebook/opt-125m", max_logprobs=2, enable_prefix_caching=False, gpu_memory_utilization=0.15, max_model_len=256, - ) + ) as runner: + # Simple prompt with standard ASCII characters + test_prompts = ["Hello world, this is a test."] - # Simple prompt with standard ASCII characters - test_prompts = ["Hello world, this is a test."] + sampling_params = SamplingParams( + max_tokens=10, + temperature=0, + logprobs=2, + ) - sampling_params = SamplingParams( - max_tokens=10, - temperature=0, - logprobs=2, - ) + results = runner.llm.generate(test_prompts, sampling_params=sampling_params) - results = runner.llm.generate(test_prompts, sampling_params=sampling_params) + for result in results: + assert result.outputs[0].logprobs is not None - for result in results: - assert result.outputs[0].logprobs is not None - - # All decoded tokens should be valid strings - for logprob_dict in result.outputs[0].logprobs: - for token_id, logprob_info in logprob_dict.items(): - decoded_token = logprob_info.decoded_token - # Valid tokens should be non-empty strings (or empty if corrected) - assert isinstance(decoded_token, str) - # Should not contain replacement character - assert "�" not in decoded_token + # All decoded tokens should be valid strings + for logprob_dict in result.outputs[0].logprobs: + for token_id, logprob_info in logprob_dict.items(): + decoded_token = logprob_info.decoded_token + # Valid tokens should be non-empty strings (or empty if corrected) + assert isinstance(decoded_token, str) + # Should not contain replacement character + assert "�" not in decoded_token @pytest.mark.parametrize("logprobs_mode", get_args(LogprobsMode)) @@ -985,16 +983,33 @@ def test_correct_decoded_token_preserves_valid_tokens(): def test_spec_decode_logprobs( logprobs_mode: LogprobsMode, model_setup: tuple[str, str, dict, int], + monkeypatch, ): """Spec decode logprobs should match those of the base model. + Runs the base model and spec decode model sequentially, ensuring + only one LLM instance is alive at a time to avoid GPU memory + contention. Both use identical chunked prefill settings and eager + mode to control for infrastructure differences. + Args: logprobs_mode: logprobs mode. model_setup: Tuple of (method, base model name, speculative_config dict, top_logprobs). + monkeypatch: pytest fixture for setting env vars. """ from vllm import LLM + # The ROCm skinny GEMM kernels (gemm_kernels.cu) are + # non-deterministic across LLM instantiations due to persistent + # workgroup scheduling and wave-level shuffle reductions, which + # causes logprob differences that get misattributed to spec decode. + # Disable them so this test isolates spec decode correctness only. + # TODO(akaratza): Remove this workaround once the follow-up to + # https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975 + # lands with a determinism fix for wvSplitK kernels. + monkeypatch.setenv("VLLM_ROCM_USE_SKINNY_GEMM", "0") + method, model_name, spec_config, top_logprobs = model_setup prompt = "Hello world " * 50 @@ -1068,8 +1083,17 @@ def test_spec_decode_logprobs( for ref_logprob, spec_logprob in zip(ref_logprobs, spec_logprobs): assert math.isclose( ref_logprob.logprob, spec_logprob.logprob, rel_tol=5e-2, abs_tol=1e-1 + ), ( + f"Logprob mismatch: ref={ref_logprob.logprob} " + f"spec={spec_logprob.logprob} " + f"diff={abs(ref_logprob.logprob - spec_logprob.logprob)} " + f"(token={ref_logprob.decoded_token!r})" + ) + assert ref_logprob.rank == spec_logprob.rank, ( + f"Rank mismatch: ref={ref_logprob.rank} " + f"spec={spec_logprob.rank} " + f"(token={ref_logprob.decoded_token!r})" ) - assert ref_logprob.rank == spec_logprob.rank assert ref_logprob.decoded_token == spec_logprob.decoded_token diff --git a/tests/v1/spec_decode/test_tree_attention.py b/tests/v1/spec_decode/test_tree_attention.py index bd700554061..52bc722cfcb 100644 --- a/tests/v1/spec_decode/test_tree_attention.py +++ b/tests/v1/spec_decode/test_tree_attention.py @@ -13,6 +13,7 @@ from tests.v1.attention.utils import ( try_get_attention_backend, ) from vllm.config import ParallelConfig, SpeculativeConfig +from vllm.platforms import current_platform from vllm.v1.attention.backend import CommonAttentionMetadata from vllm.v1.attention.backends.fa_utils import is_flash_attn_varlen_func_available from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -23,11 +24,156 @@ if not is_flash_attn_varlen_func_available(): allow_module_level=True, ) +# --------------------------------------------------------------------------- # +# KV cache layout adaptation +# --------------------------------------------------------------------------- # +# Two KV cache layouts exist across backends: +# +# Flash layout: (2, num_blocks, block_size, num_kv_heads, head_size) +# - dim 0 separates key (index 0) and value (index 1) +# - Used by: FLASH_ATTN, TREE_ATTN, ROCM_AITER_FA, ROCM_ATTN +# +# Block layout: (num_blocks, 2, block_size, num_kv_heads, head_size) +# - dim 1 separates key (index 0) and value (index 1) +# - Used by: TRITON_ATTN +# +# The test creates KV caches in flash layout (the canonical format used by +# tree attention). When a reference backend needs block layout we transpose +# dims 0 and 1. +# +# Note: ROCM_ATTN uses flash layout for storage but its forward path calls +# PagedAttention.split_kv_cache which reinterprets the raw memory as paged +# layout (num_blocks, num_kv_heads, head_size//x, block_size, x). This is +# a view-level incompatibility, not a transpose - see the TODO in +# _get_available_reference_backends for details. +# +# TODO: Replace this mapping with a `KV_CACHE_LAYOUT` class attribute on each +# AttentionImpl so the layout is self-documented by the backend itself, e.g.: +# class TritonAttentionImpl(AttentionImpl): +# KV_CACHE_LAYOUT = "block" +# --------------------------------------------------------------------------- # + +_BLOCK_KV_LAYOUT_BACKENDS = frozenset( + { + AttentionBackendEnum.TRITON_ATTN, + } +) + +# Backends whose do_kv_cache_update requires engine-level state (e.g. +# ForwardContext) that is not available in this test harness, but whose +# KV cache is flash layout and can be written with reshape_and_cache_flash. +# When a backend is listed here, forward_attention() bypasses +# do_kv_cache_update and writes directly to the cache. +_NEEDS_DIRECT_CACHE_UPDATE = frozenset( + { + AttentionBackendEnum.ROCM_AITER_FA, + } +) + +# Backends with known test-harness incompatibilities - see the TODOs +# inside _get_available_reference_backends for details. +_INCOMPATIBLE_REFERENCE_BACKENDS = frozenset( + { + AttentionBackendEnum.ROCM_AITER_FA, + AttentionBackendEnum.ROCM_ATTN, + } +) + + +def _adapt_kv_cache_for_backend( + kv_cache: torch.Tensor, + backend: AttentionBackendEnum, +) -> torch.Tensor: + """Convert kv_cache from flash layout ``(2, num_blocks, ...)`` to block + layout ``(num_blocks, 2, ...)`` if the backend requires it. Returns the + original tensor unchanged when no conversion is needed.""" + if backend in _BLOCK_KV_LAYOUT_BACKENDS: + return kv_cache.transpose(0, 1).contiguous() + return kv_cache + + +def _get_platform_default_backend() -> AttentionBackendEnum: + """Ask the platform what backend it would auto-select at runtime.""" + from vllm.v1.attention.selector import AttentionSelectorConfig + + config = AttentionSelectorConfig( + block_size=32, + kv_cache_dtype="auto", + use_mla=False, + use_sparse=False, + head_size=128, + dtype=torch.bfloat16, + ) + backend_path = current_platform.get_attn_backend_cls( + selected_backend=None, + attn_selector_config=config, + ) + for backend in AttentionBackendEnum: + try: + if backend.get_path() == backend_path: + return backend + except ValueError: + continue + raise RuntimeError( + f"Platform returned backend path '{backend_path}' " + f"that doesn't match any AttentionBackendEnum member." + ) + + +def _get_available_reference_backends() -> list[AttentionBackendEnum]: + """Collect all reference backends the current platform can run. + + On CUDA this is just FLASH_ATTN. On ROCm this includes the platform + default plus every backend the hardware supports, so the test validates + tree attention against all of them. + """ + if current_platform.is_rocm(): + backends: list[AttentionBackendEnum] = [] + + # 1. Whatever the platform would auto-select at runtime. + default_backend = _get_platform_default_backend() + if default_backend not in _INCOMPATIBLE_REFERENCE_BACKENDS: + backends.append(default_backend) + + # 2. TRITON_ATTN - always available on ROCm. + if AttentionBackendEnum.TRITON_ATTN not in backends: + backends.append(AttentionBackendEnum.TRITON_ATTN) + + # TODO: Enable ROCM_ATTN. Its forward path uses + # PagedAttention.split_kv_cache which reinterprets the raw + # cache memory as paged layout: + # key: (num_blocks, num_kv_heads, head_size//x, block_size, x) + # value: (num_blocks, num_kv_heads, head_size, block_size) + # Tree attention writes prefix data in NHD flash layout, so the + # same bytes produce completely different values when read in + # paged format. Supporting ROCM_ATTN would require writing + # prefix data via PagedAttention.write_to_paged_cache into a + # separate paged-format KV cache. + + # TODO: Enable ROCM_AITER_FA. Its metadata builder reads head + # counts from the model config at construction time and + # allocates extend_workspace with those dimensions. The test + # uses independent head count parameters (num_heads=2/4, + # num_kv_heads=2) that don't match the model config + # (Llama-3-8B: 32 q heads, 8 kv heads), causing a head count + # mismatch in flash_attn_varlen_func during extend_forward. + # Fixing this requires either matching test head counts to the + # model config or decoupling the builder from model config + # head geometry. The direct cache update path + # (_NEEDS_DIRECT_CACHE_UPDATE) is already in place for when + # this is resolved. + + return backends + + # CUDA: flash attention. + return [AttentionBackendEnum.FLASH_ATTN] + class MockAttentionLayer(torch.nn.Module): _q_scale = torch.tensor(1.0, dtype=torch.float32, device="cuda") _k_scale = torch.tensor(1.0, dtype=torch.float32, device="cuda") _v_scale = torch.tensor(1.0, dtype=torch.float32, device="cuda") + layer_name = "mock_layer" def __init__(self): super().__init__() @@ -48,6 +194,13 @@ def forward_attention( spec_token_tree: str | None = None, num_spec_tokens: int = 0, ) -> torch.Tensor: + """Run a single attention forward pass through the given backend. + + ``kv_cache`` is expected in **flash layout** + ``(2, num_blocks, block_size, num_kv_heads, head_size)``. + It is automatically converted when the target backend needs a + different layout. + """ batch_size, q_len, num_heads, dim_per_head = q.shape num_kv_heads = k.shape[-2] # Initialize the query and KV sequence lengths. @@ -116,31 +269,58 @@ def forward_attention( kv_cache_dtype="auto", ) + # Adapt KV cache layout for this backend. + adapted_kv_cache = _adapt_kv_cache_for_backend(kv_cache, backend) + # Run forward pass and return output. query = q.view(-1, num_heads, dim_per_head) key = k.view(-1, num_kv_heads, dim_per_head) value = v.view(-1, num_kv_heads, dim_per_head) output = torch.empty_like(query) if not try_backend_includes_kv_cache_update(backend): - instance.do_kv_cache_update( - layer=layer, - key=key, - value=value, - kv_cache=kv_cache, - slot_mapping=attn_metadata.slot_mapping, - ) + if backend in _NEEDS_DIRECT_CACHE_UPDATE: + # This backend's do_kv_cache_update requires engine-level + # ForwardContext that isn't available in this test harness. + # Write directly using reshape_and_cache_flash since the + # KV cache layout is identical (flash layout, unbind on dim 0). + key_cache, value_cache = adapted_kv_cache.unbind(0) + torch.ops._C_cache_ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + attn_metadata.slot_mapping, + "auto", + layer._k_scale, + layer._v_scale, + ) + else: + instance.do_kv_cache_update( + layer=layer, + key=key, + value=value, + kv_cache=adapted_kv_cache, + slot_mapping=attn_metadata.slot_mapping, + ) return instance.forward( layer=layer, query=query, key=key, value=value, - kv_cache=kv_cache.clone(), + kv_cache=adapted_kv_cache.clone(), attn_metadata=attn_metadata, output=output, ) -def test_tree_attn_correctness() -> None: +@pytest.mark.parametrize( + "reference_backend", + _get_available_reference_backends(), + ids=lambda b: b.name, +) +def test_tree_attn_correctness( + reference_backend: AttentionBackendEnum, +) -> None: torch.manual_seed(42) torch.cuda.manual_seed_all(42) @@ -205,7 +385,9 @@ def test_tree_attn_correctness() -> None: dtype=torch.bfloat16, ) - # Set up the block table and KV cache for paged KV. + # KV cache in flash layout - the canonical format for + # tree attention. forward_attention() handles conversion + # when needed. assert max_sequence_length % block_size == 0 max_blocks_per_batch = max_sequence_length // block_size kv_cache = torch.randn( @@ -263,9 +445,7 @@ def test_tree_attn_correctness() -> None: num_spec_tokens=tree_size_q - 1, ).view(batch_size, -1, num_heads, dim_per_head) - # Verify that the chain attention output for each - # branch of the tree (computed using FA3) matches - # the tree attention output. + # Verify each branch against the reference backend. for q_index in range(tree_size_q): # Get the q, k, and v for the branch. branch_mask = tree_attn_mask[q_index, :] @@ -286,8 +466,8 @@ def test_tree_attn_correctness() -> None: branch_positions, block_table, block_size ) - # Compute flash attention for the branch. - flash_attn_output = forward_attention( + # Reference attention for this branch. + ref_output = forward_attention( q=q_branch, k=k_branch, v=v_branch, @@ -295,16 +475,17 @@ def test_tree_attn_correctness() -> None: block_table=block_table, slot_mapping=branch_slot_mapping, seqlen_k=sequence_position + q_len, - backend=AttentionBackendEnum.FLASH_ATTN, + backend=reference_backend, ).view(batch_size, -1, num_heads, dim_per_head) # Compare the outputs. assert torch.allclose( tree_attn_output[:, branch_indices], - flash_attn_output, + ref_output, atol=7.81e-3, ), ( f"outputs are not close for " + f"reference_backend: {reference_backend.name}, " f"batch_size: {batch_size}, " f"num_heads: {num_heads}, " f"sequence_position: {sequence_position}, " From 2aab2bb54366c5a26add1b07f107b86f7fe28ff5 Mon Sep 17 00:00:00 2001 From: jennyyyyzhen <47012288+jennyyyyzhen@users.noreply.github.com> Date: Fri, 20 Feb 2026 20:32:05 -0800 Subject: [PATCH 54/74] [ROCM] Optimize ROCM_AITER_FA spec decode eagle performance (#34541) Signed-off-by: jennyyyyzhen --- vllm/v1/attention/backends/rocm_aiter_fa.py | 52 ++++++++++++++++++++- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index 2ea3c346f5a..0c1e1b5e00c 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -396,8 +396,7 @@ class AiterFlashAttentionMetadata: class AiterFlashAttentionMetadataBuilder( AttentionMetadataBuilder[AiterFlashAttentionMetadata] ): - _cudagraph_support = AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE - reorder_batch_threshold: int = 1 + _cudagraph_support = AttentionCGSupport.UNIFORM_BATCH def __init__( self, @@ -422,6 +421,7 @@ class AiterFlashAttentionMetadataBuilder( # populated on first build() call. self.aot_sliding_window: tuple[int, int] | None = None self.total_tokens: int = 0 + self._init_reorder_batch_threshold(1, supports_spec_as_decode=True) sliding_window_configs: set[tuple[int, int] | None] = set() layers = get_layers_from_vllm_config(self.vllm_config, Attention) @@ -466,6 +466,7 @@ class AiterFlashAttentionMetadataBuilder( common_attn_metadata: CommonAttentionMetadata, fast_build: bool = False, ) -> "AiterFlashAttentionMetadata": + assert self.reorder_batch_threshold is not None split_ret = split_decodes_prefills_and_extends( common_attn_metadata, decode_threshold=self.reorder_batch_threshold, @@ -677,6 +678,53 @@ class AiterFlashAttentionMetadataBuilder( ) return attn_metadata + def build_for_drafting( + self, + common_attn_metadata: CommonAttentionMetadata, + draft_index: int, + ) -> AiterFlashAttentionMetadata: + """ + Build attention metadata for draft model without CPU-GPU sync. + + During EAGLE drafting all requests are uniform decodes, so we can + skip split_decodes_prefills_and_extends() and avoid all .cpu() / + .item() calls that would otherwise break CUDA graph capture. + """ + num_reqs = common_attn_metadata.num_reqs + num_tokens = common_attn_metadata.num_actual_tokens + + decode_metadata = AiterFlashAttentionDecodeMetadata( + max_query_len=common_attn_metadata.max_query_len, + min_query_len=common_attn_metadata.max_query_len, # uniform batch + max_seq_len=common_attn_metadata.max_seq_len, + query_start_loc=common_attn_metadata.query_start_loc, + ) + + return AiterFlashAttentionMetadata( + num_actual_tokens=num_tokens, + num_actual_kv_tokens=0, # not used in unified_attention path + max_query_len=common_attn_metadata.max_query_len, + query_start_loc=common_attn_metadata.query_start_loc, + max_seq_len=common_attn_metadata.max_seq_len, + seq_lens=common_attn_metadata.seq_lens, + block_table=common_attn_metadata.block_table_tensor, + slot_mapping=common_attn_metadata.slot_mapping, + num_decodes=num_reqs, + num_decode_tokens=num_tokens, + num_prefills=0, + num_prefill_tokens=0, + num_extends=0, + num_extend_tokens=0, + decode_metadata=decode_metadata, + prefill_metadata=None, + extend_metadata=None, + use_cascade=False, + common_prefix_len=0, + total_tokens=self.total_tokens, + k_scale=self.scale, + v_scale=self.scale, + ) + def use_cascade_attention(self, *args, **kwargs) -> bool: return False From 8dc8a99b56e1a8427c83217a37595d0cd12b1ff2 Mon Sep 17 00:00:00 2001 From: BADAOUI Abdennacer <106801897+Abdennacer-Badaoui@users.noreply.github.com> Date: Sat, 21 Feb 2026 09:34:55 +0100 Subject: [PATCH 55/74] [ROCm] Enable bitsandbytes quantization support on ROCm (#34688) Signed-off-by: badaoui --- docs/features/quantization/bnb.md | 2 +- requirements/nightly_torch_test.txt | 2 +- requirements/rocm-test.txt | 2 + requirements/test.in | 2 +- requirements/test.txt | 3 +- tests/models/test_transformers.py | 9 +--- .../layers/quantization/bitsandbytes.py | 49 ++++++++----------- vllm/platforms/rocm.py | 4 +- 8 files changed, 29 insertions(+), 44 deletions(-) diff --git a/docs/features/quantization/bnb.md b/docs/features/quantization/bnb.md index 2348c7739c0..53419e0672b 100644 --- a/docs/features/quantization/bnb.md +++ b/docs/features/quantization/bnb.md @@ -7,7 +7,7 @@ Compared to other quantization methods, BitsAndBytes eliminates the need for cal Below are the steps to utilize BitsAndBytes with vLLM. ```bash -pip install bitsandbytes>=0.46.1 +pip install bitsandbytes>=0.49.2 ``` vLLM reads the model's config file and supports both in-flight quantization and pre-quantized checkpoint. diff --git a/requirements/nightly_torch_test.txt b/requirements/nightly_torch_test.txt index c9211b913a2..9a0bc4b20df 100644 --- a/requirements/nightly_torch_test.txt +++ b/requirements/nightly_torch_test.txt @@ -33,7 +33,7 @@ transformers==4.57.5 tokenizers==0.22.0 schemathesis>=3.39.15 # Required for openai schema test. # quantization -bitsandbytes>=0.46.1 +bitsandbytes>=0.49.2 buildkite-test-collector==0.1.9 diff --git a/requirements/rocm-test.txt b/requirements/rocm-test.txt index af77039165a..1983392a124 100644 --- a/requirements/rocm-test.txt +++ b/requirements/rocm-test.txt @@ -102,3 +102,5 @@ terratorch==1.2.2 segmentation-models-pytorch==0.5.0 # Required for Prithvi tests imagehash==4.3.2 +# Required for bitsandbytes quantization test +bitsandbytes==0.49.2 diff --git a/requirements/test.in b/requirements/test.in index 5faf1c456b8..92d8fec4bfb 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -41,7 +41,7 @@ transformers==4.57.5 tokenizers==0.22.0 schemathesis>=3.39.15 # Required for openai schema test. # quantization -bitsandbytes==0.46.1 +bitsandbytes==0.49.2 buildkite-test-collector==0.1.9 diff --git a/requirements/test.txt b/requirements/test.txt index c18d216370c..791bdc005cb 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -66,7 +66,7 @@ backoff==2.2.1 # via # -r requirements/test.in # schemathesis -bitsandbytes==0.46.1 +bitsandbytes==0.49.2 # via # -r requirements/test.in # lightning @@ -653,6 +653,7 @@ orjson==3.11.5 packaging==24.2 # via # accelerate + # bitsandbytes # black # datamodel-code-generator # datasets diff --git a/tests/models/test_transformers.py b/tests/models/test_transformers.py index 15ebb5f4a38..eadc3534c37 100644 --- a/tests/models/test_transformers.py +++ b/tests/models/test_transformers.py @@ -6,8 +6,6 @@ from typing import Any import pytest -from vllm.platforms import current_platform - from ..conftest import HfRunner, VllmRunner from ..utils import multi_gpu_test, prep_prompts from .registry import HF_EXAMPLE_MODELS @@ -131,6 +129,7 @@ def test_distributed( "quantization": "bitsandbytes", }, ), + ("unsloth/tinyllama-bnb-4bit", {}), ], ) @pytest.mark.parametrize("max_tokens", [32]) @@ -143,12 +142,6 @@ def test_quantization( max_tokens: int, num_logprobs: int, ) -> None: - if ( - current_platform.is_rocm() - and quantization_kwargs.get("quantization", "") == "bitsandbytes" - ): - pytest.skip("bitsandbytes quantization is currently not supported in rocm.") - with vllm_runner( model, model_impl="auto", diff --git a/vllm/model_executor/layers/quantization/bitsandbytes.py b/vllm/model_executor/layers/quantization/bitsandbytes.py index 0d6d0bac9c8..716a20090f6 100644 --- a/vllm/model_executor/layers/quantization/bitsandbytes.py +++ b/vllm/model_executor/layers/quantization/bitsandbytes.py @@ -28,6 +28,24 @@ from vllm.platforms import current_platform from vllm.utils.torch_utils import direct_register_custom_op +def _check_bitsandbytes_version(): + min_version = "0.49.2" if current_platform.is_rocm() else "0.48.1" + try: + import bitsandbytes + + if version.parse(bitsandbytes.__version__) < version.parse(min_version): + raise ImportError( + "bitsandbytes version is wrong. Please " + f"install bitsandbytes>={min_version}." + ) + except ImportError as err: + raise ImportError( + f"Please install bitsandbytes>={min_version} via " + f"`pip install bitsandbytes>={min_version}` to use " + "bitsandbytes quantizer." + ) from err + + class BitsAndBytesConfig(QuantizationConfig): """Config class for BitsAndBytes Quantization. @@ -183,21 +201,7 @@ class BitsAndBytesLinearMethod(LinearMethodBase): """ def __init__(self, quant_config: BitsAndBytesConfig): - try: - import bitsandbytes - - if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"): - raise ImportError( - "bitsandbytes version is wrong. Please " - "install bitsandbytes>=0.46.1." - ) - except ImportError as err: - raise ImportError( - "Please install bitsandbytes>=0.46.1 via " - "`pip install bitsandbytes>=0.46.1` to use " - "bitsandbytes quantizer." - ) from err - + _check_bitsandbytes_version() self.quant_config = quant_config def create_weights( @@ -442,20 +446,7 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase): moe: FusedMoEConfig, ): super().__init__(moe) - try: - import bitsandbytes - - if version.parse(bitsandbytes.__version__) < version.parse("0.46.1"): - raise ImportError( - "bitsandbytes version is wrong. Please " - "install bitsandbytes>=0.46.1." - ) - except ImportError as err: - raise ImportError( - "Please install bitsandbytes>=0.46.1 via " - "`pip install bitsandbytes>=0.46.1` to use " - "bitsandbytes quantizer." - ) from err + _check_bitsandbytes_version() self.quant_config = quant_config def create_weights( diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 2fedd7c6791..a8a1d59f1bf 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -244,10 +244,8 @@ class RocmPlatform(Platform): "mxfp4", "petit_nvfp4", "torchao", + "bitsandbytes", ] - # bitsandbytes not supported on gfx9 (warp size 64 limitation) - if not on_gfx9(): - supported_quantization += ["bitsandbytes"] @classmethod def import_kernels(cls) -> None: From ab6f3487a6146b325cb836711e34f40f341278e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Lucchesi?= Date: Sat, 21 Feb 2026 10:34:57 +0100 Subject: [PATCH 56/74] [PD] Change kv_load_failure_policy Default from "recompute" to "fail" (#34896) Signed-off-by: NickLucche Co-authored-by: Cyrus Leung --- docs/features/nixl_connector_usage.md | 4 ++-- .../kv_load_failure_recovery/decode_example.py | 1 + tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py | 2 +- tests/v1/kv_connector/unit/utils.py | 4 +++- vllm/config/kv_transfer.py | 6 +++--- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/docs/features/nixl_connector_usage.md b/docs/features/nixl_connector_usage.md index 3fc735efa68..a9039f0daf8 100644 --- a/docs/features/nixl_connector_usage.md +++ b/docs/features/nixl_connector_usage.md @@ -197,8 +197,8 @@ For multi-host DP deployment, only need to provide the host/port of the head ins The `kv_load_failure_policy` setting controls how the system handles failures when the decoder instance loads KV cache blocks from the prefiller instance: -- **fail** (recommended): Immediately fail the request with an error when KV load fails. This prevents performance degradation by avoiding recomputation of prefill work on the decode instance. -- **recompute** (default): Recompute failed blocks locally on the decode instance. This may cause performance _jitter_ on decode instances as the scheduled prefill will delay and interfere with other decodes. Furthermore, decode instances are typically configured with low-latency optimizations. +- **fail** (default): Immediately fail the request with an error when KV load fails. This prevents performance degradation by avoiding recomputation of prefill work on the decode instance. +- **recompute**: Recompute failed blocks locally on the decode instance. This may cause performance _jitter_ on decode instances as the scheduled prefill will delay and interfere with other decodes. Furthermore, decode instances are typically configured with low-latency optimizations. !!! warning Using `kv_load_failure_policy="recompute"` can lead to performance degradation in production deployments. When KV loads fail, the decode instance will execute prefill work with decode-optimized configurations, which is inefficient and defeats the purpose of disaggregated prefilling. This also increases tail latency for other ongoing decode requests. diff --git a/examples/offline_inference/kv_load_failure_recovery/decode_example.py b/examples/offline_inference/kv_load_failure_recovery/decode_example.py index d0df54167ae..db9c5a85f7f 100644 --- a/examples/offline_inference/kv_load_failure_recovery/decode_example.py +++ b/examples/offline_inference/kv_load_failure_recovery/decode_example.py @@ -42,6 +42,7 @@ def main(): "async_load": args.async_load, }, kv_connector_module_path="load_recovery_example_connector", + kv_load_failure_policy="recompute", ) out_file = ( "async_decode_recovered_output.txt" diff --git a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py index 6b7b2226e75..364eabb96a3 100644 --- a/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py +++ b/tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py @@ -30,7 +30,7 @@ def _make_get_num_new_matched_tokens( @pytest.fixture def scheduler(): - vllm_config = create_vllm_config() + vllm_config = create_vllm_config(kv_load_failure_policy="recompute") return create_scheduler(vllm_config) diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index d843bd6ff5b..7539da3e93f 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -5,7 +5,7 @@ from collections import defaultdict from collections.abc import Callable from dataclasses import dataclass from itertools import chain, count -from typing import Any +from typing import Any, Literal import torch @@ -96,6 +96,7 @@ def create_vllm_config( cache_dtype: str = "auto", hf_overrides: dict[str, Any] | None = None, attention_backend: str | None = None, + kv_load_failure_policy: Literal["recompute", "fail"] = "fail", ) -> VllmConfig: """Initialize VllmConfig For Testing.""" model_config = ModelConfig( @@ -125,6 +126,7 @@ def create_vllm_config( kv_role="kv_both", enable_permute_local_kv=enable_permute_local_kv, kv_connector_extra_config=kv_connector_extra_config or {}, + kv_load_failure_policy=kv_load_failure_policy, ) attention_config = AttentionConfig(backend=attention_backend) return VllmConfig( diff --git a/vllm/config/kv_transfer.py b/vllm/config/kv_transfer.py index fe3b218fbe9..eb6116d0c03 100644 --- a/vllm/config/kv_transfer.py +++ b/vllm/config/kv_transfer.py @@ -61,10 +61,10 @@ class KVTransferConfig: enable_permute_local_kv: bool = False """Experiment feature flag to enable HND to NHD KV Transfer""" - kv_load_failure_policy: Literal["recompute", "fail"] = "recompute" + kv_load_failure_policy: Literal["recompute", "fail"] = "fail" """Policy for handling KV cache load failures. - 'recompute': reschedule the request to recompute failed blocks (default) - 'fail': immediately fail the request with an error finish reason""" + 'recompute': reschedule the request to recompute failed blocks + 'fail': immediately fail the request with an error finish reason (default)""" def compute_hash(self) -> str: """ From 820d7815ebd5e88118e5be02870af9ce49a314b1 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Sat, 21 Feb 2026 01:38:28 -0800 Subject: [PATCH 57/74] [Core] Minor structured-output related scheduler optimization (#34765) Signed-off-by: Nick Hill Co-authored-by: Cyrus Leung --- vllm/v1/core/sched/scheduler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index a4b43a9b0b5..25f84802977 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -945,7 +945,7 @@ class Scheduler(SchedulerInterface): request.num_tokens + request.num_output_placeholders ) scheduler_output.has_structured_output_requests |= ( - request.use_structured_output + request.use_structured_output and not request.is_prefill_chunk ) # NOTE: _free_encoder_inputs relies on num_computed_tokens, which @@ -1232,14 +1232,14 @@ class Scheduler(SchedulerInterface): ) -> GrammarOutput | None: # Collect list of scheduled request ids that use structured output. # The corresponding rows of the bitmask will be in this order. - # PERF: in case of chunked prefill, - # request might not include any new tokens. - # Therefore, we might introduce some additional - # cycle to fill in the bitmask, which could be a big no-op. + if not scheduler_output.has_structured_output_requests: + return None + structured_output_request_ids = [ req_id for req_id in scheduler_output.num_scheduled_tokens - if (req := self.requests.get(req_id)) and req.use_structured_output + if (req := self.requests.get(req_id)) + and (req.use_structured_output and not req.is_prefill_chunk) ] if not structured_output_request_ids: return None From bebfe55b1c17c2e0fedb1b402df1dddfc1a04684 Mon Sep 17 00:00:00 2001 From: petrpechman <41995595+petrpechman@users.noreply.github.com> Date: Sat, 21 Feb 2026 10:57:53 +0100 Subject: [PATCH 58/74] [Doc] Fix example of eagle3 (#34960) Signed-off-by: Petr Pechman Co-authored-by: Petr Pechman --- docs/features/speculative_decoding/eagle.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/speculative_decoding/eagle.md b/docs/features/speculative_decoding/eagle.md index 7063e3f21ef..3e0f3add416 100644 --- a/docs/features/speculative_decoding/eagle.md +++ b/docs/features/speculative_decoding/eagle.md @@ -44,7 +44,7 @@ llm = LLM( "model": "RedHatAI/Llama-3.1-8B-Instruct-speculator.eagle3", "draft_tensor_parallel_size": 2, "num_speculative_tokens": 2, - "method": "eagle", + "method": "eagle3", }, ) From f74f1572ca3a0973d8db2187f0064bfecb6d5df2 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 21 Feb 2026 18:31:58 +0800 Subject: [PATCH 59/74] [Benchmark] Improve benchmarks (#35012) Signed-off-by: DarkLight1337 --- vllm/benchmarks/datasets.py | 26 ++++++++++++++++++-------- vllm/benchmarks/sweep/plot.py | 10 ++++++++-- vllm/benchmarks/sweep/plot_pareto.py | 12 +++++++++--- vllm/benchmarks/sweep/serve_sla.py | 3 +++ 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/vllm/benchmarks/datasets.py b/vllm/benchmarks/datasets.py index 36573a040ff..a8b6b216153 100644 --- a/vllm/benchmarks/datasets.py +++ b/vllm/benchmarks/datasets.py @@ -2627,22 +2627,26 @@ class VisionArenaDataset(HuggingFaceDataset): no_oversample: bool = False, **kwargs, ) -> list: + parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.hf_name) + if parser_fn is None: + raise ValueError(f"Unsupported dataset path: {self.hf_name}") + output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN + sampled_requests = [] for i, item in enumerate(self.data): if len(sampled_requests) >= num_requests: break - parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.hf_name) - if parser_fn is None: - raise ValueError(f"Unsupported dataset path: {self.hf_name}") + prompt = parser_fn(item) mm_content = process_image(item["images"][0]) - prompt_len = len(tokenizer(prompt).input_ids) + prompt_len = len(tokenizer.encode(prompt)) if enable_multimodal_chat: # Note: when chat is enabled the request prompt_len is no longer # accurate and we will be using request output to count the # actual prompt len prompt = self.apply_multimodal_chat_transformation(prompt, mm_content) + sampled_requests.append( SampleRequest( prompt=prompt, @@ -2652,6 +2656,7 @@ class VisionArenaDataset(HuggingFaceDataset): request_id=request_id_prefix + str(i), ) ) + self.maybe_oversample_requests( sampled_requests, num_requests, request_id_prefix, no_oversample ) @@ -2681,22 +2686,26 @@ class MMVUDataset(HuggingFaceDataset): no_oversample: bool = False, **kwargs, ) -> list: + parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.hf_name) + if parser_fn is None: + raise ValueError(f"Unsupported dataset path: {self.hf_name}") + output_len = output_len if output_len is not None else self.DEFAULT_OUTPUT_LEN + sampled_requests = [] for i, item in enumerate(self.data): if len(sampled_requests) >= num_requests: break - parser_fn = self.SUPPORTED_DATASET_PATHS.get(self.hf_name) - if parser_fn is None: - raise ValueError(f"Unsupported dataset path: {self.hf_name}") + prompt = parser_fn(item) mm_content = process_video(item["video"]) - prompt_len = len(tokenizer(prompt).input_ids) + prompt_len = len(tokenizer.encode(prompt)) if enable_multimodal_chat: # Note: when chat is enabled the request prompt_len is no longer # accurate and we will be using request output to count the # actual prompt len prompt = self.apply_multimodal_chat_transformation(prompt, mm_content) + sampled_requests.append( SampleRequest( prompt=prompt, @@ -2706,6 +2715,7 @@ class MMVUDataset(HuggingFaceDataset): request_id=request_id_prefix + str(i), ) ) + self.maybe_oversample_requests( sampled_requests, num_requests, request_id_prefix, no_oversample ) diff --git a/vllm/benchmarks/sweep/plot.py b/vllm/benchmarks/sweep/plot.py index 163d5179313..376adbb0851 100644 --- a/vllm/benchmarks/sweep/plot.py +++ b/vllm/benchmarks/sweep/plot.py @@ -19,11 +19,17 @@ from .utils import sanitize_filename try: import matplotlib.pyplot as plt - import pandas as pd - import seaborn as sns except ImportError: plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot") + +try: + import pandas as pd +except ImportError: pd = PlaceholderModule("pandas") + +try: + import seaborn as sns +except ImportError: seaborn = PlaceholderModule("seaborn") diff --git a/vllm/benchmarks/sweep/plot_pareto.py b/vllm/benchmarks/sweep/plot_pareto.py index 70472552b5c..3d17e4741c1 100644 --- a/vllm/benchmarks/sweep/plot_pareto.py +++ b/vllm/benchmarks/sweep/plot_pareto.py @@ -16,12 +16,18 @@ from .utils import sanitize_filename try: import matplotlib.pyplot as plt - import pandas as pd - import seaborn as sns except ImportError: plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot") + +try: + import pandas as pd +except ImportError: pd = PlaceholderModule("pandas") - sns = PlaceholderModule("seaborn") + +try: + import seaborn as sns +except ImportError: + seaborn = PlaceholderModule("seaborn") def _first_present(run_data: dict[str, object], keys: list[str]): diff --git a/vllm/benchmarks/sweep/serve_sla.py b/vllm/benchmarks/sweep/serve_sla.py index 26f0d6bf652..3b4d48dd2d8 100644 --- a/vllm/benchmarks/sweep/serve_sla.py +++ b/vllm/benchmarks/sweep/serve_sla.py @@ -202,6 +202,7 @@ def solve_sla( with path.open("rb") as f: past_iter_data = json.load(f) + sla_data.append(past_iter_data) history[past_sla_value] = _compute_margin(sla_comb, past_iter_data) # NOTE: We don't use equality here to be more robust against noisy results @@ -264,6 +265,8 @@ def search_sla( dry_run: bool, ): print("[SLA START]") + print(f"Serve parameters: {serve_comb.as_text() or '(None)'}") + print(f"Bench parameters: {bench_comb.as_text() or '(None)'}") print(f"SLA criteria: {sla_comb.as_text()}") result = solve_sla( From 272b535ab3315a2ed3cd1a5e9803df2b86da4f07 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sat, 21 Feb 2026 04:48:14 -0800 Subject: [PATCH 60/74] [Bugfix] Gate 256-bit instructions to CUDA 12.9+ (#34791) Signed-off-by: Huy Do Co-authored-by: Cyrus Leung --- csrc/activation_kernels.cu | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/csrc/activation_kernels.cu b/csrc/activation_kernels.cu index f1d4c137ccd..99fa42f75e9 100644 --- a/csrc/activation_kernels.cu +++ b/csrc/activation_kernels.cu @@ -14,7 +14,8 @@ struct alignas(32) u32x8_t { }; __device__ __forceinline__ void ld256(u32x8_t& val, const u32x8_t* ptr) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \ + defined(CUDA_VERSION) && CUDA_VERSION >= 12090 asm volatile("ld.global.nc.v8.u32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];\n" : "=r"(val.u0), "=r"(val.u1), "=r"(val.u2), "=r"(val.u3), "=r"(val.u4), "=r"(val.u5), "=r"(val.u6), "=r"(val.u7) @@ -35,7 +36,8 @@ __device__ __forceinline__ void ld256(u32x8_t& val, const u32x8_t* ptr) { } __device__ __forceinline__ void st256(u32x8_t& val, u32x8_t* ptr) { -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000 && \ + defined(CUDA_VERSION) && CUDA_VERSION >= 12090 asm volatile("st.global.v8.u32 [%0], {%1,%2,%3,%4,%5,%6,%7,%8};\n" : : "l"(ptr), "r"(val.u0), "r"(val.u1), "r"(val.u2), "r"(val.u3), From 98b0205c3c934849d96922e162e65f3178e0886b Mon Sep 17 00:00:00 2001 From: Roman <45857014+spacecheck@users.noreply.github.com> Date: Sat, 21 Feb 2026 13:49:41 +0100 Subject: [PATCH 61/74] [Frontend] Add automatic language detection for Whisper transcription (#34342) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: space_check Signed-off-by: Roman <45857014+spacecheck@users.noreply.github.com> Co-authored-by: Nicolò Lucchesi Co-authored-by: Nicolò Lucchesi --- .../test_transcription_validation_whisper.py | 27 +++++++ .../multimodal/generation/test_whisper.py | 41 ++++++++++ .../openai/speech_to_text/speech_to_text.py | 62 ++++++++++++++- vllm/model_executor/models/interfaces.py | 50 ++++++++++++ vllm/model_executor/models/whisper.py | 79 ++++++++++++++++--- 5 files changed, 249 insertions(+), 10 deletions(-) diff --git a/tests/entrypoints/openai/test_transcription_validation_whisper.py b/tests/entrypoints/openai/test_transcription_validation_whisper.py index 545f9a1cc68..2d5468c87c5 100644 --- a/tests/entrypoints/openai/test_transcription_validation_whisper.py +++ b/tests/entrypoints/openai/test_transcription_validation_whisper.py @@ -273,3 +273,30 @@ async def test_audio_with_max_tokens(whisper_client, mary_had_lamb): out_text = out["text"] out_tokens = tok(out_text, add_special_tokens=False)["input_ids"] assert len(out_tokens) < 450 # ~Whisper max output len + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("fixture_name", "expected_lang", "expected_text"), + [ + ("mary_had_lamb", "en", ["Mary had a little lamb"]), + ("foscolo", "it", ["zacinto", "sacre"]), + ], + ids=["english", "italian"], +) +async def test_language_auto_detect( + whisper_client, fixture_name, expected_lang, expected_text, request +): + """Auto-detect language when no language param is provided.""" + audio_file = request.getfixturevalue(fixture_name) + transcription = await whisper_client.audio.transcriptions.create( + model=MODEL_NAME, + file=audio_file, + response_format="verbose_json", + temperature=0.0, + ) + assert transcription.language == expected_lang + text_lower = transcription.text.lower() + assert any(word.lower() in text_lower for word in expected_text), ( + f"Expected {expected_lang} text but got: {transcription.text}" + ) diff --git a/tests/models/multimodal/generation/test_whisper.py b/tests/models/multimodal/generation/test_whisper.py index 150bb0e8ae1..4d58ad0a8f0 100644 --- a/tests/models/multimodal/generation/test_whisper.py +++ b/tests/models/multimodal/generation/test_whisper.py @@ -111,6 +111,47 @@ def check_model_available(model: str) -> None: model_info.check_transformers_version(on_fail="skip") +def test_parse_language_detection_output(): + """Unit test for WhisperForConditionalGeneration.parse_language_detection_output. + + No GPU or model loading required. + """ + from unittest.mock import MagicMock + + from vllm.model_executor.models.whisper import ( + WhisperForConditionalGeneration, + ) + + cls = WhisperForConditionalGeneration + + def make_tokenizer(return_value: str) -> MagicMock: + tok = MagicMock() + tok.decode = MagicMock(return_value=return_value) + return tok + + # English + assert ( + cls.parse_language_detection_output([50259], make_tokenizer("<|en|>")) == "en" + ) + + # German + assert ( + cls.parse_language_detection_output([50261], make_tokenizer("<|de|>")) == "de" + ) + + # Unsupported language code + with pytest.raises(AssertionError): + cls.parse_language_detection_output([99999], make_tokenizer("<|xx|>")) + + # No special token format + with pytest.raises(AssertionError): + cls.parse_language_detection_output([1], make_tokenizer("hello")) + + # Empty token_ids + with pytest.raises((AssertionError, IndexError)): + cls.parse_language_detection_output([], make_tokenizer("anything")) + + @pytest.mark.core_model @pytest.mark.cpu_model @pytest.mark.parametrize("model", ["openai/whisper-large-v3-turbo"]) diff --git a/vllm/entrypoints/openai/speech_to_text/speech_to_text.py b/vllm/entrypoints/openai/speech_to_text/speech_to_text.py index fdc926e9a2a..134a9640a2b 100644 --- a/vllm/entrypoints/openai/speech_to_text/speech_to_text.py +++ b/vllm/entrypoints/openai/speech_to_text/speech_to_text.py @@ -41,7 +41,10 @@ from vllm.exceptions import VLLMValidationError from vllm.inputs import ProcessorInputs from vllm.logger import init_logger from vllm.logprobs import FlatLogprobs, Logprob -from vllm.model_executor.models import SupportsTranscription, supports_transcription +from vllm.model_executor.models import ( + SupportsTranscription, + supports_transcription, +) from vllm.outputs import RequestOutput from vllm.renderers.inputs import DictPrompt, EncoderDecoderDictPrompt from vllm.renderers.inputs.preprocess import parse_enc_dec_prompt, parse_model_prompt @@ -242,10 +245,57 @@ class OpenAISpeechToText(OpenAIServing): model_cls = get_model_cls(self.model_config) return cast(type[SupportsTranscription], model_cls) + async def _detect_language( + self, + audio_chunk: np.ndarray, + request_id: str, + ) -> str: + """Auto-detect the spoken language from an audio chunk. + + Delegates prompt construction and output parsing to the model class + via ``get_language_detection_prompt`` and + ``parse_language_detection_output``. + """ + from vllm.sampling_params import SamplingParams + + prompt = self.model_cls.get_language_detection_prompt( + audio_chunk, + self.asr_config, + ) + allowed_token_ids = self.model_cls.get_language_token_ids( + self.tokenizer, + ) + sampling_params = SamplingParams( + max_tokens=1, + temperature=0.0, + allowed_token_ids=allowed_token_ids, + ) + + result_generator = self.engine_client.generate( + prompt, + sampling_params, + request_id, + ) + + final_output: RequestOutput + async for final_output in result_generator: + if final_output.finished: + break + + token_ids = list(final_output.outputs[0].token_ids) + lang = self.model_cls.parse_language_detection_output( + token_ids, + self.tokenizer, + ) + + logger.info("Auto-detected language: '%s'", lang) + return lang + async def _preprocess_speech_to_text( self, request: SpeechToTextRequest, audio_data: bytes, + request_id: str, ) -> tuple[list[ProcessorInputs], float]: # Validate request language = self.model_cls.validate_language(request.language) @@ -274,6 +324,15 @@ class OpenAISpeechToText(OpenAIServing): and duration > self.asr_config.max_audio_clip_s ) chunks = [y] if not do_split_audio else self._split_audio(y, int(sr)) + + if language is None and getattr( + self.model_cls, "supports_explicit_language_detection", False + ): + language = await self._detect_language( + chunks[0], f"{request_id}-lang_detect" + ) + request.language = language + parsed_prompts: list[DictPrompt] = [] for chunk in chunks: # The model has control over the construction, as long as it @@ -435,6 +494,7 @@ class OpenAISpeechToText(OpenAIServing): engine_prompts, duration_s = await self._preprocess_speech_to_text( request=request, audio_data=audio_data, + request_id=request_id, ) except ValueError as e: diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 672857c2388..81caf27d3a8 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -1111,6 +1111,16 @@ class SupportsTranscription(Protocol): Enables the segment timestamp option for supported models by setting this to `True`. """ + supports_explicit_language_detection: ClassVar[bool] = False + """ + Transcription models that require an explicit language detection step + (e.g. Whisper needs a separate forward pass to predict the language + token) should set this to ``True`` and implement + :meth:`get_language_detection_prompt` and + :meth:`parse_language_detection_output` and + :meth:`get_language_token_ids`. + """ + def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) # language codes in supported_languages @@ -1206,6 +1216,46 @@ class SupportsTranscription(Protocol): """ return text + @classmethod + def get_language_detection_prompt( + cls, + audio: np.ndarray, + stt_config: SpeechToTextConfig, + ) -> PromptType: + """Return a prompt that triggers language detection. + + Only needs to be implemented when + ``supports_explicit_language_detection`` is ``True``. + """ + raise NotImplementedError + + @classmethod + def parse_language_detection_output( + cls, + token_ids: list[int], + tokenizer: object, + ) -> str: + """Parse the detected language from model output token IDs. + + Only needs to be implemented when + ``supports_explicit_language_detection`` is ``True``. + """ + raise NotImplementedError + + @classmethod + def get_language_token_ids( + cls, + tokenizer: object, + ) -> list[int] | None: + """Return token IDs that represent valid language tokens. + + Used to constrain language detection to only produce valid language tokens. + + Only needs to be implemented when + ``supports_explicit_language_detection`` is ``True``. + """ + raise NotImplementedError + @overload def supports_transcription( diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index acc9bcf8fdc..96818e264fb 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -64,7 +64,11 @@ from vllm.v1.attention.backend import ( AttentionType, ) -from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsTranscription +from .interfaces import ( + MultiModalEmbeddings, + SupportsMultiModal, + SupportsTranscription, +) from .utils import ( AutoWeightsLoader, WeightsMapper, @@ -784,7 +788,9 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo dummy_inputs=WhisperDummyInputsBuilder, ) class WhisperForConditionalGeneration( - nn.Module, SupportsTranscription, SupportsMultiModal + nn.Module, + SupportsTranscription, + SupportsMultiModal, ): packed_modules_mapping = { "self_attn.qkv_proj": [ @@ -802,20 +808,18 @@ class WhisperForConditionalGeneration( # Whisper only supports audio-conditioned generation. supports_transcription_only = True supports_segment_timestamp = True + supports_explicit_language_detection = True supported_languages = ISO639_1_SUPPORTED_LANGS @classmethod def validate_language(cls, language: str | None) -> str | None: if language is None: - # TODO language should be optional and can be guessed. - # For now we default to en. See - # https://github.com/huggingface/transformers/blob/main/src/transformers/models/whisper/generation_whisper.py#L1520 - logger.warning( - "Defaulting to language='en'. If you wish to transcribe " - "audio in a different language, pass the `language` field " + logger.debug( + "No language specified. Language will be auto-detected " + "from audio. To skip detection, pass the `language` field " "in the TranscriptionRequest." ) - language = "en" + return None return super().validate_language(language) @classmethod @@ -846,6 +850,63 @@ class WhisperForConditionalGeneration( decoder_prompt=TextPrompt(prompt=decoder_text), ) + @classmethod + def get_language_token_ids( + cls, + tokenizer: object, + ) -> list[int]: + """Return token IDs for all supported language tokens. + + Used with ``SamplingParams.allowed_token_ids`` to constrain + language detection to only produce valid language tokens. + """ + token_ids = [ + tokenizer.convert_tokens_to_ids(f"<|{lang_code}|>") + for lang_code in cls.supported_languages + ] + return token_ids + + @classmethod + def get_language_detection_prompt( + cls, + audio: np.ndarray, + stt_config: SpeechToTextConfig, + ) -> PromptType: + """Return a prompt that elicits a single language token from Whisper. + + Feed only ``<|startoftranscript|>`` as the decoder input so the model + predicts the most likely language token (e.g. ``<|de|>``). + """ + return ExplicitEncoderDecoderPrompt( + encoder_prompt=TextPrompt( + prompt="", + multi_modal_data={"audio": (audio, stt_config.sample_rate)}, + ), + decoder_prompt=TextPrompt(prompt="<|startoftranscript|>"), + ) + + @classmethod + def parse_language_detection_output( + cls, + token_ids: list[int], + tokenizer: object, + ) -> str | None: + """Parse the language token predicted by Whisper. + + Decodes the first token ID and extracts the language code from the + ``<|xx|>`` format. Expects a valid language token from constrained generation. + """ + + decoded = tokenizer.decode( + [token_ids[0]], + skip_special_tokens=False, + ) + # Whisper language tokens have the form <|xx|> + assert decoded.startswith("<|") and decoded.endswith("|>") + lang_code = decoded[2:-2] + assert lang_code in cls.supported_languages + return lang_code + @classmethod def get_placeholder_str(cls, modality: str, i: int) -> str | None: if modality.startswith("audio"): From 965fe45935473e0a81cc6a0885ae7161b9c8b8cf Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sun, 22 Feb 2026 03:14:41 +0800 Subject: [PATCH 62/74] [CI/Build] Fix gRPC version mismatch (#35013) Signed-off-by: DarkLight1337 --- requirements/rocm.txt | 8 ++++++-- requirements/test.in | 6 +++++- requirements/test.txt | 5 +++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/requirements/rocm.txt b/requirements/rocm.txt index 7ac853680f2..9f2b3919935 100644 --- a/requirements/rocm.txt +++ b/requirements/rocm.txt @@ -1,6 +1,11 @@ # Common dependencies -r common.txt +# The version of gRPC libraries should be consistent with each other +grpcio==1.78.0 +grpcio-reflection==1.78.0 +grpcio-tools==1.78.0 + numba == 0.61.2 # Required for N-gram speculative decoding # Dependencies for AMD GPUs @@ -14,5 +19,4 @@ setuptools>=77.0.3,<80.0.0 setuptools-scm>=8 runai-model-streamer[s3,gcs]==0.15.3 conch-triton-kernels==1.2.1 -timm>=1.0.17 -grpcio-tools==1.78.0 # Should match `build.txt` \ No newline at end of file +timm>=1.0.17 \ No newline at end of file diff --git a/requirements/test.in b/requirements/test.in index 92d8fec4bfb..ed9bb47118f 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -48,7 +48,11 @@ buildkite-test-collector==0.1.9 genai_perf>=0.0.8 tritonclient>=2.51.0 -grpcio-tools==1.78.0 # Should match `build.txt` +# The version of gRPC libraries should be consistent with each other +grpcio==1.78.0 +grpcio-reflection==1.78.0 +grpcio-tools==1.78.0 + arctic-inference == 0.1.1 # Required for suffix decoding test numba == 0.61.2 # Required for N-gram speculative decoding numpy diff --git a/requirements/test.txt b/requirements/test.txt index 791bdc005cb..b97bbe902d8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -287,9 +287,13 @@ greenlet==3.2.3 # via sqlalchemy grpcio==1.78.0 # via + # -r requirements/test.in + # grpcio-reflection # grpcio-tools # ray # tensorboard +grpcio-reflection==1.78.0 + # via -r requirements/test.in grpcio-tools==1.78.0 # via -r requirements/test.in h11==0.14.0 @@ -758,6 +762,7 @@ protobuf==6.33.2 # via # google-api-core # googleapis-common-protos + # grpcio-reflection # grpcio-tools # opentelemetry-proto # proto-plus From a4047d4ea993fd52038433d87c16e603bee4f214 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Sat, 21 Feb 2026 12:55:24 -0800 Subject: [PATCH 63/74] [Model Runner V2] Support Eagle3 (no CUDA graph) (#35029) Signed-off-by: Woosuk Kwon --- vllm/v1/worker/gpu/model_runner.py | 101 ++++++++++++------ vllm/v1/worker/gpu/spec_decode/__init__.py | 2 +- .../worker/gpu/spec_decode/eagle/__init__.py | 0 .../cudagraph.py} | 0 .../gpu/spec_decode/eagle/eagle3_utils.py | 46 ++++++++ .../{eagle.py => eagle/speculator.py} | 17 +-- vllm/v1/worker/gpu/spec_decode/eagle/utils.py | 52 +++++++++ 7 files changed, 169 insertions(+), 49 deletions(-) create mode 100644 vllm/v1/worker/gpu/spec_decode/eagle/__init__.py rename vllm/v1/worker/gpu/spec_decode/{eagle_cudagraph.py => eagle/cudagraph.py} (100%) create mode 100644 vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py rename vllm/v1/worker/gpu/spec_decode/{eagle.py => eagle/speculator.py} (97%) create mode 100644 vllm/v1/worker/gpu/spec_decode/eagle/utils.py diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 57d25822913..37f87d7b677 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -66,6 +66,9 @@ from vllm.v1.worker.gpu.sample.output import SamplerOutput from vllm.v1.worker.gpu.sample.prompt_logprob import PromptLogprobsWorker from vllm.v1.worker.gpu.sample.sampler import Sampler from vllm.v1.worker.gpu.spec_decode import init_speculator +from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import ( + set_eagle3_aux_hidden_state_layers, +) from vllm.v1.worker.gpu.spec_decode.rejection_sample import rejection_sample from vllm.v1.worker.gpu.spec_decode.utils import DraftTokensHandler from vllm.v1.worker.gpu.states import RequestState @@ -133,14 +136,42 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.output_copy_stream = torch.cuda.Stream(self.device) self.output_copy_event = torch.cuda.Event() + # Pipeline parallelism. + self.pp_size = self.parallel_config.pipeline_parallel_size + self.use_pp = self.pp_size > 1 + if self.use_pp: + self.is_first_pp_rank = get_pp_group().is_first_rank + self.is_last_pp_rank = get_pp_group().is_last_rank + else: + self.is_first_pp_rank = True + self.is_last_pp_rank = True + + # Decode context parallelism. + self.dcp_size = self.parallel_config.decode_context_parallel_size + self.use_dcp = self.dcp_size > 1 + self.dcp_rank = get_dcp_group().rank_in_group if self.use_dcp else 0 + self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size + + self.speculator = None + self.use_aux_hidden_state_outputs = False if self.speculative_config is not None: self.do_spec_decode = True self.num_speculative_steps = self.speculative_config.num_speculative_tokens - self.speculator = init_speculator(self.vllm_config, self.device) + if self.is_last_pp_rank: + self.speculator = init_speculator(self.vllm_config, self.device) + + if self.speculative_config.method == "eagle3": + # EAGLE3 may require auxiliary hidden states from target model outputs. + self.use_aux_hidden_state_outputs = True + if self.pp_size > 1: + raise ValueError("EAGLE3 with pipeline parallel is not supported.") else: self.do_spec_decode = False self.num_speculative_steps = 0 - self.speculator = None + + # Draft tokens propagation - for spec-dec + struct outputs. + self.draft_tokens_handler = DraftTokensHandler(self.device) + self.req_states = RequestState( max_num_reqs=self.max_num_reqs, max_model_len=self.max_model_len, @@ -176,28 +207,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) # LoRA-related workers. self.lora_state = LoraState(max_num_reqs=self.max_num_reqs) - - # Draft tokens propagation - for spec-dec + struct outputs. - self.draft_tokens_handler = DraftTokensHandler(self.device) - # KV Connector if configured. self.kv_connector: KVConnector = NO_OP_KV_CONNECTOR - # Pipeline parallelism. - self.use_pp = self.parallel_config.pipeline_parallel_size > 1 - if self.use_pp: - self.is_first_pp_rank = get_pp_group().is_first_rank - self.is_last_pp_rank = get_pp_group().is_last_rank - else: - self.is_first_pp_rank = True - self.is_last_pp_rank = True - - # Decode context parallelism. - self.dcp_size = self.parallel_config.decode_context_parallel_size - self.use_dcp = self.dcp_size > 1 - self.dcp_rank = get_dcp_group().rank_in_group if self.use_dcp else 0 - self.cp_interleave = self.parallel_config.cp_kv_cache_interleave_size - def update_max_model_len(self, max_model_len: int) -> None: self.max_model_len = max_model_len self.req_states.max_model_len = max_model_len @@ -220,7 +232,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.model = self.load_lora_model( self.model, self.vllm_config, self.device ) - if self.do_spec_decode: + + if self.use_aux_hidden_state_outputs: + assert self.speculative_config is not None + set_eagle3_aux_hidden_state_layers(self.model, self.speculative_config) + if self.speculator is not None: self.speculator.load_model(self.model) time_after_load = time.perf_counter() @@ -271,7 +287,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.kv_cache_config, self.vllm_config, self.device ) check_attention_cp_compatibility(self.vllm_config) - if self.do_spec_decode: + if self.speculator is not None: # HACK(woosuk) self.speculator.set_attn( self.kv_cache_config, @@ -359,7 +375,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): return None, None assert self.execute_model_state is not None - hidden_states, input_batch, _ = self.execute_model_state + hidden_states, _, input_batch, _ = self.execute_model_state assert hidden_states is not None # Last PP rank always has hidden_states sample_hidden_states = hidden_states[input_batch.logits_indices] return hidden_states, sample_hidden_states @@ -399,7 +415,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): assert sample_hidden_states is not None self._dummy_sampler_run(sample_hidden_states) - if self.do_spec_decode: + if self.speculator is not None: num_tokens_across_dp = make_num_tokens_across_dp( self.parallel_config.data_parallel_size, self.max_num_tokens ) @@ -465,7 +481,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): kv_cache_config=self.kv_cache_config, has_lora=self.lora_config is not None, ) - if self.do_spec_decode: + if self.speculator is not None: self.speculator.capture_model() end_time = time.perf_counter() @@ -964,9 +980,14 @@ class GPUModelRunner(LoRAModelRunnerMixin): # NOTE(woosuk): Here, we don't need to pass the input tensors, # because they are already copied to the CUDA graph input buffers. self.kv_connector.pre_forward(scheduler_output) - hidden_states = self.cudagraph_manager.run_fullgraph( + model_output = self.cudagraph_manager.run_fullgraph( input_batch.num_tokens_after_padding ) + if self.use_aux_hidden_state_outputs: + hidden_states, aux_hidden_states = model_output + else: + hidden_states = model_output + aux_hidden_states = None else: # For piecewise and eager mode, just call model(). positions = input_batch.positions @@ -998,12 +1019,17 @@ class GPUModelRunner(LoRAModelRunnerMixin): slot_mapping=input_batch.slot_mappings, ): self.kv_connector.pre_forward(scheduler_output) - hidden_states = self.model( + model_output = self.model( input_ids=input_ids, positions=positions, inputs_embeds=inputs_embeds, intermediate_tensors=intermediate_tensors, ) + if self.use_aux_hidden_state_outputs: + hidden_states, aux_hidden_states = model_output + else: + hidden_states = model_output + aux_hidden_states = None kv_connector_output = self.kv_connector.post_forward(scheduler_output) @@ -1011,12 +1037,17 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Non-last PP rank: return IntermediateTensors for sending. assert isinstance(hidden_states, IntermediateTensors) hidden_states.kv_connector_output = kv_connector_output - self.execute_model_state = (None, input_batch, kv_connector_output) + self.execute_model_state = (None, None, input_batch, kv_connector_output) return hidden_states - assert isinstance(hidden_states, torch.Tensor) # Last rank (or no PP): hidden_states is a tensor for sampling. - self.execute_model_state = (hidden_states, input_batch, kv_connector_output) + assert isinstance(hidden_states, torch.Tensor) + self.execute_model_state = ( + hidden_states, + aux_hidden_states, + input_batch, + kv_connector_output, + ) return None @torch.inference_mode() @@ -1024,7 +1055,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): self, grammar_output: GrammarOutput | None ) -> AsyncOutput | ModelRunnerOutput | None: assert self.execute_model_state is not None - hidden_states, input_batch, kv_connector_output = self.execute_model_state + hidden_states, aux_hidden_states, input_batch, kv_connector_output = ( + self.execute_model_state + ) self.execute_model_state = None # type: ignore if not self.is_last_pp_rank: @@ -1084,11 +1117,11 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.postprocess( input_batch, sampler_output.sampled_token_ids, num_sampled, num_rejected ) - if self.do_spec_decode: + if self.speculator is not None: draft_tokens = self.propose_draft( input_batch, hidden_states, - None, # aux_hidden_states + aux_hidden_states, num_sampled, num_rejected, ) diff --git a/vllm/v1/worker/gpu/spec_decode/__init__.py b/vllm/v1/worker/gpu/spec_decode/__init__.py index 07026a51210..536b7526bdd 100644 --- a/vllm/v1/worker/gpu/spec_decode/__init__.py +++ b/vllm/v1/worker/gpu/spec_decode/__init__.py @@ -9,7 +9,7 @@ def init_speculator(vllm_config: VllmConfig, device: torch.device): speculative_config = vllm_config.speculative_config assert speculative_config is not None if speculative_config.use_eagle(): - from vllm.v1.worker.gpu.spec_decode.eagle import EagleSpeculator + from vllm.v1.worker.gpu.spec_decode.eagle.speculator import EagleSpeculator return EagleSpeculator(vllm_config, device) raise NotImplementedError(f"{speculative_config.method} is not supported yet.") diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/__init__.py b/vllm/v1/worker/gpu/spec_decode/eagle/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vllm/v1/worker/gpu/spec_decode/eagle_cudagraph.py b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py similarity index 100% rename from vllm/v1/worker/gpu/spec_decode/eagle_cudagraph.py rename to vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py new file mode 100644 index 00000000000..d76d69355fa --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/eagle/eagle3_utils.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import cast + +import torch.nn as nn + +from vllm.config import SpeculativeConfig +from vllm.logger import init_logger +from vllm.model_executor.models.interfaces import SupportsEagle3, supports_eagle3 + +logger = init_logger(__name__) + + +def set_eagle3_aux_hidden_state_layers( + model: nn.Module, + spec_config: SpeculativeConfig, +) -> None: + if not supports_eagle3(model): + raise RuntimeError("Model does not support EAGLE3 interface") + # mypy may infer the class-level overload for supports_eagle3. + # Narrow explicitly to the runtime protocol instance. + if isinstance(model, type): + raise RuntimeError("Expected model instance for EAGLE3 configuration") + eagle3_model = cast(SupportsEagle3, model) + + aux_layers = get_eagle3_aux_layers_from_config(spec_config) + if aux_layers: + logger.info("Using Eagle3 auxiliary layers from config: %s", aux_layers) + else: + aux_layers = eagle3_model.get_eagle3_aux_hidden_state_layers() + logger.info("Using Eagle3 auxiliary layers from model: %s", aux_layers) + eagle3_model.set_aux_hidden_state_layers(aux_layers) + + +def get_eagle3_aux_layers_from_config( + spec_config: SpeculativeConfig, +) -> tuple[int, ...] | None: + if not (spec_config and spec_config.draft_model_config): + return None + hf_config = spec_config.draft_model_config.hf_config + if not hasattr(hf_config, "eagle_aux_hidden_state_layer_ids"): + return None + layer_ids = hf_config.eagle_aux_hidden_state_layer_ids + if layer_ids and isinstance(layer_ids, (list, tuple)): + return tuple(layer_ids) + return None diff --git a/vllm/v1/worker/gpu/spec_decode/eagle.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py similarity index 97% rename from vllm/v1/worker/gpu/spec_decode/eagle.py rename to vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index abbde270f4a..3cd8afee74f 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -9,7 +9,6 @@ from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger -from vllm.model_executor.model_loader import get_model from vllm.triton_utils import tl, triton from vllm.v1.attention.backend import AttentionMetadataBuilder from vllm.v1.kv_cache_interface import KVCacheConfig @@ -20,7 +19,8 @@ from vllm.v1.worker.gpu.attn_utils import ( from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample -from vllm.v1.worker.gpu.spec_decode.eagle_cudagraph import EagleCudaGraphManager +from vllm.v1.worker.gpu.spec_decode.eagle.cudagraph import EagleCudaGraphManager +from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model logger = init_logger(__name__) @@ -73,18 +73,7 @@ class EagleSpeculator: self.cudagraph_manager = EagleCudaGraphManager(vllm_config, device) def load_model(self, target_model: nn.Module) -> None: - from vllm.compilation.backends import set_model_tag - - with set_model_tag("eagle_head"): - self.model = get_model( - vllm_config=self.vllm_config, model_config=self.draft_model_config - ) - - share_lm_head = True - if share_lm_head and hasattr(target_model, "lm_head"): - if hasattr(self.model, "lm_head"): - del self.model.lm_head - self.model.lm_head = target_model.lm_head + self.model = load_eagle_model(target_model, self.vllm_config) def set_attn( self, diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/utils.py b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py new file mode 100644 index 00000000000..ee37eadb2a8 --- /dev/null +++ b/vllm/v1/worker/gpu/spec_decode/eagle/utils.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import torch.nn as nn + +from vllm.config import VllmConfig +from vllm.model_executor.model_loader import get_model + + +def load_eagle_model(target_model: nn.Module, vllm_config: VllmConfig) -> nn.Module: + from vllm.compilation.backends import set_model_tag + + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + draft_model_config = speculative_config.draft_model_config + with set_model_tag("eagle_head"): + eagle_model = get_model( + vllm_config=vllm_config, model_config=draft_model_config + ) + + # Share target embeddings when the draft checkpoint does not include + # its own vocab embedding table. + share_embeddings = True + if hasattr(eagle_model, "has_own_embed_tokens"): + share_embeddings = not eagle_model.has_own_embed_tokens + if share_embeddings: + target_language_model = ( + target_model.get_language_model() + if hasattr(target_model, "get_language_model") + else target_model + ) + inner_model = getattr(target_language_model, "model", None) + target_embed_tokens = None + if inner_model is not None: + if hasattr(inner_model, "embed_tokens"): + target_embed_tokens = inner_model.embed_tokens + elif hasattr(inner_model, "embedding"): + target_embed_tokens = inner_model.embedding + if target_embed_tokens is not None and hasattr(eagle_model, "model"): + if hasattr(eagle_model.model, "embed_tokens"): + del eagle_model.model.embed_tokens + eagle_model.model.embed_tokens = target_embed_tokens + + # Only share target lm_head when the draft model does not own one. + share_lm_head = True + if hasattr(eagle_model, "has_own_lm_head"): + share_lm_head = not eagle_model.has_own_lm_head + if share_lm_head and hasattr(target_model, "lm_head"): + if hasattr(eagle_model, "lm_head"): + del eagle_model.lm_head + eagle_model.lm_head = target_model.lm_head + + return eagle_model From 74d90b1ce49e5984ccf054d6e918c8efbafce3c1 Mon Sep 17 00:00:00 2001 From: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Date: Sun, 22 Feb 2026 04:28:01 +0400 Subject: [PATCH 64/74] [Model Bash][DSR1] Add selective dynamic shape marking for CustomOp (#34900) Signed-off-by: Vadim Gimpelson --- vllm/model_executor/custom_op.py | 48 ++++++++++++++++--- .../layers/attention/mla_attention.py | 5 +- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/vllm/model_executor/custom_op.py b/vllm/model_executor/custom_op.py index ee75d627d55..851546297e6 100644 --- a/vllm/model_executor/custom_op.py +++ b/vllm/model_executor/custom_op.py @@ -1,5 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools +import inspect + import torch import torch.nn as nn @@ -205,9 +208,9 @@ class CustomOp(nn.Module): NOTE: this does not enable fusion across ops, so opaque custom ops should still be unwrapped wherever possible. """ - # Do not compile if compilation disabled from vllm.config.compilation import CompilationMode + # Do not compile if compilation disabled if not enable: return fn @@ -220,14 +223,42 @@ class CustomOp(nn.Module): if compilation_config.backend == "eager": return fn + compile_options = maybe_disable_graph_partition( + current_platform.simple_compile_backend + ) + backend = current_platform.simple_compile_backend + + dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None) + if dynamic_arg_dims is not None: + compiled_fn = torch.compile( + fn, + dynamic=False, + backend=backend, + options=compile_options, + ) + sig = inspect.signature(fn) + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + for name, dims in dynamic_arg_dims.items(): + arg = bound.arguments.get(name) + if arg is not None and isinstance(arg, torch.Tensor): + dims_list = [dims] if isinstance(dims, int) else dims + for d in dims_list: + real_d = arg.ndim + d if d < 0 else d + torch._dynamo.mark_dynamic(arg, real_d) + return compiled_fn(*args, **kwargs) + + return wrapper + # dynamic=True to avoid recompilations return torch.compile( fn, dynamic=True, - backend=current_platform.simple_compile_backend, - options=maybe_disable_graph_partition( - current_platform.simple_compile_backend - ), + backend=backend, + options=compile_options, ) @classmethod @@ -267,10 +298,15 @@ class CustomOp(nn.Module): # Decorator to register custom ops. @classmethod - def register(cls, name: str): + def register( + cls, + name: str, + dynamic_arg_dims: dict[str, int | list[int]] | None = None, + ): def decorator(op_cls): assert name not in op_registry, f"Duplicate op name: {name}" op_cls.name = name + op_cls._dynamic_arg_dims = dynamic_arg_dims op_registry[name] = op_cls return op_cls diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 98ff02e9d4a..faebad5968a 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -950,7 +950,10 @@ def dynamic_per_batched_tensor_quant( logger = init_logger(__name__) -@CustomOp.register("mla_decode_concat_quant_fp8") +@CustomOp.register( + "mla_decode_concat_quant_fp8", + dynamic_arg_dims={"decode_ql_nope": 0, "decode_q_pe": 0}, +) class _DecodeConcatQuantFP8(QuantFP8): """ QuantFP8 variant that concatenates decode_ql_nope and decode_q_pe before From b71fbd06e215a1a09600220c947a1bb2d5494de9 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Sat, 21 Feb 2026 16:42:53 -0800 Subject: [PATCH 65/74] [Model Runner V2] Support attention group (#35036) Signed-off-by: Woosuk Kwon --- vllm/v1/worker/gpu/attn_utils.py | 83 ++++++++++++------- vllm/v1/worker/gpu/cudagraph_utils.py | 24 ++++-- vllm/v1/worker/gpu/model_runner.py | 13 ++- .../worker/gpu/spec_decode/eagle/cudagraph.py | 10 +-- .../gpu/spec_decode/eagle/speculator.py | 10 +-- 5 files changed, 86 insertions(+), 54 deletions(-) diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index 468e77113bc..d9fc4515b88 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -7,17 +7,14 @@ import torch from vllm.config import VllmConfig, get_layers_from_vllm_config from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase -from vllm.v1.attention.backend import ( - AttentionBackend, - AttentionMetadataBuilder, - CommonAttentionMetadata, -) +from vllm.v1.attention.backend import AttentionBackend, CommonAttentionMetadata from vllm.v1.kv_cache_interface import ( AttentionSpec, KVCacheConfig, KVCacheSpec, + UniformTypeKVCacheSpecs, ) -from vllm.v1.worker.utils import bind_kv_cache +from vllm.v1.worker.utils import AttentionGroup, bind_kv_cache def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]: @@ -35,29 +32,56 @@ def init_attn_backend( kv_cache_config: KVCacheConfig, vllm_config: VllmConfig, device: torch.device ): attn_backends: dict[str, type[AttentionBackend]] = {} - attn_metadata_builders: list[AttentionMetadataBuilder] = [] - flashinfer_workspace: torch.Tensor | None = None - for kv_cache_group_spec in kv_cache_config.kv_cache_groups: + attn_groups: list[list[AttentionGroup]] = [] + attn_backend_workspace: torch.Tensor | None = None + for kv_cache_group_id, kv_cache_group_spec in enumerate( + kv_cache_config.kv_cache_groups + ): layer_names = kv_cache_group_spec.layer_names - any_layer_name = next(iter(layer_names)) layer_type = cast(type[Any], AttentionLayerBase) attn_layers = get_layers_from_vllm_config(vllm_config, layer_type, layer_names) - attn_backend = attn_layers[any_layer_name].get_attn_backend() + + group_map: dict[tuple[tuple[str, str], KVCacheSpec], AttentionGroup] = {} + group_order: list[tuple[tuple[str, str], KVCacheSpec]] = [] + for layer_name in layer_names: + attn_backend = attn_layers[layer_name].get_attn_backend() attn_backends[layer_name] = attn_backend - attn_metadata_builder = attn_backend.get_builder_cls()( - kv_cache_group_spec.kv_cache_spec, layer_names, vllm_config, device - ) - attn_metadata_builders.append(attn_metadata_builder) # type: ignore + layer_kv_cache_spec: KVCacheSpec = kv_cache_group_spec.kv_cache_spec + if isinstance(layer_kv_cache_spec, UniformTypeKVCacheSpecs): + layer_kv_cache_spec = layer_kv_cache_spec.kv_cache_specs[layer_name] - if attn_backend.get_name() == "FLASHINFER": - if flashinfer_workspace is None: - flashinfer_workspace = attn_metadata_builder._get_workspace_buffer() + key = (attn_backend.full_cls_name(), layer_kv_cache_spec) + if key not in group_map: + group_map[key] = AttentionGroup( + attn_backend, + [layer_name], + layer_kv_cache_spec, + kv_cache_group_id, + ) + group_order.append(key) else: - attn_metadata_builder.set_workspace_buffer(flashinfer_workspace) - return attn_backends, attn_metadata_builders + group_map[key].layer_names.append(layer_name) + + groups = [group_map[key] for key in group_order] + for group in groups: + group.create_metadata_builders( + vllm_config=vllm_config, + device=device, + kernel_block_size=None, + num_metadata_builders=1, + ) + builder = group.get_metadata_builder(0) + if attn_backend_workspace is None: + if hasattr(builder, "_get_workspace_buffer"): + attn_backend_workspace = builder._get_workspace_buffer() + else: + if hasattr(builder, "set_workspace_buffer"): + builder.set_workspace_buffer(attn_backend_workspace) + attn_groups.append(groups) + return attn_backends, attn_groups def _allocate_kv_cache(kv_cache_config: KVCacheConfig, device: torch.device): @@ -144,7 +168,7 @@ def build_slot_mappings_by_layer( def build_attn_metadata( - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], num_reqs: int, num_tokens: int, query_start_loc_gpu: torch.Tensor, @@ -162,8 +186,8 @@ def build_attn_metadata( dcp_local_seq_lens = dcp_local_seq_lens[:num_reqs] attn_metadata: dict[str, Any] = {} - kv_cache_groups = kv_cache_config.kv_cache_groups - for i, kv_cache_spec in enumerate(kv_cache_groups): + num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) + for i in range(num_kv_cache_groups): block_table = block_tables[i] slot_mapping = slot_mappings[i] @@ -181,10 +205,11 @@ def build_attn_metadata( dcp_local_seq_lens=dcp_local_seq_lens, ) - attn_metadata_builder = attn_metadata_builders[i] - metadata = attn_metadata_builder.build( - common_prefix_len=0, common_attn_metadata=common_attn_metadata - ) - for layer_name in kv_cache_spec.layer_names: - attn_metadata[layer_name] = metadata + for attn_group in attn_groups[i]: + attn_metadata_builder = attn_group.get_metadata_builder(0) + metadata = attn_metadata_builder.build( + common_prefix_len=0, common_attn_metadata=common_attn_metadata + ) + for layer_name in attn_group.layer_names: + attn_metadata[layer_name] = metadata return attn_metadata diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index e3839894a95..7bba7ffb928 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -13,7 +13,6 @@ from vllm.config.compilation import CUDAGraphMode from vllm.distributed.parallel_state import graph_capture, is_global_first_rank from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.utils.math_utils import cdiv -from vllm.v1.attention.backend import AttentionMetadataBuilder from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import ( build_attn_metadata, @@ -22,6 +21,7 @@ from vllm.v1.worker.gpu.attn_utils import ( from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.dp_utils import make_num_tokens_across_dp from vllm.v1.worker.gpu.input_batch import InputBuffers +from vllm.v1.worker.utils import AttentionGroup class CudaGraphManager: @@ -83,7 +83,7 @@ class CudaGraphManager: mrope_positions: torch.Tensor | None, inputs_embeds: torch.Tensor | None, block_tables: BlockTables, - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, has_lora: bool = False, uniform_decode: bool = False, @@ -116,7 +116,7 @@ class CudaGraphManager: num_tokens, input_buffers, block_tables, - attn_metadata_builders, + attn_groups, self.max_model_len, kv_cache_config, uniform_decode_query_len=( @@ -232,7 +232,7 @@ class CudaGraphManager: mrope_positions: torch.Tensor | None, inputs_embeds: torch.Tensor | None, block_tables: BlockTables, - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, has_lora: bool = False, ) -> None: @@ -244,7 +244,7 @@ class CudaGraphManager: mrope_positions=mrope_positions, inputs_embeds=inputs_embeds, block_tables=block_tables, - attn_metadata_builders=attn_metadata_builders, + attn_groups=attn_groups, kv_cache_config=kv_cache_config, has_lora=has_lora, ) @@ -286,6 +286,16 @@ class CudaGraphManager: cudagraph_mode = self.cudagraph_mode.decode_mode() else: cudagraph_mode = self.cudagraph_mode.mixed_mode() + + if ( + cudagraph_mode == CUDAGraphMode.FULL + and cudagraph_size is not None + and cudagraph_size not in self.graphs + ): + # If graph wasn't captured yet, fall back to eager. + # This might happen when the dummy run is called before capture. + cudagraph_mode = CUDAGraphMode.NONE + cudagraph_size = None return cudagraph_mode, cudagraph_size def run_fullgraph(self, num_tokens: int) -> torch.Tensor: @@ -354,7 +364,7 @@ def prepare_inputs_to_capture( num_tokens: int, input_buffers: InputBuffers, block_tables: BlockTables, - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], max_model_len: int, kv_cache_config: KVCacheConfig, uniform_decode_query_len: int = 0, @@ -386,7 +396,7 @@ def prepare_inputs_to_capture( ) attn_metadata = build_attn_metadata( - attn_metadata_builders=attn_metadata_builders, + attn_groups=attn_groups, num_reqs=num_reqs, num_tokens=num_tokens, query_start_loc_gpu=query_start_loc, diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 37f87d7b677..b909b90adf1 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -283,7 +283,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): cp_interleave=self.cp_interleave, ) - self.attn_backends, self.attn_metadata_builders = init_attn_backend( + self.attn_backends, self.attn_groups = init_attn_backend( self.kv_cache_config, self.vllm_config, self.device ) check_attention_cp_compatibility(self.vllm_config) @@ -291,7 +291,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # HACK(woosuk) self.speculator.set_attn( self.kv_cache_config, - self.attn_metadata_builders, + self.attn_groups, self.block_tables, ) @@ -305,9 +305,6 @@ class GPUModelRunner(LoRAModelRunnerMixin): ) self.kv_connector = get_kv_connector(self.vllm_config, kv_caches_dict) - # Attention groups are not supported. - self.attn_groups = [] # type: ignore - def prepare_dummy_attn_metadata(self, input_batch: InputBatch) -> None: block_tables = self.block_tables.get_dummy_block_tables(input_batch.num_reqs) slot_mappings = self.block_tables.get_dummy_slot_mappings( @@ -317,7 +314,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): slot_mappings, self.kv_cache_config ) attn_metadata = build_attn_metadata( - attn_metadata_builders=self.attn_metadata_builders, + attn_groups=self.attn_groups, num_reqs=input_batch.num_reqs, num_tokens=input_batch.num_tokens, query_start_loc_gpu=input_batch.query_start_loc, @@ -477,7 +474,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): mrope_positions=mrope_positions, inputs_embeds=inputs_embeds, block_tables=self.block_tables, - attn_metadata_builders=self.attn_metadata_builders, + attn_groups=self.attn_groups, kv_cache_config=self.kv_cache_config, has_lora=self.lora_config is not None, ) @@ -712,7 +709,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): # Layer name -> attention metadata. attn_metadata = build_attn_metadata( - attn_metadata_builders=self.attn_metadata_builders, + attn_groups=self.attn_groups, num_reqs=num_reqs, num_tokens=num_tokens, query_start_loc_gpu=query_start_loc, diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py index ae7aa40784e..c489a172c5a 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/cudagraph.py @@ -7,7 +7,6 @@ import torch from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode -from vllm.v1.attention.backend import AttentionMetadataBuilder from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.block_table import BlockTables from vllm.v1.worker.gpu.cudagraph_utils import ( @@ -17,6 +16,7 @@ from vllm.v1.worker.gpu.cudagraph_utils import ( ) from vllm.v1.worker.gpu.dp_utils import make_num_tokens_across_dp from vllm.v1.worker.gpu.input_batch import InputBuffers +from vllm.v1.worker.utils import AttentionGroup class EagleCudaGraphManager: @@ -60,7 +60,7 @@ class EagleCudaGraphManager: generate_fn: Callable, input_buffers: InputBuffers, block_tables: BlockTables, - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, ) -> None: assert capture_cg_mode in [CUDAGraphMode.PIECEWISE, CUDAGraphMode.FULL], ( @@ -77,7 +77,7 @@ class EagleCudaGraphManager: num_tokens, input_buffers, block_tables, - attn_metadata_builders, + attn_groups, self.max_model_len, kv_cache_config, uniform_decode_query_len=1, @@ -150,7 +150,7 @@ class EagleCudaGraphManager: generate_fn: Callable, input_buffers: InputBuffers, block_tables: BlockTables, - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], kv_cache_config: KVCacheConfig, ) -> None: if self.cudagraph_mode == CUDAGraphMode.NONE: @@ -165,7 +165,7 @@ class EagleCudaGraphManager: generate_fn=generate_fn, input_buffers=input_buffers, block_tables=block_tables, - attn_metadata_builders=attn_metadata_builders, + attn_groups=attn_groups, kv_cache_config=kv_cache_config, ) diff --git a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py index 3cd8afee74f..6cd13cebf99 100644 --- a/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/eagle/speculator.py @@ -10,7 +10,6 @@ from vllm.config.compilation import CUDAGraphMode from vllm.forward_context import BatchDescriptor, set_forward_context from vllm.logger import init_logger from vllm.triton_utils import tl, triton -from vllm.v1.attention.backend import AttentionMetadataBuilder from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.attn_utils import ( build_attn_metadata, @@ -21,6 +20,7 @@ from vllm.v1.worker.gpu.input_batch import InputBatch, InputBuffers from vllm.v1.worker.gpu.sample.gumbel import gumbel_sample from vllm.v1.worker.gpu.spec_decode.eagle.cudagraph import EagleCudaGraphManager from vllm.v1.worker.gpu.spec_decode.eagle.utils import load_eagle_model +from vllm.v1.worker.utils import AttentionGroup logger = init_logger(__name__) @@ -78,11 +78,11 @@ class EagleSpeculator: def set_attn( self, kv_cache_config: KVCacheConfig, - attn_metadata_builders: list[AttentionMetadataBuilder], + attn_groups: list[list[AttentionGroup]], block_tables: BlockTables, ) -> None: self.kv_cache_config = kv_cache_config - self.attn_metadata_builders = attn_metadata_builders + self.attn_groups = attn_groups self.block_tables = block_tables @torch.inference_mode() @@ -174,7 +174,7 @@ class EagleSpeculator: self.generate_draft, self.input_buffers, self.block_tables, - self.attn_metadata_builders, + self.attn_groups, self.kv_cache_config, ) @@ -298,7 +298,7 @@ class EagleSpeculator: # FIXME(woosuk): This is UNSAFE!! attn_metadata = build_attn_metadata( - attn_metadata_builders=self.attn_metadata_builders, + attn_groups=self.attn_groups, num_reqs=num_reqs, num_tokens=num_reqs, query_start_loc_gpu=query_start_loc, From d403c1da1cd5c210581a2ed4c08c6b932b45186b Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sat, 21 Feb 2026 22:01:10 -0600 Subject: [PATCH 66/74] [CI] Stabilizing ROCm amd-ci signal and minor name fix in upstream (#35008) Signed-off-by: Andreas Karatzas --- .buildkite/test-amd.yaml | 2 +- .buildkite/test_areas/distributed.yaml | 2 +- requirements/rocm-test.txt | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index ba6edb92f40..ffdf4b83c0e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -67,7 +67,7 @@ steps: timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdtentative] agent_pool: mi325_1 - grade: Blocking + # grade: Blocking source_file_dependencies: - vllm/ - tests/test_inputs.py diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 9ded5ffda8e..f15e5018b7f 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -209,7 +209,7 @@ steps: - uv pip install --system -r /vllm-workspace/requirements/kv_connectors.txt - CROSS_LAYERS_BLOCKS=True bash v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh -- label: Pipeline + Context Parallelism (4 GPUs)) +- label: Pipeline + Context Parallelism (4 GPUs) timeout_in_minutes: 60 working_dir: "/vllm-workspace/tests" num_devices: 4 diff --git a/requirements/rocm-test.txt b/requirements/rocm-test.txt index 1983392a124..5cfda430b5c 100644 --- a/requirements/rocm-test.txt +++ b/requirements/rocm-test.txt @@ -104,3 +104,5 @@ segmentation-models-pytorch==0.5.0 imagehash==4.3.2 # Required for bitsandbytes quantization test bitsandbytes==0.49.2 +# Examples (tensorizer) tests +tensorizer==2.10.1 From d24bdd7c4b3eb61a8a025068b6b3ad4d8041abf7 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Sat, 21 Feb 2026 23:23:24 -0500 Subject: [PATCH 67/74] [CI] Bump mteb version to `mteb[bm25s]>=2, <3` for pooling model unit tests (#34961) Signed-off-by: yewentao256 --- requirements/nightly_torch_test.txt | 2 +- requirements/rocm-test.txt | 2 +- requirements/test.txt | 2 +- tests/models/language/pooling_mteb_test/mteb_score_utils.py | 3 +++ 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/requirements/nightly_torch_test.txt b/requirements/nightly_torch_test.txt index 9a0bc4b20df..27299f47ff4 100644 --- a/requirements/nightly_torch_test.txt +++ b/requirements/nightly_torch_test.txt @@ -28,7 +28,7 @@ num2words # required for smolvlm test opencv-python-headless >= 4.13.0 # required for video test datamodel_code_generator # required for minicpm3 test lm-eval[api]>=0.4.11 # required for model evaluation test -mteb>=1.38.11, <2 # required for mteb test +mteb[bm25s]>=2, <3 # required for mteb test transformers==4.57.5 tokenizers==0.22.0 schemathesis>=3.39.15 # Required for openai schema test. diff --git a/requirements/rocm-test.txt b/requirements/rocm-test.txt index 5cfda430b5c..dd7f949f88b 100644 --- a/requirements/rocm-test.txt +++ b/requirements/rocm-test.txt @@ -70,7 +70,7 @@ ray[cgraph,default]>=2.48.0 torchgeo==0.7.0 # via terratorch # MTEB Benchmark Test -mteb==2.1.2 +mteb[bm25s]>=2, <3 # Utilities num2words==0.5.14 diff --git a/requirements/test.txt b/requirements/test.txt index b97bbe902d8..8aa2d6768e0 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -491,7 +491,7 @@ msgpack==1.1.0 # via # librosa # ray -mteb==2.1.2 +mteb==2.8.3 # via -r requirements/test.in multidict==6.1.0 # via diff --git a/tests/models/language/pooling_mteb_test/mteb_score_utils.py b/tests/models/language/pooling_mteb_test/mteb_score_utils.py index ad32880390e..621aff0e998 100644 --- a/tests/models/language/pooling_mteb_test/mteb_score_utils.py +++ b/tests/models/language/pooling_mteb_test/mteb_score_utils.py @@ -191,6 +191,9 @@ def run_mteb_rerank(cross_encoder: mteb.CrossEncoderProtocol, tasks, languages): mteb_tasks: list[mteb.abstasks.AbsTaskRetrieval] = mteb.get_tasks( tasks=tasks, languages=languages, eval_splits=eval_splits ) + for task in mteb_tasks: + if not task.data_loaded: + task.load_data() mteb.evaluate( bm25s, From 970861ac0cfc93d8ebdeb2c0f5d664289eafb51c Mon Sep 17 00:00:00 2001 From: Athrael Soju Date: Sun, 22 Feb 2026 04:23:41 +0000 Subject: [PATCH 68/74] [New Model] Add ColModernVBERT (#34558) Signed-off-by: Athrael Soju Signed-off-by: athrael-soju --- docs/models/supported_models.md | 1 + .../score/colmodernvbert_rerank_online.py | 166 +++++++ .../multimodal/pooling/test_colmodernvbert.py | 115 +++++ tests/models/registry.py | 3 + vllm/model_executor/models/colmodernvbert.py | 430 ++++++++++++++++++ vllm/model_executor/models/registry.py | 1 + vllm/transformers_utils/config.py | 1 + vllm/transformers_utils/configs/__init__.py | 2 + .../configs/colmodernvbert.py | 65 +++ 9 files changed, 784 insertions(+) create mode 100644 examples/pooling/score/colmodernvbert_rerank_online.py create mode 100644 tests/models/multimodal/pooling/test_colmodernvbert.py create mode 100644 vllm/model_executor/models/colmodernvbert.py create mode 100644 vllm/transformers_utils/configs/colmodernvbert.py diff --git a/docs/models/supported_models.md b/docs/models/supported_models.md index 1cad8c4a171..0551d4670bc 100644 --- a/docs/models/supported_models.md +++ b/docs/models/supported_models.md @@ -821,6 +821,7 @@ The following table lists those that are tested in vLLM. | Architecture | Models | Inputs | Example HF Models | [LoRA](../features/lora.md) | [PP](../serving/parallelism_scaling.md) | |--------------|--------|--------|-------------------|----------------------|---------------------------| | `CLIPModel` | CLIP | T / I | `openai/clip-vit-base-patch32`, `openai/clip-vit-large-patch14`, etc. | | | +| `ColModernVBertForRetrieval` | ColModernVBERT | T / I | `ModernVBERT/colmodernvbert-merged` | | | | `LlavaNextForConditionalGeneration`C | LLaVA-NeXT-based | T / I | `royokong/e5-v` | | ✅︎ | | `Phi3VForCausalLM`C | Phi-3-Vision-based | T + I | `TIGER-Lab/VLM2Vec-Full` | | ✅︎ | | `Qwen3VLForConditionalGeneration`C | Qwen3-VL | T + I + V | `Qwen/Qwen3-VL-Embedding-2B`, etc. | ✅︎ | ✅︎ | diff --git a/examples/pooling/score/colmodernvbert_rerank_online.py b/examples/pooling/score/colmodernvbert_rerank_online.py new file mode 100644 index 00000000000..de827ae0626 --- /dev/null +++ b/examples/pooling/score/colmodernvbert_rerank_online.py @@ -0,0 +1,166 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +Example of using ColModernVBERT late interaction model for reranking. + +ColModernVBERT is a multi-modal ColBERT-style model combining a SigLIP +vision encoder with a ModernBERT text encoder. It produces per-token +embeddings and uses MaxSim scoring for retrieval and reranking. +Supports both text and image inputs. + +Start the server with: + vllm serve ModernVBERT/colmodernvbert-merged --max-model-len 8192 + +Then run this script: + python colmodernvbert_rerank_online.py +""" + +import requests + +MODEL = "ModernVBERT/colmodernvbert-merged" +BASE_URL = "http://127.0.0.1:8000" + +headers = {"accept": "application/json", "Content-Type": "application/json"} + +IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/thumb/4/47/PNG_transparency_demonstration_1.png/300px-PNG_transparency_demonstration_1.png" # noqa: E501 + + +def rerank_text(): + """Text-only reranking via /rerank endpoint.""" + print("=" * 60) + print("1. Text reranking (/rerank)") + print("=" * 60) + + data = { + "model": MODEL, + "query": "What is machine learning?", + "documents": [ + "Machine learning is a subset of artificial intelligence.", + "Python is a programming language.", + "Deep learning uses neural networks for complex tasks.", + "The weather today is sunny.", + ], + } + + response = requests.post(f"{BASE_URL}/rerank", headers=headers, json=data) + + if response.status_code == 200: + result = response.json() + print("\n Ranked documents (most relevant first):") + for item in result["results"]: + doc_idx = item["index"] + score = item["relevance_score"] + print(f" [{score:.4f}] {data['documents'][doc_idx]}") + else: + print(f" Request failed: {response.status_code}") + print(f" {response.text[:300]}") + + +def score_text(): + """Text-only scoring via /score endpoint.""" + print() + print("=" * 60) + print("2. Text scoring (/score)") + print("=" * 60) + + query = "What is the capital of France?" + documents = [ + "The capital of France is Paris.", + "Berlin is the capital of Germany.", + "Python is a programming language.", + ] + + data = { + "model": MODEL, + "text_1": query, + "text_2": documents, + } + + response = requests.post(f"{BASE_URL}/score", headers=headers, json=data) + + if response.status_code == 200: + result = response.json() + print(f"\n Query: {query}\n") + for item in result["data"]: + idx = item["index"] + score = item["score"] + print(f" Doc {idx} (score={score:.4f}): {documents[idx]}") + else: + print(f" Request failed: {response.status_code}") + print(f" {response.text[:300]}") + + +def score_text_top_n(): + """Text reranking with top_n filtering via /rerank endpoint.""" + print() + print("=" * 60) + print("3. Text reranking with top_n=2 (/rerank)") + print("=" * 60) + + data = { + "model": MODEL, + "query": "What is the capital of France?", + "documents": [ + "The capital of France is Paris.", + "Berlin is the capital of Germany.", + "Python is a programming language.", + "The Eiffel Tower is in Paris.", + ], + "top_n": 2, + } + + response = requests.post(f"{BASE_URL}/rerank", headers=headers, json=data) + + if response.status_code == 200: + result = response.json() + print(f"\n Top {data['top_n']} results:") + for item in result["results"]: + doc_idx = item["index"] + score = item["relevance_score"] + print(f" [{score:.4f}] {data['documents'][doc_idx]}") + else: + print(f" Request failed: {response.status_code}") + print(f" {response.text[:300]}") + + +def rerank_multimodal(): + """Multimodal reranking with text and image documents via /rerank.""" + print() + print("=" * 60) + print("4. Multimodal reranking: text query vs image document (/rerank)") + print("=" * 60) + + data = { + "model": MODEL, + "query": "A colorful logo with transparency", + "documents": [ + {"content": [{"type": "image_url", "image_url": {"url": IMAGE_URL}}]}, + "Python is a programming language.", + "The weather today is sunny.", + ], + } + + response = requests.post(f"{BASE_URL}/rerank", headers=headers, json=data) + + if response.status_code == 200: + result = response.json() + print("\n Ranked documents (most relevant first):") + labels = ["[image]", "Python doc", "Weather doc"] + for item in result["results"]: + doc_idx = item["index"] + score = item["relevance_score"] + print(f" [{score:.4f}] {labels[doc_idx]}") + else: + print(f" Request failed: {response.status_code}") + print(f" {response.text[:300]}") + + +def main(): + rerank_text() + score_text() + score_text_top_n() + rerank_multimodal() + + +if __name__ == "__main__": + main() diff --git a/tests/models/multimodal/pooling/test_colmodernvbert.py b/tests/models/multimodal/pooling/test_colmodernvbert.py new file mode 100644 index 00000000000..01f3843c34e --- /dev/null +++ b/tests/models/multimodal/pooling/test_colmodernvbert.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for ColModernVBERT multimodal late-interaction model. + +ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder +with a pixel shuffle connector and ColBERT-style 128-dim per-token +embeddings for visual document retrieval. +""" + +import pytest +import torch + +from vllm.entrypoints.pooling.score.utils import compute_maxsim_score + +MODEL_NAME = "ModernVBERT/colmodernvbert-merged" +COLBERT_DIM = 128 +DTYPE = "half" + + +# ----------------------------------------------------------------------- +# Text-only tests +# ----------------------------------------------------------------------- + + +def test_colmodernvbert_text_token_embed(vllm_runner): + """Text query produces per-token embeddings with shape (seq_len, 128).""" + with vllm_runner( + MODEL_NAME, + runner="pooling", + dtype=DTYPE, + enforce_eager=True, + ) as vllm_model: + outputs = vllm_model.token_embed(["What is machine learning?"]) + + assert len(outputs) == 1 + emb = torch.tensor(outputs[0]) + assert emb.dim() == 2 + assert emb.shape[1] == COLBERT_DIM + assert emb.shape[0] > 1 + + +def test_colmodernvbert_text_relevance_ordering(vllm_runner): + """Relevant documents score higher than irrelevant ones.""" + query = "What is machine learning?" + documents = [ + "Machine learning is a subset of artificial intelligence.", + "The weather in Paris is mild in spring.", + ] + + with vllm_runner( + MODEL_NAME, + runner="pooling", + dtype=DTYPE, + enforce_eager=True, + ) as vllm_model: + scores = vllm_model.score(query, documents) + + assert len(scores) == 2 + assert scores[0] > scores[1], "ML doc should score higher than weather doc" + + +def test_colmodernvbert_text_late_interaction(vllm_runner): + """MaxSim scoring via vLLM matches manual computation.""" + query = "What is the capital of France?" + doc = "The capital of France is Paris." + + with vllm_runner( + MODEL_NAME, + runner="pooling", + dtype=DTYPE, + enforce_eager=True, + ) as vllm_model: + q_out = vllm_model.token_embed([query]) + d_out = vllm_model.token_embed([doc]) + + q_emb = torch.tensor(q_out[0]) + d_emb = torch.tensor(d_out[0]) + manual_score = compute_maxsim_score(q_emb, d_emb).item() + + vllm_scores = vllm_model.score(query, doc) + + assert len(vllm_scores) == 1 + assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01) + + +# ----------------------------------------------------------------------- +# Image tests +# ----------------------------------------------------------------------- + + +def test_colmodernvbert_image_token_embed(vllm_runner, image_assets): + """Image input produces per-token embeddings including vision tokens.""" + with vllm_runner( + MODEL_NAME, + runner="pooling", + dtype=DTYPE, + enforce_eager=True, + ) as vllm_model: + image = image_assets[0].pil_image + inputs = vllm_model.get_inputs( + [""], + images=[image], + ) + req_outputs = vllm_model.llm.encode( + inputs, + pooling_task="token_embed", + ) + outputs = [req_output.outputs.data for req_output in req_outputs] + + assert len(outputs) == 1 + emb = torch.tensor(outputs[0]) + assert emb.dim() == 2 + assert emb.shape[1] == COLBERT_DIM + # Should have at least the image tokens (64 after pixel shuffle) + assert emb.shape[0] >= 64 diff --git a/tests/models/registry.py b/tests/models/registry.py index b37dfb6d8cd..64a0794b8ea 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -592,6 +592,9 @@ _EMBEDDING_EXAMPLE_MODELS = { ), # [Multimodal] "CLIPModel": _HfExamplesInfo("openai/clip-vit-base-patch32"), + "ColModernVBertForRetrieval": _HfExamplesInfo( + "ModernVBERT/colmodernvbert-merged", + ), "LlavaNextForConditionalGeneration": _HfExamplesInfo("royokong/e5-v"), "Phi3VForCausalLM": _HfExamplesInfo( "TIGER-Lab/VLM2Vec-Full", trust_remote_code=True diff --git a/vllm/model_executor/models/colmodernvbert.py b/vllm/model_executor/models/colmodernvbert.py new file mode 100644 index 00000000000..29efb4a5ff3 --- /dev/null +++ b/vllm/model_executor/models/colmodernvbert.py @@ -0,0 +1,430 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""ColModernVBERT: multimodal late-interaction retrieval model. + +Combines SigLIP vision encoder + ModernBERT text encoder with a pixel +shuffle connector and ColBERT-style 128-dim per-token embeddings. + +Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged +""" + +from collections.abc import Iterable, Mapping, Sequence +from typing import ClassVar, Literal + +import torch +from torch import nn +from transformers import BatchFeature + +from vllm.config import VllmConfig +from vllm.config.multimodal import BaseDummyOptions +from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed +from vllm.model_executor.model_loader.weight_utils import default_weight_loader +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import ( + MultiModalDataDict, + MultiModalFieldConfig, + MultiModalKwargsItems, +) +from vllm.multimodal.parse import ImageSize, MultiModalDataItems +from vllm.multimodal.processing import ( + BaseDummyInputsBuilder, + BaseMultiModalProcessor, + BaseProcessingInfo, + PromptIndexTargets, + PromptReplacement, + PromptUpdate, +) +from vllm.sequence import IntermediateTensors +from vllm.transformers_utils.configs.colmodernvbert import ColModernVBertConfig + +from .interfaces import MultiModalEmbeddings, SupportsMultiModal +from .interfaces_base import default_pooling_type +from .modernbert import ModernBertEmbeddings, ModernBertLayer +from .siglip import SiglipVisionModel +from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix + +# --------------------------------------------------------------------------- +# Connector: pixel shuffle + simple linear projection +# --------------------------------------------------------------------------- + + +class ColModernVBertConnector(nn.Module): + """Pixel shuffle spatial reduction followed by a linear projection. + + Reduces the vision encoder's token count by ``factor^2`` via pixel-shuffle + spatial rearrangement, then projects the concatenated channels to the text + encoder's hidden size with a single bias-free linear layer. + """ + + def __init__(self, config: ColModernVBertConfig): + super().__init__() + self.pixel_shuffle_factor = config.pixel_shuffle_factor + vision_hidden_size = config.vision_config.hidden_size + input_size = vision_hidden_size * (self.pixel_shuffle_factor**2) + output_size = config.hidden_size + self.proj = nn.Linear(input_size, output_size, bias=False) + + def pixel_shuffle(self, features: torch.Tensor) -> torch.Tensor: + """Spatial rearrangement that reduces seq length by factor^2.""" + batch_size, seq_length, hidden_size = features.shape + height = width = int(seq_length**0.5) + factor = self.pixel_shuffle_factor + + # Reshape to (B, H, W, C) + features = features.view(batch_size, height, width, hidden_size) + + # Reshape to (B, H/f, f, W/f, f, C) + features = features.view( + batch_size, height // factor, factor, width // factor, factor, hidden_size + ) + + # Permute to (B, H/f, W/f, f, f, C) + features = features.permute(0, 1, 3, 2, 4, 5) + + # Reshape to (B, H/f, W/f, C * f^2) + new_hidden_size = hidden_size * (factor**2) + features = features.reshape( + batch_size, height // factor, width // factor, new_hidden_size + ) + + return features + + def forward(self, features: torch.Tensor) -> torch.Tensor: + features = self.pixel_shuffle(features) + batch_size = features.shape[0] + features = features.reshape(batch_size, -1, features.shape[-1]) + return self.proj(features) + + +# --------------------------------------------------------------------------- +# Multimodal processing +# --------------------------------------------------------------------------- + + +class ColModernVBertProcessingInfo(BaseProcessingInfo): + def get_hf_config(self) -> ColModernVBertConfig: + return self.ctx.get_hf_config(ColModernVBertConfig) + + def get_supported_mm_limits(self) -> Mapping[str, int | None]: + return {"image": None} + + def get_image_size_with_most_features(self) -> ImageSize: + config = self.get_hf_config() + size = config.vision_config.image_size + return ImageSize(width=size, height=size) + + def get_num_image_tokens( + self, + *, + image_width: int, + image_height: int, + ) -> int: + return self.get_hf_config().image_seq_len + + +class ColModernVBertDummyInputsBuilder( + BaseDummyInputsBuilder[ColModernVBertProcessingInfo], +): + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: + return "" + + def get_dummy_mm_data( + self, + seq_len: int, + mm_counts: Mapping[str, int], + mm_options: Mapping[str, BaseDummyOptions] | None = None, + mm_processor_kwargs: Mapping[str, object] | None = None, + ) -> MultiModalDataDict: + num_images = mm_counts.get("image", 0) + target_width, target_height = self.info.get_image_size_with_most_features() + image_overrides = mm_options.get("image") if mm_options else None + return { + "image": self._get_dummy_images( + width=target_width, + height=target_height, + num_images=num_images, + overrides=image_overrides, + ) + } + + +class ColModernVBertMultiModalProcessor( + BaseMultiModalProcessor[ColModernVBertProcessingInfo], +): + def _call_hf_processor( + self, + prompt: str, + mm_data: Mapping[str, object], + mm_kwargs: Mapping[str, object], + tok_kwargs: Mapping[str, object], + ) -> BatchFeature: + tokenizer = self.info.get_tokenizer() + text_encoding = tokenizer( + prompt, + return_tensors="pt", + **tok_kwargs, + ) + result = BatchFeature(data=dict(text_encoding)) + + images = mm_data.get("images") + if images: + from transformers import Idefics3ImageProcessor + + image_processor = Idefics3ImageProcessor.from_pretrained( + self.info.ctx.model_config.model, + revision=self.info.ctx.model_config.revision, + ) + image_outputs = image_processor( + images=images, + do_image_splitting=False, + return_tensors="pt", + ) + result.update(image_outputs) + + return result + + def _hf_processor_applies_updates( + self, + prompt_text: str, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + tokenization_kwargs: Mapping[str, object], + ) -> bool: + return False + + def _get_mm_fields_config( + self, + hf_inputs: BatchFeature, + hf_processor_mm_kwargs: Mapping[str, object], + ) -> Mapping[str, MultiModalFieldConfig]: + return dict( + pixel_values=MultiModalFieldConfig.batched("image"), + ) + + def _get_prompt_updates( + self, + mm_items: MultiModalDataItems, + hf_processor_mm_kwargs: Mapping[str, object], + out_mm_kwargs: MultiModalKwargsItems, + ) -> Sequence[PromptUpdate]: + config = self.info.get_hf_config() + image_token_id = config.image_token_id + num_tokens = config.image_seq_len + + def get_replacement(item_idx: int): + return [image_token_id] * num_tokens + + return [ + PromptReplacement( + modality="image", + target=PromptIndexTargets.start(), + replacement=get_replacement, + ), + ] + + +# --------------------------------------------------------------------------- +# Model +# --------------------------------------------------------------------------- + + +@MULTIMODAL_REGISTRY.register_processor( + ColModernVBertMultiModalProcessor, + info=ColModernVBertProcessingInfo, + dummy_inputs=ColModernVBertDummyInputsBuilder, +) +@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL") +class ColModernVBertForRetrieval(nn.Module, SupportsMultiModal): + """ColModernVBERT multimodal late-interaction retrieval model. + + Architecture: + Image -> SiglipVisionModel -> ColModernVBertConnector + ↓ + Text -> ModernBertEmbeddings → [merge] → ModernBertLayers → norm + ↓ + custom_text_proj → L2 norm + ↓ + per-token 128-d embeddings + """ + + is_pooling_model = True + supports_late_interaction: ClassVar[Literal[True]] = True + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__() + config: ColModernVBertConfig = vllm_config.model_config.hf_config + self.config = config + text_config = config.text_config + quant_config = vllm_config.quant_config + + # --- Vision encoder (reuses SiglipVisionModel from siglip.py) --- + self.vision_model = SiglipVisionModel( + config.vision_config, + quant_config, + prefix=maybe_prefix(prefix, "vision_model"), + ) + + # --- Connector (pixel shuffle + linear projection) --- + self.connector = ColModernVBertConnector(config) + + # --- Text encoder (built from ModernBERT components directly) --- + # We build the components individually rather than wrapping + # ``ModernBertModel`` because ``ModernBertEncoderLayer`` reads + # ``vllm_config.model_config.hf_config`` which would be + # ``ColModernVBertConfig``, not ``ModernBertConfig``. + self.text_embeddings = ModernBertEmbeddings(text_config) + self.text_layers = nn.ModuleList( + [ + ModernBertLayer( + config=text_config, + layer_id=i, + prefix=f"{prefix}.text_layers.{i}", + ) + for i in range(text_config.num_hidden_layers) + ] + ) + self.text_final_norm = nn.LayerNorm( + text_config.hidden_size, + eps=text_config.norm_eps, + bias=text_config.norm_bias, + ) + + # --- ColBERT projection (768 -> 128, with bias) --- + self.custom_text_proj = nn.Linear( + text_config.hidden_size, + config.embedding_dim, + bias=True, + dtype=vllm_config.model_config.head_dtype, + ) + + # --- Pooler (applies projection + L2 normalize) --- + pooler_config = vllm_config.model_config.pooler_config + assert pooler_config is not None + self.pooler = pooler_for_token_embed( + pooler_config, + projector=self.custom_text_proj, + ) + + # ---- multimodal --------------------------------------------------------- + + def _get_image_features( + self, + pixel_values: torch.Tensor, + ) -> torch.Tensor: + # Idefics3ImageProcessor may return (batch, tiles, C, H, W); + # flatten to (batch*tiles, C, H, W) for SiglipVisionModel. + if pixel_values.dim() == 5: + b, t, c, h, w = pixel_values.shape + pixel_values = pixel_values.reshape(b * t, c, h, w) + vision_outputs = self.vision_model( + pixel_values.to(dtype=self.vision_model.dtype), + ) + return self.connector(vision_outputs) + + def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: + pixel_values = kwargs.pop("pixel_values", None) + if pixel_values is None: + return [] + assert isinstance(pixel_values, torch.Tensor) + image_features = self._get_image_features(pixel_values) + return list(image_features) + + # ---- forward ------------------------------------------------------------ + + def forward( + self, + input_ids: torch.Tensor, + positions: torch.Tensor, + intermediate_tensors: IntermediateTensors | None = None, + inputs_embeds: torch.Tensor | None = None, + ) -> torch.Tensor: + hidden_states = self.text_embeddings(input_ids, inputs_embeds=inputs_embeds) + + for layer in self.text_layers: + hidden_states = layer(hidden_states, positions) + + return self.text_final_norm(hidden_states) + + # ---- weight loading ----------------------------------------------------- + + # Checkpoint prefix → vLLM param prefix. + # More-specific prefixes must appear before shorter ones. + hf_to_vllm_mapper = WeightsMapper( + orig_to_new_prefix={ + "model.text_model.layers.": "text_layers.", + "model.text_model.embeddings.": "text_embeddings.", + "model.text_model.final_norm.": "text_final_norm.", + "model.connector.modality_projection.": "connector.", + "model.custom_text_proj.": "custom_text_proj.", + "model.vision_model.": "vision_model.vision_model.", + "model.": "", + }, + ) + + # Checkpoint names for DecoupledEmbedding parts + _BASE_EMB = "model.text_model.embeddings.tok_embeddings.weight" + _EXTRA_EMB = ( + "model.text_model.embeddings.tok_embeddings.additional_embedding.weight" + ) + + def load_weights( + self, + weights: Iterable[tuple[str, torch.Tensor]], + ) -> set[str]: + # DecoupledEmbedding requires concatenating base + additional + # embedding tensors before loading, so we extract them first. + base_embedding_weight: torch.Tensor | None = None + additional_embedding_weight: torch.Tensor | None = None + remaining: list[tuple[str, torch.Tensor]] = [] + + for name, tensor in weights: + if name == self._BASE_EMB: + base_embedding_weight = tensor + elif name == self._EXTRA_EMB: + additional_embedding_weight = tensor + else: + remaining.append((name, tensor)) + + # Load all non-embedding weights via AutoWeightsLoader + loader = AutoWeightsLoader(self) + loaded_params = loader.load_weights( + remaining, + mapper=self.hf_to_vllm_mapper, + ) + + # Concatenate and load DecoupledEmbedding weights + if base_embedding_weight is not None: + combined = base_embedding_weight + if additional_embedding_weight is not None: + combined = torch.cat( + [base_embedding_weight, additional_embedding_weight], + dim=0, + ) + param_name = "text_embeddings.tok_embeddings.weight" + params_dict = dict(self.named_parameters()) + if param_name in params_dict: + param = params_dict[param_name] + weight_loader = getattr( + param, + "weight_loader", + default_weight_loader, + ) + weight_loader(param, combined) + loaded_params.add(param_name) + elif additional_embedding_weight is not None: + raise ValueError( + "Found 'text_model.embeddings.tok_embeddings" + ".additional_embedding.weight' but not " + "'text_model.embeddings.tok_embeddings.weight'" + ) + + # The pooler wraps ``custom_text_proj`` as its head projector. + # Mark those params as loaded under the pooler path too. + if hasattr(self, "pooler") and hasattr(self.pooler, "head"): + head = self.pooler.head + projector = getattr(head, "projector", None) + if projector is not None and isinstance(projector, nn.Module): + for pname, _ in projector.named_parameters(): + loaded_params.add(f"pooler.head.projector.{pname}") + + return loaded_params diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 598df91d996..329411d6251 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -248,6 +248,7 @@ _EMBEDDING_MODELS = { "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"), # [Multimodal] "CLIPModel": ("clip", "CLIPEmbeddingModel"), + "ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"), "LlavaNextForConditionalGeneration": ( "llava_next", "LlavaNextForConditionalGeneration", diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 852e1d2a31d..00129d52eae 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -74,6 +74,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( afmoe="AfmoeConfig", bagel="BagelConfig", chatglm="ChatGLMConfig", + colmodernvbert="ColModernVBertConfig", colqwen3="ColQwen3Config", ops_colqwen3="OpsColQwen3Config", qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig", diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index d02ab01d741..541bc4de61b 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -18,6 +18,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "AfmoeConfig": "vllm.transformers_utils.configs.afmoe", "BagelConfig": "vllm.transformers_utils.configs.bagel", "ChatGLMConfig": "vllm.transformers_utils.configs.chatglm", + "ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert", "ColQwen3Config": "vllm.transformers_utils.configs.colqwen3", "OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3", "Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3", @@ -71,6 +72,7 @@ __all__ = [ "AfmoeConfig", "BagelConfig", "ChatGLMConfig", + "ColModernVBertConfig", "ColQwen3Config", "OpsColQwen3Config", "Qwen3VLNemotronEmbedConfig", diff --git a/vllm/transformers_utils/configs/colmodernvbert.py b/vllm/transformers_utils/configs/colmodernvbert.py new file mode 100644 index 00000000000..97fad16bcf9 --- /dev/null +++ b/vllm/transformers_utils/configs/colmodernvbert.py @@ -0,0 +1,65 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Configuration for ColModernVBERT visual document retrieval model. + +ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder +with a pixel shuffle connector and ColBERT-style 128-dim per-token embeddings. + +Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged +""" + +from transformers import ModernBertConfig, PretrainedConfig, SiglipVisionConfig + + +class ColModernVBertConfig(PretrainedConfig): + model_type = "colmodernvbert" + + def __init__( + self, + embedding_dim: int = 128, + vlm_config: dict | None = None, + **kwargs, + ): + super().__init__(**kwargs) + self.embedding_dim = embedding_dim + + if vlm_config is None: + vlm_config = {} + + # Top-level VLM fields + self.image_token_id = vlm_config.get("image_token_id", 50407) + self.pixel_shuffle_factor = vlm_config.get("pixel_shuffle_factor", 4) + self.hidden_size = vlm_config.get("hidden_size", 768) + additional_vocab_size = vlm_config.get("additional_vocab_size", 40) + + # Text config (ModernBERT) + text_cfg = vlm_config.get("text_config", {}) + base_vocab = text_cfg.get("vocab_size", 50368) + self.text_config = ModernBertConfig( + vocab_size=base_vocab + additional_vocab_size, + hidden_size=text_cfg.get("hidden_size", 768), + intermediate_size=text_cfg.get("intermediate_size", 1152), + num_hidden_layers=text_cfg.get("num_hidden_layers", 22), + num_attention_heads=text_cfg.get("num_attention_heads", 12), + mlp_bias=text_cfg.get("mlp_bias", False), + max_position_embeddings=vlm_config.get("max_position_embeddings", 8192), + ) + + # Vision config (SigLIP) + vis_cfg = vlm_config.get("vision_config", {}) + self.vision_config = SiglipVisionConfig( + hidden_size=vis_cfg.get("embed_dim", 768), + image_size=vis_cfg.get("image_size", 512), + patch_size=vis_cfg.get("patch_size", 16), + num_hidden_layers=vis_cfg.get("num_hidden_layers", 12), + intermediate_size=vis_cfg.get("intermediate_size", 3072), + num_attention_heads=vis_cfg.get("num_attention_heads", 12), + ) + + @property + def image_seq_len(self) -> int: + ps = self.vision_config.image_size // self.vision_config.patch_size + return (ps * ps) // (self.pixel_shuffle_factor**2) + + def get_text_config(self, **kwargs): + return self.text_config From cbd95a2dd19a5786e8b6572a8e6599c8375c4abf Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sun, 22 Feb 2026 12:26:48 +0800 Subject: [PATCH 69/74] [Benchmark] Use `sns.relplot` for plotting (#35027) Signed-off-by: DarkLight1337 --- vllm/benchmarks/sweep/plot.py | 76 ++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/vllm/benchmarks/sweep/plot.py b/vllm/benchmarks/sweep/plot.py index 376adbb0851..87323757ed7 100644 --- a/vllm/benchmarks/sweep/plot.py +++ b/vllm/benchmarks/sweep/plot.py @@ -346,7 +346,45 @@ def _plot_fig( else "(All)" ) - g = sns.FacetGrid(df, row="row_group", col="col_group", height=fig_height) + if len(curve_by) <= 3: + hue, style, size, *_ = (*curve_by, None, None, None) + + g = sns.relplot( + df, + x=var_x, + y=var_y, + hue=hue, + style=style, + size=size, + markers=True, + errorbar="sd" if error_bars else None, + kind="line", + row="row_group", + col="col_group", + height=fig_height, + ) + else: + df["curve_group"] = ( + pd.concat( + [k + "=" + df[k].astype(str) for k in curve_by], + axis=1, + ).agg("\n".join, axis=1) + if curve_by + else "(All)" + ) + + g = sns.relplot( + df, + x=var_x, + y=var_y, + hue="curve_group", + markers=True, + errorbar="sd" if error_bars else None, + kind="line", + row="row_group", + col="col_group", + height=fig_height, + ) if row_by and col_by: g.set_titles("{row_name}\n{col_name}") @@ -362,42 +400,6 @@ def _plot_fig( if scale_y: g.set(yscale=scale_y) - if len(curve_by) <= 3: - hue, style, size, *_ = (*curve_by, None, None, None) - - g.map_dataframe( - sns.lineplot, - x=var_x, - y=var_y, - hue=hue, - style=style, - size=size, - markers=True, - errorbar="sd" if error_bars else None, - ) - - g.add_legend(title=hue) - else: - df["curve_group"] = ( - pd.concat( - [k + "=" + df[k].astype(str) for k in curve_by], - axis=1, - ).agg("\n".join, axis=1) - if curve_by - else "(All)" - ) - - g.map_dataframe( - sns.lineplot, - x=var_x, - y=var_y, - hue="curve_group", - markers=True, - errorbar="sd" if error_bars else None, - ) - - g.add_legend() - g.savefig(fig_path, dpi=fig_dpi) plt.close(g.figure) From 30132cd144af8876e7c0d2aac28cabaea3710254 Mon Sep 17 00:00:00 2001 From: Xiao Li Date: Sat, 21 Feb 2026 21:11:54 -0800 Subject: [PATCH 70/74] Fix apply_top_k_top_p_triton called by non-cuda logits Tensor (#35030) Signed-off-by: Xiao Li --- vllm/v1/sample/ops/topk_topp_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/v1/sample/ops/topk_topp_sampler.py b/vllm/v1/sample/ops/topk_topp_sampler.py index 33f7090e4e3..dcae8f974f2 100644 --- a/vllm/v1/sample/ops/topk_topp_sampler.py +++ b/vllm/v1/sample/ops/topk_topp_sampler.py @@ -248,7 +248,7 @@ def apply_top_k_top_p( if p is None and k is None: return logits - if HAS_TRITON and logits.shape[0] >= 8: + if HAS_TRITON and logits.shape[0] >= 8 and logits.is_cuda: return apply_top_k_top_p_triton(logits, k, p) # Use pytorch sort implementation for small batch sizes. From 2cbf9656ce6013f7b531bc5a0909d03b88c14862 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Sat, 21 Feb 2026 21:42:50 -0800 Subject: [PATCH 71/74] [Model Runner V2] Enable CUDA graph for Eagle3 (#35040) Signed-off-by: Woosuk Kwon --- vllm/v1/worker/gpu/cudagraph_utils.py | 46 +++++++++++++++++++++++---- vllm/v1/worker/gpu/model_runner.py | 7 ++-- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/vllm/v1/worker/gpu/cudagraph_utils.py b/vllm/v1/worker/gpu/cudagraph_utils.py index 7bba7ffb928..5665937a03e 100644 --- a/vllm/v1/worker/gpu/cudagraph_utils.py +++ b/vllm/v1/worker/gpu/cudagraph_utils.py @@ -25,10 +25,17 @@ from vllm.v1.worker.utils import AttentionGroup class CudaGraphManager: - def __init__(self, vllm_config: VllmConfig, uses_mrope: bool, device: torch.device): + def __init__( + self, + vllm_config: VllmConfig, + uses_mrope: bool, + use_aux_hidden_state_outputs: bool, + device: torch.device, + ): self.vllm_config = vllm_config self.scheduler_config = vllm_config.scheduler_config self.uses_mrope = uses_mrope + self.use_aux_hidden_state_outputs = use_aux_hidden_state_outputs self.device = device self.max_model_len = vllm_config.model_config.max_model_len @@ -63,6 +70,7 @@ class CudaGraphManager: if self.cudagraph_mode != CUDAGraphMode.NONE: self.pool = torch.cuda.graph_pool_handle() self.hidden_states: torch.Tensor | None = None + self.aux_hidden_states: list[torch.Tensor] = [] def needs_capture(self) -> bool: return len(self.cudagraph_sizes) > 0 @@ -134,13 +142,22 @@ class CudaGraphManager: num_tokens_across_dp=num_tokens_across_dp, slot_mapping=slot_mappings, ): - hidden_states = model( + model_output = model( input_ids=input_ids, positions=positions, inputs_embeds=inputs_embeds, ) - if self.hidden_states is None: - self.hidden_states = torch.empty_like(hidden_states) + if self.use_aux_hidden_state_outputs: + hidden_states, aux_hidden_states = model_output + else: + hidden_states = model_output + aux_hidden_states = None + + # Allocate output buffers if not already done. + if self.hidden_states is None: + self.hidden_states = torch.empty_like(hidden_states) + if self.use_aux_hidden_state_outputs and not self.aux_hidden_states: + self.aux_hidden_states = [torch.empty_like(x) for x in aux_hidden_states] capture_fn( num_tokens=num_tokens, @@ -183,13 +200,23 @@ class CudaGraphManager: ), torch.cuda.graph(graph, self.pool), ): - hidden_states = model( + model_output = model( input_ids=input_ids, positions=positions, inputs_embeds=inputs_embeds, ) + if self.use_aux_hidden_state_outputs: + hidden_states, aux_hidden_states = model_output + else: + hidden_states = model_output + aux_hidden_states = None + + # Copy outputs to the output buffers. assert self.hidden_states is not None self.hidden_states[:num_tokens] = hidden_states + if self.use_aux_hidden_state_outputs: + for i, aux_hidden in enumerate(aux_hidden_states): + self.aux_hidden_states[i][:num_tokens] = aux_hidden self.graphs[num_tokens] = graph def _capture_piecewise_graph( @@ -298,11 +325,16 @@ class CudaGraphManager: cudagraph_size = None return cudagraph_mode, cudagraph_size - def run_fullgraph(self, num_tokens: int) -> torch.Tensor: + def run_fullgraph( + self, num_tokens: int + ) -> torch.Tensor | tuple[torch.Tensor, list[torch.Tensor]]: assert num_tokens in self.graphs, f"No cudagraph for {num_tokens} tokens" self.graphs[num_tokens].replay() assert self.hidden_states is not None - return self.hidden_states[:num_tokens] + hidden_states = self.hidden_states[:num_tokens] + if not self.use_aux_hidden_state_outputs: + return hidden_states + return hidden_states, [x[:num_tokens] for x in self.aux_hidden_states] def get_cudagraph_sizes( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index b909b90adf1..cdea0b2aa8c 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -197,7 +197,10 @@ class GPUModelRunner(LoRAModelRunnerMixin): # CUDA graphs. self.cudagraph_manager = CudaGraphManager( - self.vllm_config, self.uses_mrope, self.device + self.vllm_config, + self.uses_mrope, + self.use_aux_hidden_state_outputs, + self.device, ) # Structured outputs worker. self.structured_outputs_worker = StructuredOutputsWorker( @@ -1044,7 +1047,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): aux_hidden_states, input_batch, kv_connector_output, - ) + ) # type: ignore return None @torch.inference_mode() From 40f88d8318aea1792ac7eabfe33241fd26660be7 Mon Sep 17 00:00:00 2001 From: Roger Wang Date: Sat, 21 Feb 2026 23:15:35 -0800 Subject: [PATCH 72/74] [Bugfix] Fix Qwen3/Qwen3.5 Reasoning Parser (#34779) Signed-off-by: Roger Wang --- .../reasoning/test_qwen3_reasoning_parser.py | 133 ++++++++++++++++-- .../openai/chat_completion/serving.py | 47 ++++--- vllm/reasoning/qwen3_reasoning_parser.py | 100 ++++++++++--- 3 files changed, 233 insertions(+), 47 deletions(-) diff --git a/tests/reasoning/test_qwen3_reasoning_parser.py b/tests/reasoning/test_qwen3_reasoning_parser.py index 92a8b6ab376..db2bc16ffbb 100644 --- a/tests/reasoning/test_qwen3_reasoning_parser.py +++ b/tests/reasoning/test_qwen3_reasoning_parser.py @@ -4,46 +4,79 @@ import pytest from transformers import AutoTokenizer -from tests.reasoning.utils import run_reasoning_extraction +from tests.reasoning.utils import ( + StreamingReasoningReconstructor, + run_reasoning_extraction, + run_reasoning_extraction_streaming, +) from vllm.reasoning import ReasoningParser, ReasoningParserManager parser_name = "qwen3" start_token = "" end_token = "" -REASONING_MODEL_NAME = "Qwen/Qwen3-0.6B" +REASONING_MODEL_NAMES = [ + "Qwen/Qwen3-0.6B", + "Qwen/Qwen3.5-397B-A17B", + "Qwen/Qwen3-4B-Thinking-2507", +] -@pytest.fixture(scope="module") -def qwen3_tokenizer(): - return AutoTokenizer.from_pretrained(REASONING_MODEL_NAME) +@pytest.fixture(scope="module", params=REASONING_MODEL_NAMES) +def qwen3_tokenizer(request): + return AutoTokenizer.from_pretrained(request.param) -# 带 ，非stream +# --- in prompt, only in output (typical) --- + +WITHOUT_START_TOKEN = { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", +} +WITHOUT_START_TOKEN_STREAM = { + "output": "This is a reasoning sectionThis is the rest", + "reasoning": "This is a reasoning section", + "content": "This is the rest", +} +WITHOUT_START_TOKEN_COMPLETE_REASONING = { + "output": "This is a reasoning section", + "reasoning": "This is a reasoning section", + "content": None, +} + +# --- present in output (old template / edge case) --- + WITH_THINK = { "output": "This is a reasoning sectionThis is the rest", "reasoning": "This is a reasoning section", "content": "This is the rest", } -# 带 ，stream WITH_THINK_STREAM = { "output": "This is a reasoning sectionThis is the rest", "reasoning": "This is a reasoning section", "content": "This is the rest", } -# 不带 ，非stream + +# --- No think tokens at all (thinking disabled) --- + WITHOUT_THINK = { "output": "This is the rest", "reasoning": None, "content": "This is the rest", } -# 不带 ，stream +# In streaming, the parser cannot distinguish "thinking disabled" from +# "reasoning in progress" when no think tokens have appeared yet. +# It assumes reasoning. The serving layer handles the "thinking disabled" +# case by checking prompt_is_reasoning_end_arr before calling the parser. WITHOUT_THINK_STREAM = { "output": "This is the rest", - "reasoning": None, - "content": "This is the rest", + "reasoning": "This is the rest", + "content": None, } +# --- Edge cases --- + COMPLETE_REASONING = { "output": "This is a reasoning section", "reasoning": "This is a reasoning section", @@ -57,7 +90,7 @@ MULTILINE_REASONING = { ONLY_OPEN_TAG = { "output": "This is a reasoning section", "reasoning": None, - "content": "This is a reasoning section", + "content": "This is a reasoning section", } ONLY_OPEN_TAG_STREAM = { @@ -67,6 +100,26 @@ ONLY_OPEN_TAG_STREAM = { } TEST_CASES = [ + pytest.param( + False, + WITHOUT_START_TOKEN, + id="without_start_token", + ), + pytest.param( + True, + WITHOUT_START_TOKEN_STREAM, + id="without_start_token_stream", + ), + pytest.param( + False, + WITHOUT_START_TOKEN_COMPLETE_REASONING, + id="without_start_token_complete_reasoning", + ), + pytest.param( + True, + WITHOUT_START_TOKEN_COMPLETE_REASONING, + id="without_start_token_complete_reasoning_stream", + ), pytest.param( False, WITH_THINK, @@ -140,3 +193,59 @@ def test_reasoning( assert reasoning == param_dict["reasoning"] assert content == param_dict["content"] + + +# Multi-token delta tests: simulate real-world streaming where a single +# delta can contain multiple tokens (e.g., speculative decoding). +MULTI_TOKEN_DELTA_CASES = [ + pytest.param( + # grouped with following text in one delta + ["This is a reasoning section", "", "This is the rest"], + "This is a reasoning section", + "This is the rest", + id="start_token_grouped_with_text", + ), + pytest.param( + # grouped with following content in one delta + ["reasoning section", "This is the rest"], + "reasoning section", + "This is the rest", + id="end_token_grouped_with_content", + ), + pytest.param( + # and in the same delta, no content after + ["reasoning"], + "reasoning", + None, + id="start_and_end_in_one_delta_no_content", + ), + pytest.param( + # No start token, end grouped with content (Qwen3.5 style) + ["reasoning section", "content"], + "reasoning section", + "content", + id="no_start_end_grouped_with_content", + ), +] + + +@pytest.mark.parametrize( + "deltas, expected_reasoning, expected_content", MULTI_TOKEN_DELTA_CASES +) +def test_reasoning_streaming_multi_token_deltas( + deltas: list[str], + expected_reasoning: str | None, + expected_content: str | None, + qwen3_tokenizer, +): + """Test that multi-token deltas don't leak into reasoning.""" + parser: ReasoningParser = ReasoningParserManager.get_reasoning_parser(parser_name)( + qwen3_tokenizer + ) + + reconstructor: StreamingReasoningReconstructor = run_reasoning_extraction_streaming( + parser, deltas + ) + + assert reconstructor.reasoning == expected_reasoning + assert (reconstructor.other_content or None) == expected_content diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index f1af14dd902..ef5620bb869 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -900,6 +900,17 @@ class OpenAIServingChat(OpenAIServing): harmony_tools_streamed[i] |= tools_streamed_flag # handle streaming deltas for tools with named tool_choice elif tool_choice_function_name: + # When encountering think end id in prompt_token_ids + # i.e {"enable_thinking": False}, + # check BEFORE calling the parser to avoid a spurious + # reasoning delta on the first chunk. + if ( + reasoning_parser + and not reasoning_end_arr[i] + and prompt_is_reasoning_end_arr[i] + ): + reasoning_end_arr[i] = True + if ( reasoning_parser and not reasoning_end_arr[i] @@ -918,16 +929,11 @@ class OpenAIServingChat(OpenAIServing): output.token_ids, ) ) - # When encountering think end id in delta_token_ids - # or think end id in prompt_token_ids - # i.e {"enable_thinking": False}, + # When encountering think end id in delta_token_ids, # set reasoning status to end. # Only keep 'content', remove 'reasoning'. - if ( - reasoning_parser.is_reasoning_end( - as_list(output.token_ids) - ) - or prompt_is_reasoning_end_arr[i] + if reasoning_parser.is_reasoning_end( + as_list(output.token_ids) ): reasoning_end_arr[i] = True if delta_message and delta_message.content: @@ -1116,14 +1122,23 @@ class OpenAIServingChat(OpenAIServing): # when only reasoning elif reasoning_parser: - delta_message = reasoning_parser.extract_reasoning_streaming( - previous_text, - current_text, - delta_text, - previous_token_ids, - current_token_ids, - output.token_ids, - ) + # When encountering think end id in prompt_token_ids + # i.e {"enable_thinking": False}, + # set reasoning status to end. + # Route all generated tokens as content directly. + if prompt_is_reasoning_end_arr[i]: + delta_message = DeltaMessage(content=delta_text) + else: + delta_message = ( + reasoning_parser.extract_reasoning_streaming( + previous_text, + current_text, + delta_text, + previous_token_ids, + current_token_ids, + output.token_ids, + ) + ) # handle streaming just a content delta else: delta_message = DeltaMessage(content=delta_text) diff --git a/vllm/reasoning/qwen3_reasoning_parser.py b/vllm/reasoning/qwen3_reasoning_parser.py index fc12ce540d0..0c09d40999e 100644 --- a/vllm/reasoning/qwen3_reasoning_parser.py +++ b/vllm/reasoning/qwen3_reasoning_parser.py @@ -1,9 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from collections.abc import Sequence + from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) +from vllm.entrypoints.openai.engine.protocol import DeltaMessage from vllm.entrypoints.openai.responses.protocol import ( ResponsesRequest, ) @@ -12,13 +15,22 @@ from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser class Qwen3ReasoningParser(BaseThinkingReasoningParser): """ - Reasoning parser for the Qwen3 model. + Reasoning parser for the Qwen3/Qwen3.5 model family. - The Qwen3 model uses ... tokens to denote reasoning text - within its output. The model provides a strict switch to disable reasoning - output via the 'enable_thinking=False' parameter. This parser extracts the - reasoning content enclosed by and tokens from the model's - output. + The Qwen3 model family uses ... tokens to denote reasoning + text. Starting with Qwen3.5, the chat template places in the + prompt so only appears in the generated output. The model + provides a strict switch to disable reasoning output via the + 'enable_thinking=False' parameter. + + When thinking is disabled, the template places \\n\\n\\n\\n + in the prompt. The serving layer detects this via prompt_is_reasoning_end + and routes deltas as content without calling the streaming parser. + + NOTE: Models up to the 2507 release (e.g., Qwen/Qwen3-235B-A22B-Instruct-2507) + use an older chat template where the model generates itself. + This parser handles both styles: if appears in the generated output + it is stripped before extraction (non-streaming) or skipped (streaming). """ @property @@ -37,31 +49,27 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): """ Extract reasoning content from the model output. - Qwen3 has stricter requirements - it needs both start and end tokens - to be present, unlike other models that work with just the end token. + The token is placed in the prompt by the chat template, + so typically only appears in the generated output. + If is present (e.g. from a different template), it is + stripped before extraction. - For text abcxyz: - - 'abc' goes to reasoning - - 'xyz' goes to content + When thinking is disabled (no in output), returns + (None, model_output) to indicate all output is content. Returns: tuple[Optional[str], Optional[str]]: reasoning content and content """ - # Check if the model output contains both and tokens. - if self.start_token not in model_output or self.end_token not in model_output: - return None, model_output - - # Check if the is present in the model output, remove it - # if it is present. + # Strip if present in the generated output. model_output_parts = model_output.partition(self.start_token) model_output = ( model_output_parts[2] if model_output_parts[1] else model_output_parts[0] ) - # Check if the model output contains the tokens. - # If the end token is not found, return the model output as is. if self.end_token not in model_output: + # No end token means thinking is disabled or the model + # did not produce reasoning. Treat everything as content. return None, model_output # Extract reasoning content from the model output. @@ -69,3 +77,57 @@ class Qwen3ReasoningParser(BaseThinkingReasoningParser): final_content = content or None return reasoning, final_content + + def extract_reasoning_streaming( + self, + previous_text: str, + current_text: str, + delta_text: str, + previous_token_ids: Sequence[int], + current_token_ids: Sequence[int], + delta_token_ids: Sequence[int], + ) -> DeltaMessage | None: + """ + Extract reasoning content from a streaming delta. + + Since is placed in the prompt by the chat template, all + generated tokens before are reasoning and tokens after + are content. + + NOTE: When thinking is disabled, no think tokens appear in the + generated output. The serving layer detects this via + prompt_is_reasoning_end and routes deltas as content without + calling this method. + """ + # Strip from delta if present (old template / edge case + # where the model generates itself). + if self.start_token_id in delta_token_ids: + start_idx = delta_text.find(self.start_token) + if start_idx >= 0: + delta_text = delta_text[start_idx + len(self.start_token) :] + + if self.end_token_id in delta_token_ids: + # End token in this delta: split reasoning from content. + end_index = delta_text.find(self.end_token) + if end_index >= 0: + reasoning = delta_text[:end_index] + content = delta_text[end_index + len(self.end_token) :] + if not reasoning and not content: + return None + return DeltaMessage( + reasoning=reasoning if reasoning else None, + content=content if content else None, + ) + # end_token_id in IDs but not in text (already stripped) + return None + + # No end token in this delta. + if not delta_text: + # Nothing left after stripping start token. + return None + elif self.end_token_id in previous_token_ids: + # End token already passed: everything is content now. + return DeltaMessage(content=delta_text) + else: + # No end token yet: still in reasoning phase. + return DeltaMessage(reasoning=delta_text) From a8a47c17b68fbd4229a86cc1d4202ebc94bdb9fe Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 22 Feb 2026 03:03:44 -0600 Subject: [PATCH 73/74] [ROCm][CI] Fix flaky embedding chat test by using tolerance-based comparison (#35050) Signed-off-by: Andreas Karatzas --- .../entrypoints/pooling/embed/test_online.py | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/entrypoints/pooling/embed/test_online.py b/tests/entrypoints/pooling/embed/test_online.py index d2a5974b757..89341670c7f 100644 --- a/tests/entrypoints/pooling/embed/test_online.py +++ b/tests/entrypoints/pooling/embed/test_online.py @@ -58,13 +58,19 @@ if current_platform.is_rocm(): torch.backends.cuda.enable_mem_efficient_sdp(False) torch.backends.cuda.enable_math_sdp(True) +# On ROCm, floating-point reductions in attention and GEMM kernels are +# non-associative and sensitive to batch geometry. Force LLM instances +# into an identical, deterministic execution mode: +ROCM_DETERMINISM_ARGS: list[str] = ( + ["--max-num-seqs", "1"] if current_platform.is_rocm() else [] +) + @pytest.fixture(scope="module") def server(): args = [ "--runner", "pooling", - # use half precision for speed and memory savings in CI environment "--dtype", DTYPE, "--enforce-eager", @@ -72,12 +78,9 @@ def server(): "512", "--chat-template", DUMMY_CHAT_TEMPLATE, + *ROCM_DETERMINISM_ARGS, ] - # ROCm: Use Flex Attention to support encoder-only self-attention. - if current_platform.is_rocm(): - args.extend(["--attention-backend", "FLEX_ATTENTION"]) - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server @@ -343,8 +346,15 @@ async def test_chat_request( assert chat_embeddings.id is not None assert completion_embeddings.id is not None assert chat_embeddings.created <= completion_embeddings.created - assert chat_embeddings.model_dump(exclude={"id", "created"}) == ( - completion_embeddings.model_dump(exclude={"id", "created"}) + # Use tolerance-based comparison for embeddings + check_embeddings_close( + embeddings_0_lst=[d.embedding for d in chat_embeddings.data], + embeddings_1_lst=[d.embedding for d in completion_embeddings.data], + name_0="chat", + name_1="completion", + ) + assert chat_embeddings.model_dump(exclude={"id", "created", "data"}) == ( + completion_embeddings.model_dump(exclude={"id", "created", "data"}) ) # test add_generation_prompt From dd8c3a7fb2b2448d04bb00934f2cacf43bb14c3b Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Sun, 22 Feb 2026 04:07:18 -0600 Subject: [PATCH 74/74] [ROCm][CI] Fix realtime test timeouts caused by aiter JIT compilation delays (#35052) Signed-off-by: Andreas Karatzas --- .../openai/test_realtime_validation.py | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/openai/test_realtime_validation.py b/tests/entrypoints/openai/test_realtime_validation.py index 8f12a3764c1..273a034e177 100644 --- a/tests/entrypoints/openai/test_realtime_validation.py +++ b/tests/entrypoints/openai/test_realtime_validation.py @@ -4,6 +4,7 @@ import asyncio import base64 import json +import warnings import librosa import numpy as np @@ -85,7 +86,41 @@ async def test_multi_chunk_streaming( await send_event(ws, {"type": "session.update", "model": model_name}) - # Send commit to start transcription + # Wait for the server to acknowledge the session update. + try: + while True: + event = await receive_event(ws, timeout=5.0) + if event["type"] == "session.updated": + break + except TimeoutError: + warnings.warn( + f"session.updated not received within {5.0}s after " + "session.update. The server may not implement this event.", + stacklevel=2, + ) + + # (ROCm) Warm-up: send a non-final commit (required to start + # transcription) with a small audio chunk to trigger aiter + # compilation on first use. + await send_event(ws, {"type": "input_audio_buffer.commit"}) + await send_event( + ws, + { + "type": "input_audio_buffer.append", + "audio": mary_had_lamb_audio_chunks[0], + }, + ) + await send_event(ws, {"type": "input_audio_buffer.commit", "final": True}) + + # (ROCm) Drain all warm-up responses with generous timeout for + # JIT compilation + warmup_done = False + while not warmup_done: + event = await receive_event(ws, timeout=360.0) + if event["type"] in ("transcription.done", "error"): + warmup_done = True + + # Now send the real test audio await send_event(ws, {"type": "input_audio_buffer.commit"}) # Send multiple audio chunks @@ -153,6 +188,18 @@ async def test_empty_commit_does_not_crash_engine( await send_event(ws, {"type": "session.update", "model": model_name}) + try: + while True: + event = await receive_event(ws, timeout=5.0) + if event["type"] == "session.updated": + break + except TimeoutError: + warnings.warn( + f"session.updated not received within {5.0}s after " + "session.update. The server may not implement this event.", + stacklevel=2, + ) + # Start generation without sending any audio await send_event(ws, {"type": "input_audio_buffer.commit"}) @@ -161,7 +208,8 @@ async def test_empty_commit_does_not_crash_engine( # We should get *some* response (error or empty transcription), # but the engine must NOT crash. - event = await receive_event(ws, timeout=30.0) + # (ROCm) Use generous timeout for first request (aiter JIT compilation) + event = await receive_event(ws, timeout=360.0) assert event["type"] in ( "error", "transcription.done", @@ -176,6 +224,19 @@ async def test_empty_commit_does_not_crash_engine( await send_event(ws, {"type": "session.update", "model": model_name}) + try: + while True: + event = await receive_event(ws, timeout=5.0) + if event["type"] == "session.updated": + break + except TimeoutError: + warnings.warn( + f"session.updated not received within {5.0}s after " + "session.update. The server may not implement this event.", + stacklevel=2, + ) + + # Start transcription await send_event(ws, {"type": "input_audio_buffer.commit"}) for chunk in mary_had_lamb_audio_chunks: