From 33d7cbe02ca100d3f0314cd22f4342d5cd23ba15 Mon Sep 17 00:00:00 2001 From: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Date: Sat, 23 May 2026 19:37:24 -0400 Subject: [PATCH] [Model Runner v2] Force v1 runner for tests (#43233) Signed-off-by: yewentao256 --- tests/compile/correctness_e2e/test_async_tp.py | 16 ++++++++++++++-- .../correctness_e2e/test_sequence_parallel.py | 8 +++++++- .../compile/fullgraph/test_basic_correctness.py | 3 ++- tests/distributed/test_pipeline_parallel.py | 9 ++++++++- tests/models/quantization/test_bitsandbytes.py | 6 +++++- tests/utils.py | 14 ++++++++++++++ 6 files changed, 50 insertions(+), 6 deletions(-) diff --git a/tests/compile/correctness_e2e/test_async_tp.py b/tests/compile/correctness_e2e/test_async_tp.py index 932e513258d..28c7eb6fbc2 100644 --- a/tests/compile/correctness_e2e/test_async_tp.py +++ b/tests/compile/correctness_e2e/test_async_tp.py @@ -92,7 +92,13 @@ def test_async_tp_pass_correctness( "mp", ] - compare_two_settings(model_id, async_tp_args, tp_args, method="generate") + compare_two_settings( + model_id, + async_tp_args, + tp_args, + method="generate", + force_v1_runner=True, + ) @create_new_process_for_each_test() @@ -154,4 +160,10 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch): "mp", ] - compare_two_settings(NVFP4_MODEL_ID, async_tp_args, tp_args, method="generate") + compare_two_settings( + NVFP4_MODEL_ID, + async_tp_args, + tp_args, + method="generate", + force_v1_runner=True, + ) diff --git a/tests/compile/correctness_e2e/test_sequence_parallel.py b/tests/compile/correctness_e2e/test_sequence_parallel.py index 295277ddacf..e320f5a1120 100644 --- a/tests/compile/correctness_e2e/test_sequence_parallel.py +++ b/tests/compile/correctness_e2e/test_sequence_parallel.py @@ -294,7 +294,13 @@ def _compare_sp( "mp", ] - compare_two_settings(model_id, tp_sp_args, tp_args, method=method) + compare_two_settings( + model_id, + tp_sp_args, + tp_args, + method=method, + force_v1_runner=True, + ) SP_TEXT_GENERATION_MODELS = { diff --git a/tests/compile/fullgraph/test_basic_correctness.py b/tests/compile/fullgraph/test_basic_correctness.py index bec0032871b..35989dcde1d 100644 --- a/tests/compile/fullgraph/test_basic_correctness.py +++ b/tests/compile/fullgraph/test_basic_correctness.py @@ -145,6 +145,7 @@ def test_compile_correctness( all_args, all_envs, method=method if method != "generate" else "generate_close", + force_v1_runner=True, ) all_envs.clear() all_args.clear() @@ -158,4 +159,4 @@ def test_compile_correctness( all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"]) all_envs.append({}) - compare_all_settings(model, all_args, all_envs, method=method) + compare_all_settings(model, all_args, all_envs, method=method, force_v1_runner=True) diff --git a/tests/distributed/test_pipeline_parallel.py b/tests/distributed/test_pipeline_parallel.py index 3a05440e41c..2742663093f 100644 --- a/tests/distributed/test_pipeline_parallel.py +++ b/tests/distributed/test_pipeline_parallel.py @@ -349,7 +349,14 @@ def _compare_tp( "mp", ] - compare_two_settings(model_id, pp_args, tp_args, pp_env, tp_env, method=method) + compare_two_settings( + model_id, + pp_args, + tp_args, + pp_env, + tp_env, + method=method, + ) @pytest.mark.parametrize( diff --git a/tests/models/quantization/test_bitsandbytes.py b/tests/models/quantization/test_bitsandbytes.py index de4f19aff5c..d6f2b86c7af 100644 --- a/tests/models/quantization/test_bitsandbytes.py +++ b/tests/models/quantization/test_bitsandbytes.py @@ -137,7 +137,11 @@ def test_load_pp_4bit_bnb_model(model_name, description) -> None: "--pipeline-parallel-size", "2", ] - compare_two_settings(model_name, common_args, pp_args) + compare_two_settings( + model_name, + common_args, + pp_args, + ) @pytest.mark.skipif( diff --git a/tests/utils.py b/tests/utils.py index fb3bbfd9162..7fa7fd8ad85 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1117,6 +1117,7 @@ def compare_two_settings( method: str = "generate", max_wait_seconds: float | None = None, include_seeded_sampling: bool = True, + force_v1_runner: bool = False, ) -> None: """ Launch API server with two different sets of arguments/environments @@ -1130,6 +1131,9 @@ def compare_two_settings( env2: The second set of environment variables to pass to the API server. include_seeded_sampling: Whether to include temperature=1.0 seeded sampling checks in the default generate comparison. + force_v1_runner: Whether to pin all compared settings to the v1 model + runner to avoid mixing model runner differences into correctness + tests. """ compare_all_settings( @@ -1139,6 +1143,7 @@ def compare_two_settings( method=method, max_wait_seconds=max_wait_seconds, include_seeded_sampling=include_seeded_sampling, + force_v1_runner=force_v1_runner, ) @@ -1150,6 +1155,7 @@ def compare_all_settings( method: str = "generate", max_wait_seconds: float | None = None, include_seeded_sampling: bool = True, + force_v1_runner: bool = False, ) -> None: """ Launch API server with several different sets of arguments/environments @@ -1160,8 +1166,16 @@ def compare_all_settings( all_envs: A list of environment dictionaries to pass to the API server. include_seeded_sampling: Whether to include temperature=1.0 seeded sampling checks in the default generate comparison. + force_v1_runner: Whether to pin all compared settings to the v1 model + runner to avoid mixing model runner differences into correctness + tests. """ + if force_v1_runner: + all_envs = [ + {"VLLM_USE_V2_MODEL_RUNNER": "0", **(env or {})} for env in all_envs + ] + trust_remote_code = False for args in all_args: if "--trust-remote-code" in args: