diff --git a/tests/entrypoints/serve/utils/test_ssl_cert_refresher.py b/tests/entrypoints/serve/utils/test_ssl_cert_refresher.py
index 57a856ce118..8f5251374a6 100644
--- a/tests/entrypoints/serve/utils/test_ssl_cert_refresher.py
+++ b/tests/entrypoints/serve/utils/test_ssl_cert_refresher.py
@@ -41,6 +41,28 @@ def touch_file(path: str) -> None:
     Path(path).touch()
 
 
+async def wait_for_counts(
+    ssl_context: MockSSLContext,
+    *,
+    cert_chain_count: int,
+    ca_count: int,
+    timeout: float = 5.0,
+) -> None:
+    deadline = asyncio.get_running_loop().time() + timeout
+    while True:
+        if (
+            ssl_context.load_cert_chain_count >= cert_chain_count
+            and ssl_context.load_ca_count >= ca_count
+        ):
+            return
+
+        if asyncio.get_running_loop().time() >= deadline:
+            assert ssl_context.load_cert_chain_count >= cert_chain_count
+            assert ssl_context.load_ca_count >= ca_count
+
+        await asyncio.sleep(0.05)
+
+
 @pytest.mark.asyncio
 async def test_ssl_refresher():
     ssl_context = MockSSLContext()
@@ -53,20 +75,28 @@ async def test_ssl_refresher():
     assert ssl_context.load_ca_count == 0
 
     touch_file(key_path)
-    await asyncio.sleep(1)
-    assert ssl_context.load_cert_chain_count == 1
+    await wait_for_counts(
+        ssl_context,
+        cert_chain_count=1,
+        ca_count=0,
+    )
     assert ssl_context.load_ca_count == 0
 
     touch_file(cert_path)
     touch_file(ca_path)
-    await asyncio.sleep(1)
-    assert ssl_context.load_cert_chain_count == 2
-    assert ssl_context.load_ca_count == 1
+    await wait_for_counts(
+        ssl_context,
+        cert_chain_count=2,
+        ca_count=1,
+    )
 
     ssl_refresher.stop()
+    await asyncio.sleep(0)
+    cert_chain_count = ssl_context.load_cert_chain_count
+    ca_count = ssl_context.load_ca_count
 
     touch_file(cert_path)
     touch_file(ca_path)
     await asyncio.sleep(1)
-    assert ssl_context.load_cert_chain_count == 2
-    assert ssl_context.load_ca_count == 1
+    assert ssl_context.load_cert_chain_count == cert_chain_count
+    assert ssl_context.load_ca_count == ca_count