From 7f4c52f2ba38a77d5341fb90a31c2cedfc0066b8 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 28 Jul 2026 18:41:48 -0700 Subject: [PATCH] [CI] Add comment-based Buildkite triggers (#50132) Signed-off-by: khluu Co-authored-by: OpenAI Codex --- .github/workflows/new_pr_bot.yml | 4 +- .github/workflows/pre-commit.yml | 2 +- .github/workflows/run-ci-command.yml | 40 ++ .github/workflows/scripts/run_ci_command.py | 580 ++++++++++++++++++ .../workflows/scripts/test_run_ci_command.py | 362 +++++++++++ docs/contributing/README.md | 6 +- 6 files changed, 989 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/run-ci-command.yml create mode 100644 .github/workflows/scripts/run_ci_command.py create mode 100644 .github/workflows/scripts/test_run_ci_command.py diff --git a/.github/workflows/new_pr_bot.yml b/.github/workflows/new_pr_bot.yml index 4124583d96d..a2f09eb8a45 100644 --- a/.github/workflows/new_pr_bot.yml +++ b/.github/workflows/new_pr_bot.yml @@ -80,9 +80,9 @@ jobs: '', '\u{1f4ac} Join our developer Slack at https://slack.vllm.ai to discuss your PR in `#pr-reviews`, coordinate on features in `#feat-` channels, or join special interest groups in `#sig-` channels.', '', - 'PRs do not trigger a full CI run by default. Once the PR is approved and ready to go, your PR reviewer(s) can run CI to test the changes comprehensively before merging.', + 'PRs do not trigger a full CI run by default. Reviewers with write access and configured trusted contributors can comment `/ci run` whenever CI signals are needed.', '', - 'To run CI, PR reviewers can either: Add `ready` label to the PR or enable auto-merge.', + 'Once the PR is approved or has the `ready` label, the PR author can also use `/ci run` or `/ci retry`. New commits do not start CI automatically.', '', 'If you have any questions, please reach out to us on Slack at https://slack.vllm.ai.', '', diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 143fc427a49..aa1f437ab6a 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -41,7 +41,7 @@ jobs: if (hasReadyLabel || hasVerifiedLabel || mergedCount >= 4) { core.info(`Check passed: verified label=${hasVerifiedLabel}, ready label=${hasReadyLabel}, 4+ merged PRs=${mergedCount >= 4}`); } else { - core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label (the ready labels also trigger tests) or the author must have at least 4 merged PRs (found ${mergedCount}).`); + core.setFailed(`PR must have the 'verified', 'ready', or 'ready-run-all-tests' label to run pre-commit, or the author must have at least 4 merged PRs (found ${mergedCount}).`); } pre-commit: diff --git a/.github/workflows/run-ci-command.yml b/.github/workflows/run-ci-command.yml new file mode 100644 index 00000000000..ff55c65dd75 --- /dev/null +++ b/.github/workflows/run-ci-command.yml @@ -0,0 +1,40 @@ +name: Run CI from PR comment + +on: + issue_comment: + types: [created] + +concurrency: + group: run-ci-comment-${{ github.event.issue.number }} + cancel-in-progress: false + +permissions: + contents: read + issues: write + pull-requests: read + +jobs: + run-ci-command: + if: >- + github.event.issue.pull_request && + (github.event.comment.body == '/ci run' || + github.event.comment.body == '/ci retry') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.repository.default_branch }} + persist-credentials: false + - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + with: + python-version: "3.12" + - name: Authorize and run CI command + run: >- + uv run --no-project --python 3.12 + .github/workflows/scripts/run_ci_command.py + env: + BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }} + BUILDKITE_ORGANIZATION: vllm + BUILDKITE_PIPELINE: ci + CI_TRUSTED_USERS: ${{ vars.CI_TRUSTED_USERS }} + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/scripts/run_ci_command.py b/.github/workflows/scripts/run_ci_command.py new file mode 100644 index 00000000000..7880fc5568a --- /dev/null +++ b/.github/workflows/scripts/run_ci_command.py @@ -0,0 +1,580 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping, Sequence +from typing import Any + +COMMAND_RUN_CI = "/ci run" +COMMAND_RETRY_FAILED = "/ci retry" +READY_LABELS = {"ready", "ready-run-all-tests"} +TRUSTED_PERMISSIONS = {"admin", "maintain", "write"} +ACTIVE_BUILD_STATES = { + "blocked", + "creating", + "scheduled", + "running", + "failing", + "canceling", + "waiting", + "waiting_failed", +} +RETRY_STATES = "failed,timed_out,expired" + + +class ApiError(RuntimeError): + def __init__(self, status: int | None, message: str) -> None: + super().__init__(message) + self.status = status + + +class HttpTransport: + def request( + self, + url: str, + *, + body: Mapping[str, Any] | None = None, + headers: Mapping[str, str] | None = None, + method: str = "GET", + ) -> Any: + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request( + url, + data=data, + headers=dict(headers or {}), + method=method, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + response_body = response.read().decode() + except urllib.error.HTTPError as error: + response_body = error.read().decode() + message = self._error_message(response_body, error.reason) + raise ApiError( + error.code, + f"API returned {error.code}: {message}", + ) from error + except urllib.error.URLError as error: + raise ApiError(None, f"API request failed: {error.reason}") from error + + if not response_body: + return None + try: + return json.loads(response_body) + except json.JSONDecodeError as error: + raise ApiError(None, "API returned a non-JSON response.") from error + + @staticmethod + def _error_message(response_body: str, fallback: str) -> str: + try: + parsed = json.loads(response_body) + except json.JSONDecodeError: + return fallback + return str(parsed.get("message", fallback)) + + +class GitHubClient: + def __init__( + self, + token: str, + repository: str, + transport: HttpTransport | None = None, + ) -> None: + if not token: + raise RuntimeError("GH_TOKEN is not set.") + self.owner, self.repo = repository.split("/", maxsplit=1) + self.transport = transport or HttpTransport() + self.headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "User-Agent": "vllm-ci-command", + "X-GitHub-Api-Version": "2022-11-28", + } + + def _request( + self, + path: str, + *, + body: Mapping[str, Any] | None = None, + method: str = "GET", + ) -> Any: + return self.transport.request( + f"https://api.github.com{path}", + body=body, + headers=self.headers, + method=method, + ) + + def _repo_path(self, suffix: str) -> str: + owner = urllib.parse.quote(self.owner, safe="") + repo = urllib.parse.quote(self.repo, safe="") + return f"/repos/{owner}/{repo}{suffix}" + + def _paginate(self, path: str) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + separator = "&" if "?" in path else "?" + for page in range(1, 101): + response = self._request(f"{path}{separator}per_page=100&page={page}") + if not isinstance(response, list): + raise ApiError(None, "GitHub API returned an invalid list response.") + results.extend(response) + if len(response) < 100: + return results + raise ApiError(None, "GitHub API pagination exceeded 10,000 results.") + + def get_pr(self, number: int) -> dict[str, Any]: + return self._request(self._repo_path(f"/pulls/{number}")) + + def get_permission(self, actor: str) -> str: + username = urllib.parse.quote(actor, safe="") + try: + response = self._request( + self._repo_path(f"/collaborators/{username}/permission") + ) + except ApiError as error: + if error.status == 404: + return "none" + raise + return str(response["permission"]) + + def get_review_decision(self, number: int) -> str | None: + query = """ + query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + pullRequest(number: $number) { + reviewDecision + } + } + } + """ + response = self._request( + "/graphql", + body={ + "query": query, + "variables": { + "number": number, + "owner": self.owner, + "repo": self.repo, + }, + }, + method="POST", + ) + return response["data"]["repository"]["pullRequest"]["reviewDecision"] + + def list_reviews(self, number: int) -> list[dict[str, Any]]: + return self._paginate(self._repo_path(f"/pulls/{number}/reviews")) + + def list_reactions(self, comment_id: int) -> list[dict[str, Any]]: + return self._paginate( + self._repo_path(f"/issues/comments/{comment_id}/reactions") + ) + + def add_reaction(self, comment_id: int, content: str) -> None: + self._request( + self._repo_path(f"/issues/comments/{comment_id}/reactions"), + body={"content": content}, + method="POST", + ) + + def add_comment(self, issue_number: int, body: str) -> None: + self._request( + self._repo_path(f"/issues/{issue_number}/comments"), + body={"body": body}, + method="POST", + ) + + +class BuildkiteClient: + def __init__( + self, + token: str, + organization: str, + pipeline: str, + transport: HttpTransport | None = None, + ) -> None: + self.token = token + self.transport = transport or HttpTransport() + organization = urllib.parse.quote(organization, safe="") + pipeline = urllib.parse.quote(pipeline, safe="") + self.base_url = ( + "https://api.buildkite.com/v2/organizations/" + f"{organization}/pipelines/{pipeline}/builds" + ) + + def _request( + self, + *, + body: Mapping[str, Any] | None = None, + method: str = "GET", + path: str = "", + query: Sequence[tuple[str, str]] = (), + ) -> Any: + if not self.token: + raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.") + url = f"{self.base_url}{path}" + if query: + url = f"{url}?{urllib.parse.urlencode(query)}" + return self.transport.request( + url, + body=body, + headers={ + "Authorization": f"Bearer {self.token}", + "Content-Type": "application/json", + "User-Agent": "vllm-ci-command", + }, + method=method, + ) + + def list_builds( + self, + commit: str, + *, + metadata: tuple[str, str] | None = None, + ) -> list[dict[str, Any]]: + query = [ + ("commit", commit), + ("exclude_jobs", "true"), + ("exclude_pipeline", "true"), + ("per_page", "100"), + ] + if metadata: + key, value = metadata + query.append((f"meta_data[{key}]", value)) + response = self._request(query=query) + if not isinstance(response, list): + raise ApiError(None, "Buildkite API returned an invalid build list.") + return response + + def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]: + return self._request(body=body, method="POST") + + def retry_failed_jobs( + self, + build_number: int, + states: str, + ) -> dict[str, Any]: + number = urllib.parse.quote(str(build_number), safe="") + return self._request( + body={"states": states}, + method="PUT", + path=f"/{number}/retry_failed_jobs", + ) + + +def parse_command(body: str) -> str | None: + if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}: + return body + return None + + +def parse_trusted_users(value: str = "") -> set[str]: + return { + user.casefold() for item in value.split(",") for user in item.split() if user + } + + +def has_ready_label(pr: Mapping[str, Any]) -> bool: + return any(label["name"] in READY_LABELS for label in pr["labels"]) + + +def is_trusted_permission(permission: str) -> bool: + return permission in TRUSTED_PERMISSIONS + + +def authorize( + *, + actor: str, + permission: str, + pr: Mapping[str, Any], + trusted_approval: bool = False, + trusted_users: set[str] | None = None, +) -> tuple[bool, str]: + trusted_users = trusted_users or set() + if is_trusted_permission(permission): + return True, f"repository {permission} permission" + if actor.casefold() in trusted_users: + return True, "configured trusted contributor" + if actor.casefold() != pr["user"]["login"].casefold(): + return ( + False, + "Only reviewers with write access can run CI before it is " + "delegated to the PR author.", + ) + if pr["draft"]: + return False, "PR authors cannot run CI while the PR is a draft." + if has_ready_label(pr): + return True, "ready label" + if trusted_approval: + return True, "approval from a trusted reviewer" + return ( + False, + "A reviewer with write access must run `/ci run`, approve the PR, " + "or add the `ready` label first.", + ) + + +def has_trusted_approval( + github: GitHubClient, + number: int, + trusted_users: set[str], +) -> bool: + if github.get_review_decision(number) != "APPROVED": + return False + + latest_review_states: dict[str, tuple[str, str]] = {} + for review in github.list_reviews(number): + user = review.get("user") or {} + login = user.get("login") + state = review.get("state") + if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}: + latest_review_states[login.casefold()] = (login, state) + + for login, state in latest_review_states.values(): + if state != "APPROVED": + continue + if login.casefold() in trusted_users: + return True + if is_trusted_permission(github.get_permission(login)): + return True + return False + + +def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool: + pull_request = build.get("pull_request") + if isinstance(pull_request, Mapping): + build_pr_number = pull_request.get("id", pull_request.get("number")) + if build_pr_number is not None: + return str(build_pr_number) == str(pr_number) + metadata = build.get("meta_data") or {} + return str(metadata.get("github-pr-number")) == str(pr_number) + + +def is_active_build(build: Mapping[str, Any]) -> bool: + return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES + + +def select_latest_build( + builds: Sequence[dict[str, Any]], + pr_number: int, +) -> dict[str, Any] | None: + matching = [build for build in builds if is_build_for_pr(build, pr_number)] + return max(matching, key=lambda build: build.get("created_at", ""), default=None) + + +def create_build_payload( + *, + actor: str, + comment_id: int, + pr: Mapping[str, Any], +) -> dict[str, Any]: + return { + "commit": pr["head"]["sha"], + "branch": pr["head"]["ref"], + "message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}", + "pull_request_id": pr["number"], + "pull_request_base_branch": pr["base"]["ref"], + "pull_request_repository": pr["head"]["repo"]["clone_url"], + "pull_request_labels": [label["name"] for label in pr["labels"]], + "env": { + "VLLM_CI_GITHUB_COMMENT_ID": str(comment_id), + "VLLM_CI_TRIGGERED_BY": actor, + }, + "meta_data": { + "github-comment-id": str(comment_id), + "github-pr-number": str(pr["number"]), + "github-triggered-by": actor, + }, + } + + +def add_reaction_safely( + github: GitHubClient, + comment_id: int, + content: str, +) -> None: + try: + github.add_reaction(comment_id, content) + except Exception as error: + print(f"Could not add {content} reaction: {error}", file=sys.stderr) + + +def is_already_handled(github: GitHubClient, comment_id: int) -> bool: + return any( + reaction.get("content") in {"rocket", "-1"} + and (reaction.get("user") or {}).get("login") == "github-actions[bot]" + for reaction in github.list_reactions(comment_id) + ) + + +def handle_run_ci( + *, + actor: str, + buildkite: BuildkiteClient, + comment_id: int, + github: GitHubClient, + pr: Mapping[str, Any], +) -> str: + duplicate_builds = buildkite.list_builds( + pr["head"]["sha"], + metadata=("github-comment-id", str(comment_id)), + ) + duplicate = select_latest_build(duplicate_builds, pr["number"]) + if duplicate: + return f"CI was already requested by this comment: {duplicate['web_url']}" + + current_builds = buildkite.list_builds(pr["head"]["sha"]) + active_build = next( + ( + build + for build in current_builds + if is_build_for_pr(build, pr["number"]) and is_active_build(build) + ), + None, + ) + if active_build: + return f"CI is already running for this commit: {active_build['web_url']}" + + current_pr = github.get_pr(pr["number"]) + if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]: + return ( + "The PR head changed while processing the command. Comment `/ci run` again." + ) + + build = buildkite.create_build( + create_build_payload( + actor=actor, + comment_id=comment_id, + pr=current_pr, + ) + ) + return ( + f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) " + f"for commit `{current_pr['head']['sha'][:12]}`." + ) + + +def handle_retry_failed( + *, + buildkite: BuildkiteClient, + pr: Mapping[str, Any], +) -> str: + builds = buildkite.list_builds(pr["head"]["sha"]) + build = select_latest_build(builds, pr["number"]) + if not build: + return "No CI build exists for the current PR commit. Use `/ci run` first." + if not build.get("finished_at") or is_active_build(build): + return f"CI is still running for this commit: {build['web_url']}" + + retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES) + if retried["retried_jobs_count"] == 0: + return ( + f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}" + ) + return ( + f"Queued {retried['retried_jobs_count']} failed job(s) for retry in " + f"[Buildkite CI #{build['number']}]({build['web_url']})." + ) + + +def run( + event: Mapping[str, Any], + github: GitHubClient, + buildkite: BuildkiteClient, + trusted_users_value: str = "", +) -> None: + command = parse_command(event["comment"]["body"]) + if not command or "pull_request" not in event["issue"]: + return + + issue_number = event["issue"]["number"] + comment_id = event["comment"]["id"] + actor = event["comment"]["user"]["login"] + + if is_already_handled(github, comment_id): + print(f"Comment {comment_id} was already handled.") + return + add_reaction_safely(github, comment_id, "eyes") + + try: + pr = github.get_pr(issue_number) + permission = github.get_permission(actor) + if pr["state"] != "open": + github.add_comment(issue_number, "CI commands require an open PR.") + return + + trusted_users = parse_trusted_users(trusted_users_value) + should_check_approval = ( + not is_trusted_permission(permission) + and actor.casefold() not in trusted_users + and actor.casefold() == pr["user"]["login"].casefold() + and not pr["draft"] + and not has_ready_label(pr) + ) + trusted_approval = should_check_approval and has_trusted_approval( + github, + issue_number, + trusted_users, + ) + allowed, reason = authorize( + actor=actor, + permission=permission, + pr=pr, + trusted_approval=trusted_approval, + trusted_users=trusted_users, + ) + if not allowed: + add_reaction_safely(github, comment_id, "-1") + github.add_comment(issue_number, f"@{actor}, {reason}") + return + + print(f"Authorized @{actor}: {reason}") + if command == COMMAND_RUN_CI: + message = handle_run_ci( + actor=actor, + buildkite=buildkite, + comment_id=comment_id, + github=github, + pr=pr, + ) + else: + message = handle_retry_failed(buildkite=buildkite, pr=pr) + add_reaction_safely(github, comment_id, "rocket") + github.add_comment(issue_number, message) + except Exception: + add_reaction_safely(github, comment_id, "confused") + raise + + +def main() -> None: + event_path = os.environ["GITHUB_EVENT_PATH"] + with open(event_path, encoding="utf-8") as event_file: + event = json.load(event_file) + + if not parse_command(event["comment"]["body"]): + return + + github = GitHubClient( + os.environ.get("GH_TOKEN", ""), + os.environ["GITHUB_REPOSITORY"], + ) + buildkite = BuildkiteClient( + os.environ.get("BUILDKITE_API_TOKEN", ""), + os.environ.get("BUILDKITE_ORGANIZATION", "vllm"), + os.environ.get("BUILDKITE_PIPELINE", "ci"), + ) + run( + event, + github, + buildkite, + os.environ.get("CI_TRUSTED_USERS", ""), + ) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/test_run_ci_command.py b/.github/workflows/scripts/test_run_ci_command.py new file mode 100644 index 00000000000..e691438d6af --- /dev/null +++ b/.github/workflows/scripts/test_run_ci_command.py @@ -0,0 +1,362 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import unittest +from typing import Any + +from run_ci_command import ( + COMMAND_RETRY_FAILED, + COMMAND_RUN_CI, + RETRY_STATES, + BuildkiteClient, + authorize, + create_build_payload, + has_trusted_approval, + is_active_build, + is_build_for_pr, + parse_command, + parse_trusted_users, + run, + select_latest_build, +) + + +def make_pr(**overrides: Any) -> dict[str, Any]: + pr = { + "base": {"ref": "main"}, + "draft": False, + "head": { + "ref": "feature", + "repo": {"clone_url": "https://github.com/contributor/vllm.git"}, + "sha": "0123456789abcdef", + }, + "labels": [], + "number": 42, + "state": "open", + "user": {"login": "author"}, + } + pr.update(overrides) + return pr + + +def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]: + return { + "comment": { + "body": command, + "id": 99, + "user": {"login": actor}, + }, + "issue": { + "number": 42, + "pull_request": {}, + }, + } + + +class FakeGitHub: + def __init__( + self, + *, + permission: str = "write", + permissions: dict[str, str] | None = None, + pr: dict[str, Any] | None = None, + review_decision: str = "REVIEW_REQUIRED", + reviews: list[dict[str, Any]] | None = None, + ) -> None: + self.comments: list[str] = [] + self.permission = permission + self.permissions = permissions or {} + self.pr = pr or make_pr() + self.reactions: list[str] = [] + self.review_decision = review_decision + self.reviews = reviews or [] + + def get_pr(self, number: int) -> dict[str, Any]: + return self.pr + + def get_permission(self, actor: str) -> str: + return self.permissions.get(actor, self.permission) + + def get_review_decision(self, number: int) -> str: + return self.review_decision + + def list_reviews(self, number: int) -> list[dict[str, Any]]: + return self.reviews + + def list_reactions(self, comment_id: int) -> list[dict[str, Any]]: + return [] + + def add_reaction(self, comment_id: int, content: str) -> None: + self.reactions.append(content) + + def add_comment(self, issue_number: int, body: str) -> None: + self.comments.append(body) + + +class FakeBuildkite: + def __init__( + self, + build_lists: list[list[dict[str, Any]]] | None = None, + ) -> None: + self.build_lists = build_lists or [] + self.created_builds: list[dict[str, Any]] = [] + self.list_calls: list[tuple[str, tuple[str, str] | None]] = [] + self.retry_calls: list[tuple[int, str]] = [] + + def list_builds( + self, + commit: str, + *, + metadata: tuple[str, str] | None = None, + ) -> list[dict[str, Any]]: + self.list_calls.append((commit, metadata)) + return self.build_lists.pop(0) + + def create_build(self, body: dict[str, Any]) -> dict[str, Any]: + self.created_builds.append(body) + return { + "number": 123, + "web_url": "https://buildkite.example/builds/123", + } + + def retry_failed_jobs( + self, + build_number: int, + states: str, + ) -> dict[str, Any]: + self.retry_calls.append((build_number, states)) + return {"retried_jobs_count": 3} + + +class FakeTransport: + def __init__(self, response: Any) -> None: + self.calls: list[dict[str, Any]] = [] + self.response = response + + def request(self, url: str, **kwargs: Any) -> Any: + self.calls.append({"url": url, **kwargs}) + return self.response + + +class RunCiCommandTest(unittest.TestCase): + def test_only_exact_ci_commands_are_accepted(self) -> None: + self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI) + self.assertEqual( + parse_command(COMMAND_RETRY_FAILED), + COMMAND_RETRY_FAILED, + ) + self.assertIsNone(parse_command("/ci run please")) + self.assertIsNone(parse_command(" /ci run")) + + def test_write_access_authorizes_reviewers_and_authors(self) -> None: + allowed, _ = authorize( + actor="reviewer", + permission="write", + pr=make_pr(), + ) + self.assertTrue(allowed) + + def test_configured_trusted_contributors_can_run_ci(self) -> None: + trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO") + allowed, _ = authorize( + actor="trusted-two", + permission="read", + pr=make_pr(), + trusted_users=trusted_users, + ) + self.assertTrue(allowed) + + def test_authors_need_an_approval_or_ready_label(self) -> None: + pending, _ = authorize( + actor="author", + permission="read", + pr=make_pr(), + ) + approved, _ = authorize( + actor="author", + permission="read", + pr=make_pr(), + trusted_approval=True, + ) + ready, _ = authorize( + actor="author", + permission="read", + pr=make_pr(labels=[{"name": "ready"}]), + ) + self.assertFalse(pending) + self.assertTrue(approved) + self.assertTrue(ready) + + def test_non_author_contributors_without_write_are_denied(self) -> None: + allowed, _ = authorize( + actor="contributor", + permission="read", + pr=make_pr(), + trusted_approval=True, + ) + self.assertFalse(allowed) + + def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None: + allowed, _ = authorize( + actor="author", + permission="read", + pr=make_pr(draft=True, labels=[{"name": "ready"}]), + trusted_approval=True, + ) + self.assertFalse(allowed) + + def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None: + approved_review = { + "state": "APPROVED", + "user": {"login": "reviewer"}, + } + trusted = FakeGitHub( + permission="read", + permissions={"reviewer": "write"}, + review_decision="APPROVED", + reviews=[approved_review], + ) + untrusted = FakeGitHub( + permission="read", + review_decision="APPROVED", + reviews=[approved_review], + ) + self.assertTrue(has_trusted_approval(trusted, 42, set())) + self.assertFalse(has_trusted_approval(untrusted, 42, set())) + + def test_build_matching_is_scoped_to_the_pr(self) -> None: + self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42)) + self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42)) + self.assertTrue( + is_build_for_pr( + {"meta_data": {"github-pr-number": "42"}}, + 42, + ) + ) + + def test_latest_build_selection_ignores_other_prs(self) -> None: + latest = select_latest_build( + [ + { + "created_at": "2026-07-28T02:00:00Z", + "number": 3, + "pull_request": {"id": 43}, + }, + { + "created_at": "2026-07-28T01:00:00Z", + "number": 2, + "pull_request": {"id": 42}, + }, + { + "created_at": "2026-07-28T00:00:00Z", + "number": 1, + "pull_request": {"id": 42}, + }, + ], + 42, + ) + self.assertEqual(latest["number"], 2) + + def test_active_build_states_prevent_duplicate_runs(self) -> None: + self.assertTrue(is_active_build({"state": "scheduled"})) + self.assertTrue(is_active_build({"state": "running"})) + self.assertTrue(is_active_build({"state": "waiting"})) + self.assertTrue(is_active_build({"blocked": True, "state": "passed"})) + self.assertFalse(is_active_build({"state": "failed"})) + + def test_build_payload_preserves_pr_context(self) -> None: + payload = create_build_payload( + actor="reviewer", + comment_id=99, + pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]), + ) + self.assertEqual( + payload, + { + "commit": "0123456789abcdef", + "branch": "feature", + "message": "PR #42 /ci run by @reviewer", + "pull_request_id": 42, + "pull_request_base_branch": "main", + "pull_request_repository": ("https://github.com/contributor/vllm.git"), + "pull_request_labels": ["ready", "v1"], + "env": { + "VLLM_CI_GITHUB_COMMENT_ID": "99", + "VLLM_CI_TRIGGERED_BY": "reviewer", + }, + "meta_data": { + "github-comment-id": "99", + "github-pr-number": "42", + "github-triggered-by": "reviewer", + }, + }, + ) + + def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None: + github = FakeGitHub() + buildkite = FakeBuildkite([[], []]) + run(make_event(COMMAND_RUN_CI), github, buildkite) + + self.assertEqual(len(buildkite.created_builds), 1) + self.assertEqual( + buildkite.created_builds[0]["message"], + "PR #42 /ci run by @reviewer", + ) + self.assertEqual(github.reactions, ["eyes", "rocket"]) + self.assertIn("Buildkite CI #123", github.comments[0]) + + def test_unapproved_authors_are_denied_without_buildkite(self) -> None: + github = FakeGitHub( + permission="read", + pr=make_pr(), + review_decision="REVIEW_REQUIRED", + ) + buildkite = FakeBuildkite() + run(make_event(COMMAND_RUN_CI, "author"), github, buildkite) + + self.assertEqual(buildkite.list_calls, []) + self.assertEqual(github.reactions, ["eyes", "-1"]) + self.assertIn("approve the PR", github.comments[0]) + + def test_ci_retry_uses_latest_current_sha_build(self) -> None: + github = FakeGitHub( + permission="read", + pr=make_pr(labels=[{"name": "ready"}]), + ) + buildkite = FakeBuildkite( + [ + [ + { + "created_at": "2026-07-28T01:00:00Z", + "finished_at": "2026-07-28T02:00:00Z", + "number": 123, + "pull_request": {"id": 42}, + "state": "failed", + "web_url": "https://buildkite.example/builds/123", + } + ] + ] + ) + run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite) + + self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)]) + self.assertIn("Queued 3 failed job", github.comments[0]) + + def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None: + transport = FakeTransport({"retried_jobs_count": 2}) + client = BuildkiteClient( + "secret", + "vllm", + "ci", + transport=transport, + ) + client.retry_failed_jobs(123, RETRY_STATES) + + call = transport.calls[0] + self.assertEqual(call["method"], "PUT") + self.assertTrue(call["url"].endswith("/123/retry_failed_jobs")) + self.assertEqual(call["body"], {"states": RETRY_STATES}) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/contributing/README.md b/docs/contributing/README.md index 34dc385db78..89acc6b7f5a 100644 --- a/docs/contributing/README.md +++ b/docs/contributing/README.md @@ -301,8 +301,10 @@ review process: isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion. - Note that not all CI checks will be executed due to limited computational - resources. The reviewer will add `ready` label to the PR when the PR is - ready to merge or a full CI run is needed. + resources. Reviewers with write access and configured trusted contributors + can comment `/ci run` when CI signals are needed before a PR is ready. After + the PR is approved or has the `ready` label, the PR author can use `/ci run` + or `/ci retry`. New commits do not start CI automatically. ### Pull Request Limits and Escalation