forked from Karylab-cklius/vllm
[CI] Add comment-based Buildkite triggers (#50132)
Signed-off-by: khluu <khluu000@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Kevin H. Luu
co-authored by
OpenAI Codex
OpenAI Codex
parent
6fbbcf2151
commit
7f4c52f2ba
@@ -0,0 +1,580 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
COMMAND_RUN_CI = "/ci run"
|
||||
COMMAND_RETRY_FAILED = "/ci retry"
|
||||
READY_LABELS = {"ready", "ready-run-all-tests"}
|
||||
TRUSTED_PERMISSIONS = {"admin", "maintain", "write"}
|
||||
ACTIVE_BUILD_STATES = {
|
||||
"blocked",
|
||||
"creating",
|
||||
"scheduled",
|
||||
"running",
|
||||
"failing",
|
||||
"canceling",
|
||||
"waiting",
|
||||
"waiting_failed",
|
||||
}
|
||||
RETRY_STATES = "failed,timed_out,expired"
|
||||
|
||||
|
||||
class ApiError(RuntimeError):
|
||||
def __init__(self, status: int | None, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
|
||||
|
||||
class HttpTransport:
|
||||
def request(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
body: Mapping[str, Any] | None = None,
|
||||
headers: Mapping[str, str] | None = None,
|
||||
method: str = "GET",
|
||||
) -> Any:
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
data=data,
|
||||
headers=dict(headers or {}),
|
||||
method=method,
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
response_body = response.read().decode()
|
||||
except urllib.error.HTTPError as error:
|
||||
response_body = error.read().decode()
|
||||
message = self._error_message(response_body, error.reason)
|
||||
raise ApiError(
|
||||
error.code,
|
||||
f"API returned {error.code}: {message}",
|
||||
) from error
|
||||
except urllib.error.URLError as error:
|
||||
raise ApiError(None, f"API request failed: {error.reason}") from error
|
||||
|
||||
if not response_body:
|
||||
return None
|
||||
try:
|
||||
return json.loads(response_body)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ApiError(None, "API returned a non-JSON response.") from error
|
||||
|
||||
@staticmethod
|
||||
def _error_message(response_body: str, fallback: str) -> str:
|
||||
try:
|
||||
parsed = json.loads(response_body)
|
||||
except json.JSONDecodeError:
|
||||
return fallback
|
||||
return str(parsed.get("message", fallback))
|
||||
|
||||
|
||||
class GitHubClient:
|
||||
def __init__(
|
||||
self,
|
||||
token: str,
|
||||
repository: str,
|
||||
transport: HttpTransport | None = None,
|
||||
) -> None:
|
||||
if not token:
|
||||
raise RuntimeError("GH_TOKEN is not set.")
|
||||
self.owner, self.repo = repository.split("/", maxsplit=1)
|
||||
self.transport = transport or HttpTransport()
|
||||
self.headers = {
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "vllm-ci-command",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
|
||||
def _request(
|
||||
self,
|
||||
path: str,
|
||||
*,
|
||||
body: Mapping[str, Any] | None = None,
|
||||
method: str = "GET",
|
||||
) -> Any:
|
||||
return self.transport.request(
|
||||
f"https://api.github.com{path}",
|
||||
body=body,
|
||||
headers=self.headers,
|
||||
method=method,
|
||||
)
|
||||
|
||||
def _repo_path(self, suffix: str) -> str:
|
||||
owner = urllib.parse.quote(self.owner, safe="")
|
||||
repo = urllib.parse.quote(self.repo, safe="")
|
||||
return f"/repos/{owner}/{repo}{suffix}"
|
||||
|
||||
def _paginate(self, path: str) -> list[dict[str, Any]]:
|
||||
results: list[dict[str, Any]] = []
|
||||
separator = "&" if "?" in path else "?"
|
||||
for page in range(1, 101):
|
||||
response = self._request(f"{path}{separator}per_page=100&page={page}")
|
||||
if not isinstance(response, list):
|
||||
raise ApiError(None, "GitHub API returned an invalid list response.")
|
||||
results.extend(response)
|
||||
if len(response) < 100:
|
||||
return results
|
||||
raise ApiError(None, "GitHub API pagination exceeded 10,000 results.")
|
||||
|
||||
def get_pr(self, number: int) -> dict[str, Any]:
|
||||
return self._request(self._repo_path(f"/pulls/{number}"))
|
||||
|
||||
def get_permission(self, actor: str) -> str:
|
||||
username = urllib.parse.quote(actor, safe="")
|
||||
try:
|
||||
response = self._request(
|
||||
self._repo_path(f"/collaborators/{username}/permission")
|
||||
)
|
||||
except ApiError as error:
|
||||
if error.status == 404:
|
||||
return "none"
|
||||
raise
|
||||
return str(response["permission"])
|
||||
|
||||
def get_review_decision(self, number: int) -> str | None:
|
||||
query = """
|
||||
query($owner: String!, $repo: String!, $number: Int!) {
|
||||
repository(owner: $owner, name: $repo) {
|
||||
pullRequest(number: $number) {
|
||||
reviewDecision
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
response = self._request(
|
||||
"/graphql",
|
||||
body={
|
||||
"query": query,
|
||||
"variables": {
|
||||
"number": number,
|
||||
"owner": self.owner,
|
||||
"repo": self.repo,
|
||||
},
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
return response["data"]["repository"]["pullRequest"]["reviewDecision"]
|
||||
|
||||
def list_reviews(self, number: int) -> list[dict[str, Any]]:
|
||||
return self._paginate(self._repo_path(f"/pulls/{number}/reviews"))
|
||||
|
||||
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
|
||||
return self._paginate(
|
||||
self._repo_path(f"/issues/comments/{comment_id}/reactions")
|
||||
)
|
||||
|
||||
def add_reaction(self, comment_id: int, content: str) -> None:
|
||||
self._request(
|
||||
self._repo_path(f"/issues/comments/{comment_id}/reactions"),
|
||||
body={"content": content},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
def add_comment(self, issue_number: int, body: str) -> None:
|
||||
self._request(
|
||||
self._repo_path(f"/issues/{issue_number}/comments"),
|
||||
body={"body": body},
|
||||
method="POST",
|
||||
)
|
||||
|
||||
|
||||
class BuildkiteClient:
|
||||
def __init__(
|
||||
self,
|
||||
token: str,
|
||||
organization: str,
|
||||
pipeline: str,
|
||||
transport: HttpTransport | None = None,
|
||||
) -> None:
|
||||
self.token = token
|
||||
self.transport = transport or HttpTransport()
|
||||
organization = urllib.parse.quote(organization, safe="")
|
||||
pipeline = urllib.parse.quote(pipeline, safe="")
|
||||
self.base_url = (
|
||||
"https://api.buildkite.com/v2/organizations/"
|
||||
f"{organization}/pipelines/{pipeline}/builds"
|
||||
)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
*,
|
||||
body: Mapping[str, Any] | None = None,
|
||||
method: str = "GET",
|
||||
path: str = "",
|
||||
query: Sequence[tuple[str, str]] = (),
|
||||
) -> Any:
|
||||
if not self.token:
|
||||
raise RuntimeError("The BUILDKITE_API_TOKEN repository secret is not set.")
|
||||
url = f"{self.base_url}{path}"
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
return self.transport.request(
|
||||
url,
|
||||
body=body,
|
||||
headers={
|
||||
"Authorization": f"Bearer {self.token}",
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "vllm-ci-command",
|
||||
},
|
||||
method=method,
|
||||
)
|
||||
|
||||
def list_builds(
|
||||
self,
|
||||
commit: str,
|
||||
*,
|
||||
metadata: tuple[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = [
|
||||
("commit", commit),
|
||||
("exclude_jobs", "true"),
|
||||
("exclude_pipeline", "true"),
|
||||
("per_page", "100"),
|
||||
]
|
||||
if metadata:
|
||||
key, value = metadata
|
||||
query.append((f"meta_data[{key}]", value))
|
||||
response = self._request(query=query)
|
||||
if not isinstance(response, list):
|
||||
raise ApiError(None, "Buildkite API returned an invalid build list.")
|
||||
return response
|
||||
|
||||
def create_build(self, body: Mapping[str, Any]) -> dict[str, Any]:
|
||||
return self._request(body=body, method="POST")
|
||||
|
||||
def retry_failed_jobs(
|
||||
self,
|
||||
build_number: int,
|
||||
states: str,
|
||||
) -> dict[str, Any]:
|
||||
number = urllib.parse.quote(str(build_number), safe="")
|
||||
return self._request(
|
||||
body={"states": states},
|
||||
method="PUT",
|
||||
path=f"/{number}/retry_failed_jobs",
|
||||
)
|
||||
|
||||
|
||||
def parse_command(body: str) -> str | None:
|
||||
if body in {COMMAND_RUN_CI, COMMAND_RETRY_FAILED}:
|
||||
return body
|
||||
return None
|
||||
|
||||
|
||||
def parse_trusted_users(value: str = "") -> set[str]:
|
||||
return {
|
||||
user.casefold() for item in value.split(",") for user in item.split() if user
|
||||
}
|
||||
|
||||
|
||||
def has_ready_label(pr: Mapping[str, Any]) -> bool:
|
||||
return any(label["name"] in READY_LABELS for label in pr["labels"])
|
||||
|
||||
|
||||
def is_trusted_permission(permission: str) -> bool:
|
||||
return permission in TRUSTED_PERMISSIONS
|
||||
|
||||
|
||||
def authorize(
|
||||
*,
|
||||
actor: str,
|
||||
permission: str,
|
||||
pr: Mapping[str, Any],
|
||||
trusted_approval: bool = False,
|
||||
trusted_users: set[str] | None = None,
|
||||
) -> tuple[bool, str]:
|
||||
trusted_users = trusted_users or set()
|
||||
if is_trusted_permission(permission):
|
||||
return True, f"repository {permission} permission"
|
||||
if actor.casefold() in trusted_users:
|
||||
return True, "configured trusted contributor"
|
||||
if actor.casefold() != pr["user"]["login"].casefold():
|
||||
return (
|
||||
False,
|
||||
"Only reviewers with write access can run CI before it is "
|
||||
"delegated to the PR author.",
|
||||
)
|
||||
if pr["draft"]:
|
||||
return False, "PR authors cannot run CI while the PR is a draft."
|
||||
if has_ready_label(pr):
|
||||
return True, "ready label"
|
||||
if trusted_approval:
|
||||
return True, "approval from a trusted reviewer"
|
||||
return (
|
||||
False,
|
||||
"A reviewer with write access must run `/ci run`, approve the PR, "
|
||||
"or add the `ready` label first.",
|
||||
)
|
||||
|
||||
|
||||
def has_trusted_approval(
|
||||
github: GitHubClient,
|
||||
number: int,
|
||||
trusted_users: set[str],
|
||||
) -> bool:
|
||||
if github.get_review_decision(number) != "APPROVED":
|
||||
return False
|
||||
|
||||
latest_review_states: dict[str, tuple[str, str]] = {}
|
||||
for review in github.list_reviews(number):
|
||||
user = review.get("user") or {}
|
||||
login = user.get("login")
|
||||
state = review.get("state")
|
||||
if login and state in {"APPROVED", "CHANGES_REQUESTED", "DISMISSED"}:
|
||||
latest_review_states[login.casefold()] = (login, state)
|
||||
|
||||
for login, state in latest_review_states.values():
|
||||
if state != "APPROVED":
|
||||
continue
|
||||
if login.casefold() in trusted_users:
|
||||
return True
|
||||
if is_trusted_permission(github.get_permission(login)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_build_for_pr(build: Mapping[str, Any], pr_number: int) -> bool:
|
||||
pull_request = build.get("pull_request")
|
||||
if isinstance(pull_request, Mapping):
|
||||
build_pr_number = pull_request.get("id", pull_request.get("number"))
|
||||
if build_pr_number is not None:
|
||||
return str(build_pr_number) == str(pr_number)
|
||||
metadata = build.get("meta_data") or {}
|
||||
return str(metadata.get("github-pr-number")) == str(pr_number)
|
||||
|
||||
|
||||
def is_active_build(build: Mapping[str, Any]) -> bool:
|
||||
return bool(build.get("blocked")) or build.get("state") in ACTIVE_BUILD_STATES
|
||||
|
||||
|
||||
def select_latest_build(
|
||||
builds: Sequence[dict[str, Any]],
|
||||
pr_number: int,
|
||||
) -> dict[str, Any] | None:
|
||||
matching = [build for build in builds if is_build_for_pr(build, pr_number)]
|
||||
return max(matching, key=lambda build: build.get("created_at", ""), default=None)
|
||||
|
||||
|
||||
def create_build_payload(
|
||||
*,
|
||||
actor: str,
|
||||
comment_id: int,
|
||||
pr: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"commit": pr["head"]["sha"],
|
||||
"branch": pr["head"]["ref"],
|
||||
"message": f"PR #{pr['number']} {COMMAND_RUN_CI} by @{actor}",
|
||||
"pull_request_id": pr["number"],
|
||||
"pull_request_base_branch": pr["base"]["ref"],
|
||||
"pull_request_repository": pr["head"]["repo"]["clone_url"],
|
||||
"pull_request_labels": [label["name"] for label in pr["labels"]],
|
||||
"env": {
|
||||
"VLLM_CI_GITHUB_COMMENT_ID": str(comment_id),
|
||||
"VLLM_CI_TRIGGERED_BY": actor,
|
||||
},
|
||||
"meta_data": {
|
||||
"github-comment-id": str(comment_id),
|
||||
"github-pr-number": str(pr["number"]),
|
||||
"github-triggered-by": actor,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def add_reaction_safely(
|
||||
github: GitHubClient,
|
||||
comment_id: int,
|
||||
content: str,
|
||||
) -> None:
|
||||
try:
|
||||
github.add_reaction(comment_id, content)
|
||||
except Exception as error:
|
||||
print(f"Could not add {content} reaction: {error}", file=sys.stderr)
|
||||
|
||||
|
||||
def is_already_handled(github: GitHubClient, comment_id: int) -> bool:
|
||||
return any(
|
||||
reaction.get("content") in {"rocket", "-1"}
|
||||
and (reaction.get("user") or {}).get("login") == "github-actions[bot]"
|
||||
for reaction in github.list_reactions(comment_id)
|
||||
)
|
||||
|
||||
|
||||
def handle_run_ci(
|
||||
*,
|
||||
actor: str,
|
||||
buildkite: BuildkiteClient,
|
||||
comment_id: int,
|
||||
github: GitHubClient,
|
||||
pr: Mapping[str, Any],
|
||||
) -> str:
|
||||
duplicate_builds = buildkite.list_builds(
|
||||
pr["head"]["sha"],
|
||||
metadata=("github-comment-id", str(comment_id)),
|
||||
)
|
||||
duplicate = select_latest_build(duplicate_builds, pr["number"])
|
||||
if duplicate:
|
||||
return f"CI was already requested by this comment: {duplicate['web_url']}"
|
||||
|
||||
current_builds = buildkite.list_builds(pr["head"]["sha"])
|
||||
active_build = next(
|
||||
(
|
||||
build
|
||||
for build in current_builds
|
||||
if is_build_for_pr(build, pr["number"]) and is_active_build(build)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if active_build:
|
||||
return f"CI is already running for this commit: {active_build['web_url']}"
|
||||
|
||||
current_pr = github.get_pr(pr["number"])
|
||||
if current_pr["state"] != "open" or current_pr["head"]["sha"] != pr["head"]["sha"]:
|
||||
return (
|
||||
"The PR head changed while processing the command. Comment `/ci run` again."
|
||||
)
|
||||
|
||||
build = buildkite.create_build(
|
||||
create_build_payload(
|
||||
actor=actor,
|
||||
comment_id=comment_id,
|
||||
pr=current_pr,
|
||||
)
|
||||
)
|
||||
return (
|
||||
f"Triggered [Buildkite CI #{build['number']}]({build['web_url']}) "
|
||||
f"for commit `{current_pr['head']['sha'][:12]}`."
|
||||
)
|
||||
|
||||
|
||||
def handle_retry_failed(
|
||||
*,
|
||||
buildkite: BuildkiteClient,
|
||||
pr: Mapping[str, Any],
|
||||
) -> str:
|
||||
builds = buildkite.list_builds(pr["head"]["sha"])
|
||||
build = select_latest_build(builds, pr["number"])
|
||||
if not build:
|
||||
return "No CI build exists for the current PR commit. Use `/ci run` first."
|
||||
if not build.get("finished_at") or is_active_build(build):
|
||||
return f"CI is still running for this commit: {build['web_url']}"
|
||||
|
||||
retried = buildkite.retry_failed_jobs(build["number"], RETRY_STATES)
|
||||
if retried["retried_jobs_count"] == 0:
|
||||
return (
|
||||
f"No failed, timed-out, or expired jobs need retrying: {build['web_url']}"
|
||||
)
|
||||
return (
|
||||
f"Queued {retried['retried_jobs_count']} failed job(s) for retry in "
|
||||
f"[Buildkite CI #{build['number']}]({build['web_url']})."
|
||||
)
|
||||
|
||||
|
||||
def run(
|
||||
event: Mapping[str, Any],
|
||||
github: GitHubClient,
|
||||
buildkite: BuildkiteClient,
|
||||
trusted_users_value: str = "",
|
||||
) -> None:
|
||||
command = parse_command(event["comment"]["body"])
|
||||
if not command or "pull_request" not in event["issue"]:
|
||||
return
|
||||
|
||||
issue_number = event["issue"]["number"]
|
||||
comment_id = event["comment"]["id"]
|
||||
actor = event["comment"]["user"]["login"]
|
||||
|
||||
if is_already_handled(github, comment_id):
|
||||
print(f"Comment {comment_id} was already handled.")
|
||||
return
|
||||
add_reaction_safely(github, comment_id, "eyes")
|
||||
|
||||
try:
|
||||
pr = github.get_pr(issue_number)
|
||||
permission = github.get_permission(actor)
|
||||
if pr["state"] != "open":
|
||||
github.add_comment(issue_number, "CI commands require an open PR.")
|
||||
return
|
||||
|
||||
trusted_users = parse_trusted_users(trusted_users_value)
|
||||
should_check_approval = (
|
||||
not is_trusted_permission(permission)
|
||||
and actor.casefold() not in trusted_users
|
||||
and actor.casefold() == pr["user"]["login"].casefold()
|
||||
and not pr["draft"]
|
||||
and not has_ready_label(pr)
|
||||
)
|
||||
trusted_approval = should_check_approval and has_trusted_approval(
|
||||
github,
|
||||
issue_number,
|
||||
trusted_users,
|
||||
)
|
||||
allowed, reason = authorize(
|
||||
actor=actor,
|
||||
permission=permission,
|
||||
pr=pr,
|
||||
trusted_approval=trusted_approval,
|
||||
trusted_users=trusted_users,
|
||||
)
|
||||
if not allowed:
|
||||
add_reaction_safely(github, comment_id, "-1")
|
||||
github.add_comment(issue_number, f"@{actor}, {reason}")
|
||||
return
|
||||
|
||||
print(f"Authorized @{actor}: {reason}")
|
||||
if command == COMMAND_RUN_CI:
|
||||
message = handle_run_ci(
|
||||
actor=actor,
|
||||
buildkite=buildkite,
|
||||
comment_id=comment_id,
|
||||
github=github,
|
||||
pr=pr,
|
||||
)
|
||||
else:
|
||||
message = handle_retry_failed(buildkite=buildkite, pr=pr)
|
||||
add_reaction_safely(github, comment_id, "rocket")
|
||||
github.add_comment(issue_number, message)
|
||||
except Exception:
|
||||
add_reaction_safely(github, comment_id, "confused")
|
||||
raise
|
||||
|
||||
|
||||
def main() -> None:
|
||||
event_path = os.environ["GITHUB_EVENT_PATH"]
|
||||
with open(event_path, encoding="utf-8") as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
if not parse_command(event["comment"]["body"]):
|
||||
return
|
||||
|
||||
github = GitHubClient(
|
||||
os.environ.get("GH_TOKEN", ""),
|
||||
os.environ["GITHUB_REPOSITORY"],
|
||||
)
|
||||
buildkite = BuildkiteClient(
|
||||
os.environ.get("BUILDKITE_API_TOKEN", ""),
|
||||
os.environ.get("BUILDKITE_ORGANIZATION", "vllm"),
|
||||
os.environ.get("BUILDKITE_PIPELINE", "ci"),
|
||||
)
|
||||
run(
|
||||
event,
|
||||
github,
|
||||
buildkite,
|
||||
os.environ.get("CI_TRUSTED_USERS", ""),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,362 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import unittest
|
||||
from typing import Any
|
||||
|
||||
from run_ci_command import (
|
||||
COMMAND_RETRY_FAILED,
|
||||
COMMAND_RUN_CI,
|
||||
RETRY_STATES,
|
||||
BuildkiteClient,
|
||||
authorize,
|
||||
create_build_payload,
|
||||
has_trusted_approval,
|
||||
is_active_build,
|
||||
is_build_for_pr,
|
||||
parse_command,
|
||||
parse_trusted_users,
|
||||
run,
|
||||
select_latest_build,
|
||||
)
|
||||
|
||||
|
||||
def make_pr(**overrides: Any) -> dict[str, Any]:
|
||||
pr = {
|
||||
"base": {"ref": "main"},
|
||||
"draft": False,
|
||||
"head": {
|
||||
"ref": "feature",
|
||||
"repo": {"clone_url": "https://github.com/contributor/vllm.git"},
|
||||
"sha": "0123456789abcdef",
|
||||
},
|
||||
"labels": [],
|
||||
"number": 42,
|
||||
"state": "open",
|
||||
"user": {"login": "author"},
|
||||
}
|
||||
pr.update(overrides)
|
||||
return pr
|
||||
|
||||
|
||||
def make_event(command: str, actor: str = "reviewer") -> dict[str, Any]:
|
||||
return {
|
||||
"comment": {
|
||||
"body": command,
|
||||
"id": 99,
|
||||
"user": {"login": actor},
|
||||
},
|
||||
"issue": {
|
||||
"number": 42,
|
||||
"pull_request": {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class FakeGitHub:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
permission: str = "write",
|
||||
permissions: dict[str, str] | None = None,
|
||||
pr: dict[str, Any] | None = None,
|
||||
review_decision: str = "REVIEW_REQUIRED",
|
||||
reviews: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
self.comments: list[str] = []
|
||||
self.permission = permission
|
||||
self.permissions = permissions or {}
|
||||
self.pr = pr or make_pr()
|
||||
self.reactions: list[str] = []
|
||||
self.review_decision = review_decision
|
||||
self.reviews = reviews or []
|
||||
|
||||
def get_pr(self, number: int) -> dict[str, Any]:
|
||||
return self.pr
|
||||
|
||||
def get_permission(self, actor: str) -> str:
|
||||
return self.permissions.get(actor, self.permission)
|
||||
|
||||
def get_review_decision(self, number: int) -> str:
|
||||
return self.review_decision
|
||||
|
||||
def list_reviews(self, number: int) -> list[dict[str, Any]]:
|
||||
return self.reviews
|
||||
|
||||
def list_reactions(self, comment_id: int) -> list[dict[str, Any]]:
|
||||
return []
|
||||
|
||||
def add_reaction(self, comment_id: int, content: str) -> None:
|
||||
self.reactions.append(content)
|
||||
|
||||
def add_comment(self, issue_number: int, body: str) -> None:
|
||||
self.comments.append(body)
|
||||
|
||||
|
||||
class FakeBuildkite:
|
||||
def __init__(
|
||||
self,
|
||||
build_lists: list[list[dict[str, Any]]] | None = None,
|
||||
) -> None:
|
||||
self.build_lists = build_lists or []
|
||||
self.created_builds: list[dict[str, Any]] = []
|
||||
self.list_calls: list[tuple[str, tuple[str, str] | None]] = []
|
||||
self.retry_calls: list[tuple[int, str]] = []
|
||||
|
||||
def list_builds(
|
||||
self,
|
||||
commit: str,
|
||||
*,
|
||||
metadata: tuple[str, str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
self.list_calls.append((commit, metadata))
|
||||
return self.build_lists.pop(0)
|
||||
|
||||
def create_build(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
self.created_builds.append(body)
|
||||
return {
|
||||
"number": 123,
|
||||
"web_url": "https://buildkite.example/builds/123",
|
||||
}
|
||||
|
||||
def retry_failed_jobs(
|
||||
self,
|
||||
build_number: int,
|
||||
states: str,
|
||||
) -> dict[str, Any]:
|
||||
self.retry_calls.append((build_number, states))
|
||||
return {"retried_jobs_count": 3}
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self, response: Any) -> None:
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
self.response = response
|
||||
|
||||
def request(self, url: str, **kwargs: Any) -> Any:
|
||||
self.calls.append({"url": url, **kwargs})
|
||||
return self.response
|
||||
|
||||
|
||||
class RunCiCommandTest(unittest.TestCase):
|
||||
def test_only_exact_ci_commands_are_accepted(self) -> None:
|
||||
self.assertEqual(parse_command(COMMAND_RUN_CI), COMMAND_RUN_CI)
|
||||
self.assertEqual(
|
||||
parse_command(COMMAND_RETRY_FAILED),
|
||||
COMMAND_RETRY_FAILED,
|
||||
)
|
||||
self.assertIsNone(parse_command("/ci run please"))
|
||||
self.assertIsNone(parse_command(" /ci run"))
|
||||
|
||||
def test_write_access_authorizes_reviewers_and_authors(self) -> None:
|
||||
allowed, _ = authorize(
|
||||
actor="reviewer",
|
||||
permission="write",
|
||||
pr=make_pr(),
|
||||
)
|
||||
self.assertTrue(allowed)
|
||||
|
||||
def test_configured_trusted_contributors_can_run_ci(self) -> None:
|
||||
trusted_users = parse_trusted_users("trusted-one, TRUSTED-TWO")
|
||||
allowed, _ = authorize(
|
||||
actor="trusted-two",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
trusted_users=trusted_users,
|
||||
)
|
||||
self.assertTrue(allowed)
|
||||
|
||||
def test_authors_need_an_approval_or_ready_label(self) -> None:
|
||||
pending, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
)
|
||||
approved, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
trusted_approval=True,
|
||||
)
|
||||
ready, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(labels=[{"name": "ready"}]),
|
||||
)
|
||||
self.assertFalse(pending)
|
||||
self.assertTrue(approved)
|
||||
self.assertTrue(ready)
|
||||
|
||||
def test_non_author_contributors_without_write_are_denied(self) -> None:
|
||||
allowed, _ = authorize(
|
||||
actor="contributor",
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
trusted_approval=True,
|
||||
)
|
||||
self.assertFalse(allowed)
|
||||
|
||||
def test_authors_cannot_use_ready_state_on_draft_prs(self) -> None:
|
||||
allowed, _ = authorize(
|
||||
actor="author",
|
||||
permission="read",
|
||||
pr=make_pr(draft=True, labels=[{"name": "ready"}]),
|
||||
trusted_approval=True,
|
||||
)
|
||||
self.assertFalse(allowed)
|
||||
|
||||
def test_only_trusted_reviewers_can_delegate_through_approval(self) -> None:
|
||||
approved_review = {
|
||||
"state": "APPROVED",
|
||||
"user": {"login": "reviewer"},
|
||||
}
|
||||
trusted = FakeGitHub(
|
||||
permission="read",
|
||||
permissions={"reviewer": "write"},
|
||||
review_decision="APPROVED",
|
||||
reviews=[approved_review],
|
||||
)
|
||||
untrusted = FakeGitHub(
|
||||
permission="read",
|
||||
review_decision="APPROVED",
|
||||
reviews=[approved_review],
|
||||
)
|
||||
self.assertTrue(has_trusted_approval(trusted, 42, set()))
|
||||
self.assertFalse(has_trusted_approval(untrusted, 42, set()))
|
||||
|
||||
def test_build_matching_is_scoped_to_the_pr(self) -> None:
|
||||
self.assertTrue(is_build_for_pr({"pull_request": {"id": 42}}, 42))
|
||||
self.assertFalse(is_build_for_pr({"pull_request": {"id": 43}}, 42))
|
||||
self.assertTrue(
|
||||
is_build_for_pr(
|
||||
{"meta_data": {"github-pr-number": "42"}},
|
||||
42,
|
||||
)
|
||||
)
|
||||
|
||||
def test_latest_build_selection_ignores_other_prs(self) -> None:
|
||||
latest = select_latest_build(
|
||||
[
|
||||
{
|
||||
"created_at": "2026-07-28T02:00:00Z",
|
||||
"number": 3,
|
||||
"pull_request": {"id": 43},
|
||||
},
|
||||
{
|
||||
"created_at": "2026-07-28T01:00:00Z",
|
||||
"number": 2,
|
||||
"pull_request": {"id": 42},
|
||||
},
|
||||
{
|
||||
"created_at": "2026-07-28T00:00:00Z",
|
||||
"number": 1,
|
||||
"pull_request": {"id": 42},
|
||||
},
|
||||
],
|
||||
42,
|
||||
)
|
||||
self.assertEqual(latest["number"], 2)
|
||||
|
||||
def test_active_build_states_prevent_duplicate_runs(self) -> None:
|
||||
self.assertTrue(is_active_build({"state": "scheduled"}))
|
||||
self.assertTrue(is_active_build({"state": "running"}))
|
||||
self.assertTrue(is_active_build({"state": "waiting"}))
|
||||
self.assertTrue(is_active_build({"blocked": True, "state": "passed"}))
|
||||
self.assertFalse(is_active_build({"state": "failed"}))
|
||||
|
||||
def test_build_payload_preserves_pr_context(self) -> None:
|
||||
payload = create_build_payload(
|
||||
actor="reviewer",
|
||||
comment_id=99,
|
||||
pr=make_pr(labels=[{"name": "ready"}, {"name": "v1"}]),
|
||||
)
|
||||
self.assertEqual(
|
||||
payload,
|
||||
{
|
||||
"commit": "0123456789abcdef",
|
||||
"branch": "feature",
|
||||
"message": "PR #42 /ci run by @reviewer",
|
||||
"pull_request_id": 42,
|
||||
"pull_request_base_branch": "main",
|
||||
"pull_request_repository": ("https://github.com/contributor/vllm.git"),
|
||||
"pull_request_labels": ["ready", "v1"],
|
||||
"env": {
|
||||
"VLLM_CI_GITHUB_COMMENT_ID": "99",
|
||||
"VLLM_CI_TRIGGERED_BY": "reviewer",
|
||||
},
|
||||
"meta_data": {
|
||||
"github-comment-id": "99",
|
||||
"github-pr-number": "42",
|
||||
"github-triggered-by": "reviewer",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def test_ci_run_dispatches_build_with_current_pr_metadata(self) -> None:
|
||||
github = FakeGitHub()
|
||||
buildkite = FakeBuildkite([[], []])
|
||||
run(make_event(COMMAND_RUN_CI), github, buildkite)
|
||||
|
||||
self.assertEqual(len(buildkite.created_builds), 1)
|
||||
self.assertEqual(
|
||||
buildkite.created_builds[0]["message"],
|
||||
"PR #42 /ci run by @reviewer",
|
||||
)
|
||||
self.assertEqual(github.reactions, ["eyes", "rocket"])
|
||||
self.assertIn("Buildkite CI #123", github.comments[0])
|
||||
|
||||
def test_unapproved_authors_are_denied_without_buildkite(self) -> None:
|
||||
github = FakeGitHub(
|
||||
permission="read",
|
||||
pr=make_pr(),
|
||||
review_decision="REVIEW_REQUIRED",
|
||||
)
|
||||
buildkite = FakeBuildkite()
|
||||
run(make_event(COMMAND_RUN_CI, "author"), github, buildkite)
|
||||
|
||||
self.assertEqual(buildkite.list_calls, [])
|
||||
self.assertEqual(github.reactions, ["eyes", "-1"])
|
||||
self.assertIn("approve the PR", github.comments[0])
|
||||
|
||||
def test_ci_retry_uses_latest_current_sha_build(self) -> None:
|
||||
github = FakeGitHub(
|
||||
permission="read",
|
||||
pr=make_pr(labels=[{"name": "ready"}]),
|
||||
)
|
||||
buildkite = FakeBuildkite(
|
||||
[
|
||||
[
|
||||
{
|
||||
"created_at": "2026-07-28T01:00:00Z",
|
||||
"finished_at": "2026-07-28T02:00:00Z",
|
||||
"number": 123,
|
||||
"pull_request": {"id": 42},
|
||||
"state": "failed",
|
||||
"web_url": "https://buildkite.example/builds/123",
|
||||
}
|
||||
]
|
||||
]
|
||||
)
|
||||
run(make_event(COMMAND_RETRY_FAILED, "author"), github, buildkite)
|
||||
|
||||
self.assertEqual(buildkite.retry_calls, [(123, RETRY_STATES)])
|
||||
self.assertIn("Queued 3 failed job", github.comments[0])
|
||||
|
||||
def test_buildkite_retry_uses_retry_failed_jobs_endpoint(self) -> None:
|
||||
transport = FakeTransport({"retried_jobs_count": 2})
|
||||
client = BuildkiteClient(
|
||||
"secret",
|
||||
"vllm",
|
||||
"ci",
|
||||
transport=transport,
|
||||
)
|
||||
client.retry_failed_jobs(123, RETRY_STATES)
|
||||
|
||||
call = transport.calls[0]
|
||||
self.assertEqual(call["method"], "PUT")
|
||||
self.assertTrue(call["url"].endswith("/123/retry_failed_jobs"))
|
||||
self.assertEqual(call["body"], {"states": RETRY_STATES})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user