Skip to content

Commit 1a1502e

Browse files
traskCopilot
andauthored
Unblock GitHub Actions queue retries (#464)
Use smaller jobs pages, share bounded pending retries between failed and unfinished runs, and recognize terminal job evidence when GitHub retains a stale status. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: dce070cb-ab76-4725-80eb-0db99ed545b4
1 parent 104194c commit 1a1502e

3 files changed

Lines changed: 504 additions & 22 deletions

File tree

‎.github/scripts/github-actions-queue/collect.py‎

Lines changed: 59 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,21 @@
2323
API_ROOT = "https://api.github.com"
2424
API_VERSION = "2022-11-28"
2525
PAGE_SIZE = 100
26+
JOBS_PAGE_SIZE = 50
2627
PENDING_RETRY_BATCH_SIZE = 25
28+
PENDING_FAILURE_RETRY_BATCH_SIZE = 5
2729
STATE_VERSION = 1
2830
USER_AGENT = "open-telemetry-github-actions-queue-collector"
2931
COLLECTION_ID_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$")
32+
TERMINAL_JOB_CONCLUSIONS = {
33+
"success",
34+
"failure",
35+
"neutral",
36+
"cancelled",
37+
"skipped",
38+
"timed_out",
39+
"action_required",
40+
}
3041

3142

3243
class ApiError(RuntimeError):
@@ -461,7 +472,7 @@ def list_jobs(
461472
if not isinstance(total, int) or not isinstance(jobs, list):
462473
raise ApiError("workflow jobs response has an invalid shape")
463474

464-
pages = math.ceil(total / PAGE_SIZE)
475+
pages = math.ceil(total / JOBS_PAGE_SIZE)
465476
for page in range(2, pages + 1):
466477
response = self._jobs_page(org, repository, run_id, page)
467478
page_jobs = response.get("jobs")
@@ -488,7 +499,7 @@ def _jobs_page(
488499
f"/repos/{quote(org)}/{quote(repository)}/actions/runs/{run_id}/jobs",
489500
{
490501
"filter": "all",
491-
"per_page": str(PAGE_SIZE),
502+
"per_page": str(JOBS_PAGE_SIZE),
492503
"page": str(page),
493504
},
494505
)
@@ -644,14 +655,7 @@ def _collect(
644655
for item in state.pending_runs
645656
if item["repository"] in current_repositories
646657
]
647-
pending_to_retry = sorted(
648-
state.pending_runs,
649-
key=lambda item: (
650-
item.get("last_attempt_at") or item.get("created_at") or "",
651-
item["repository"],
652-
item["run_id"],
653-
),
654-
)[:PENDING_RETRY_BATCH_SIZE]
658+
pending_to_retry = _pending_runs_to_retry(state.pending_runs)
655659
self._collect_pending_runs(state, pending_to_retry, records)
656660

657661
while (
@@ -789,7 +793,9 @@ def _collect_pending_runs(
789793
else:
790794
_merge_pending_runs(
791795
state,
792-
[_pending_run_attempt(item, self._now())],
796+
[
797+
_pending_run_attempt(item, self._now(), error_resolved=True)
798+
],
793799
)
794800
except RateLimitExhausted:
795801
raise
@@ -835,7 +841,7 @@ def _collect_completed_run(
835841
run: dict[str, Any],
836842
) -> tuple[list[dict[str, Any]], bool]:
837843
jobs = self._client.list_jobs(self._org, repository, run["id"])
838-
if any(job.get("status") != "completed" for job in jobs):
844+
if any(not _job_is_terminal(job) for job in jobs):
839845
return [], False
840846
jobs = [job for job in jobs if job.get("runner_name")]
841847
collected_at = _format_instant(self._now())
@@ -892,6 +898,41 @@ def collect_live_and_backfill(
892898
)
893899

894900

901+
def _job_is_terminal(job: dict[str, Any]) -> bool:
902+
return job.get("status") == "completed" or (
903+
job.get("status") == "in_progress"
904+
and bool(job.get("completed_at"))
905+
and job.get("conclusion") in TERMINAL_JOB_CONCLUSIONS
906+
)
907+
908+
909+
def _pending_runs_to_retry(
910+
pending: list[dict[str, Any]],
911+
) -> list[dict[str, Any]]:
912+
def priority(item: dict[str, Any]) -> tuple[str, str, int]:
913+
return (
914+
item.get("last_attempt_at") or item.get("created_at") or "",
915+
item["repository"],
916+
item["run_id"],
917+
)
918+
919+
ordered = sorted(pending, key=priority)
920+
failed = [item for item in ordered if item.get("last_error")]
921+
ordinary = [item for item in ordered if not item.get("last_error")]
922+
failed_slots = max(
923+
PENDING_FAILURE_RETRY_BATCH_SIZE,
924+
PENDING_RETRY_BATCH_SIZE - len(ordinary),
925+
)
926+
selected_failures = failed[:failed_slots]
927+
return sorted(
928+
[
929+
*selected_failures,
930+
*ordinary[: PENDING_RETRY_BATCH_SIZE - len(selected_failures)],
931+
],
932+
key=priority,
933+
)
934+
935+
895936
def _pending_run(
896937
repository: str,
897938
run: dict[str, Any],
@@ -919,11 +960,16 @@ def _pending_run_failure(
919960
def _pending_run_attempt(
920961
item: dict[str, Any],
921962
attempted_at: datetime,
963+
*,
964+
error_resolved: bool = False,
922965
) -> dict[str, Any]:
923-
return {
966+
attempted = {
924967
**item,
925968
"last_attempt_at": _format_instant(attempted_at),
926969
}
970+
if error_resolved:
971+
attempted.pop("last_error", None)
972+
return attempted
927973

928974

929975
def _merge_pending_runs(

0 commit comments

Comments
 (0)