2323API_ROOT = "https://api.github.com"
2424API_VERSION = "2022-11-28"
2525PAGE_SIZE = 100
26+ JOBS_PAGE_SIZE = 50
2627PENDING_RETRY_BATCH_SIZE = 25
28+ PENDING_FAILURE_RETRY_BATCH_SIZE = 5
2729STATE_VERSION = 1
2830USER_AGENT = "open-telemetry-github-actions-queue-collector"
2931COLLECTION_ID_PATTERN = re .compile (r"^[A-Za-z0-9._-]+$" )
32+ TERMINAL_JOB_CONCLUSIONS = {
33+ "success" ,
34+ "failure" ,
35+ "neutral" ,
36+ "cancelled" ,
37+ "skipped" ,
38+ "timed_out" ,
39+ "action_required" ,
40+ }
3041
3142
3243class ApiError (RuntimeError ):
@@ -461,7 +472,7 @@ def list_jobs(
461472 if not isinstance (total , int ) or not isinstance (jobs , list ):
462473 raise ApiError ("workflow jobs response has an invalid shape" )
463474
464- pages = math .ceil (total / PAGE_SIZE )
475+ pages = math .ceil (total / JOBS_PAGE_SIZE )
465476 for page in range (2 , pages + 1 ):
466477 response = self ._jobs_page (org , repository , run_id , page )
467478 page_jobs = response .get ("jobs" )
@@ -488,7 +499,7 @@ def _jobs_page(
488499 f"/repos/{ quote (org )} /{ quote (repository )} /actions/runs/{ run_id } /jobs" ,
489500 {
490501 "filter" : "all" ,
491- "per_page" : str (PAGE_SIZE ),
502+ "per_page" : str (JOBS_PAGE_SIZE ),
492503 "page" : str (page ),
493504 },
494505 )
@@ -644,14 +655,7 @@ def _collect(
644655 for item in state .pending_runs
645656 if item ["repository" ] in current_repositories
646657 ]
647- pending_to_retry = sorted (
648- state .pending_runs ,
649- key = lambda item : (
650- item .get ("last_attempt_at" ) or item .get ("created_at" ) or "" ,
651- item ["repository" ],
652- item ["run_id" ],
653- ),
654- )[:PENDING_RETRY_BATCH_SIZE ]
658+ pending_to_retry = _pending_runs_to_retry (state .pending_runs )
655659 self ._collect_pending_runs (state , pending_to_retry , records )
656660
657661 while (
@@ -789,7 +793,9 @@ def _collect_pending_runs(
789793 else :
790794 _merge_pending_runs (
791795 state ,
792- [_pending_run_attempt (item , self ._now ())],
796+ [
797+ _pending_run_attempt (item , self ._now (), error_resolved = True )
798+ ],
793799 )
794800 except RateLimitExhausted :
795801 raise
@@ -835,7 +841,7 @@ def _collect_completed_run(
835841 run : dict [str , Any ],
836842 ) -> tuple [list [dict [str , Any ]], bool ]:
837843 jobs = self ._client .list_jobs (self ._org , repository , run ["id" ])
838- if any (job . get ( "status" ) != "completed" for job in jobs ):
844+ if any (not _job_is_terminal ( job ) for job in jobs ):
839845 return [], False
840846 jobs = [job for job in jobs if job .get ("runner_name" )]
841847 collected_at = _format_instant (self ._now ())
@@ -892,6 +898,41 @@ def collect_live_and_backfill(
892898 )
893899
894900
901+ def _job_is_terminal (job : dict [str , Any ]) -> bool :
902+ return job .get ("status" ) == "completed" or (
903+ job .get ("status" ) == "in_progress"
904+ and bool (job .get ("completed_at" ))
905+ and job .get ("conclusion" ) in TERMINAL_JOB_CONCLUSIONS
906+ )
907+
908+
909+ def _pending_runs_to_retry (
910+ pending : list [dict [str , Any ]],
911+ ) -> list [dict [str , Any ]]:
912+ def priority (item : dict [str , Any ]) -> tuple [str , str , int ]:
913+ return (
914+ item .get ("last_attempt_at" ) or item .get ("created_at" ) or "" ,
915+ item ["repository" ],
916+ item ["run_id" ],
917+ )
918+
919+ ordered = sorted (pending , key = priority )
920+ failed = [item for item in ordered if item .get ("last_error" )]
921+ ordinary = [item for item in ordered if not item .get ("last_error" )]
922+ failed_slots = max (
923+ PENDING_FAILURE_RETRY_BATCH_SIZE ,
924+ PENDING_RETRY_BATCH_SIZE - len (ordinary ),
925+ )
926+ selected_failures = failed [:failed_slots ]
927+ return sorted (
928+ [
929+ * selected_failures ,
930+ * ordinary [: PENDING_RETRY_BATCH_SIZE - len (selected_failures )],
931+ ],
932+ key = priority ,
933+ )
934+
935+
895936def _pending_run (
896937 repository : str ,
897938 run : dict [str , Any ],
@@ -919,11 +960,16 @@ def _pending_run_failure(
919960def _pending_run_attempt (
920961 item : dict [str , Any ],
921962 attempted_at : datetime ,
963+ * ,
964+ error_resolved : bool = False ,
922965) -> dict [str , Any ]:
923- return {
966+ attempted = {
924967 ** item ,
925968 "last_attempt_at" : _format_instant (attempted_at ),
926969 }
970+ if error_resolved :
971+ attempted .pop ("last_error" , None )
972+ return attempted
927973
928974
929975def _merge_pending_runs (
0 commit comments