From 9b33a39338c3ea002e3b5798d076703d728abe18 Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Sat, 1 Aug 2026 12:30:43 +0530 Subject: [PATCH 1/2] ci(playwright): raise the chromium shard budget to 21 minutes The chromium lane outgrew a 19-minute shard. At the COMMON_MAX_SHARDS ceiling of 24 the heaviest shard is predicted at 19.2m, so assign_lane_within_budget() raises SystemExit and full-mode planning aborts before a single test runs. Every merge_group run today failed this way (PRs #30705, #30768, #30458, #30725, #30754), while pull_request_target runs pass because targeted selection is far smaller. Raise COMMON_SHARD_BUDGET_MS from 19m to 21m. At 24 shards the heaviest is 19.2m, so the loop is guaranteed to converge at or before the ceiling. 21m stays inside the 25m `timeout` wrapper around `npx playwright test` and the 35m playwright-ci-postgresql job clock, leaving ~4m of headroom. Note the common lane now sits 1m above the dedicated lanes rather than 1m below. The strict 20-minute TARGET_MS ceiling is unaffected: it bounds a single atomic unit, not a shard, so a 21m shard built from units each under 20m does not trip it. Co-Authored-By: Claude Opus 5 --- .github/playwright/README.md | 4 ++-- .github/scripts/build_playwright_shards.py | 7 ++++++- .../scripts/tests/test_playwright_ci_planning.py | 15 +++++++++------ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/playwright/README.md b/.github/playwright/README.md index 4733af42594e..1d89306ab8b3 100644 --- a/.github/playwright/README.md +++ b/.github/playwright/README.md @@ -14,10 +14,10 @@ SSO stays in its dedicated workflow, while knowledge graph and ontology share on `build_playwright_shards.py` discovers stable Playwright test IDs and assigns hook-inclusive p75 duration from the latest three successful full runs. It uses longest-processing-time-first balancing and computes the common shard count as: ```text -ceil(total weighted worker time / (3 workers * 19 minutes * 0.85)) +ceil(total weighted worker time / (3 workers * 21 minutes * 0.85)) ``` -The common matrix is bounded to 5–24 runners and uses a 19-minute allocation budget, targeting one minute of reserve before the planner's strict 20-minute validation ceiling. Dedicated lanes use a 20-minute allocation budget. Planner weights use the hook-inclusive observed duration, including retries. Only an exact stable test ID explicitly reported as skipped may retain a zero weight; every other zero-duration observation and every unseen test uses the conservative fallback, and zero weights never transfer through the file/title identity fallback. The versioned bootstrap baseline uses stable expected and skipped observations from the coverage-complete but failed full run `29984209316`, while unexpected and flaky tests retain their prior duration weights from run `29980474263`. It is bootstrap data, not a fabricated successful history; normal planning still uses p75 from the latest three successful full runs when those artifacts exist. This keeps expensive internally parallel suites together instead of multiplying their shared setup across runners. Serial/global behavior stays in one-worker lanes. Large suites listed in `AUDITED_PARALLEL_SUITES` are split at test granularity only after confirming that they are not serial and do not depend on earlier tests. The planner fails when any remaining atomic unit or bounded lane exceeds the 20-minute ceiling. +The common matrix is bounded to 5–24 runners and uses a 21-minute allocation budget. Dedicated lanes use a 20-minute allocation budget. The common lane previously sat a minute below the dedicated lanes, but the chromium suite outgrew what 24 runners could hold at 19 minutes, so full-mode planning aborted outright; 21 minutes restores headroom while staying inside the 25-minute `timeout` wrapper around `npx playwright test` and the 35-minute `playwright-ci-postgresql` job clock. Note that the allocation budget bounds a whole *shard*, whereas the strict 20-minute ceiling below bounds a single *atomic unit*, so the two are independent. Planner weights use the hook-inclusive observed duration, including retries. Only an exact stable test ID explicitly reported as skipped may retain a zero weight; every other zero-duration observation and every unseen test uses the conservative fallback, and zero weights never transfer through the file/title identity fallback. The versioned bootstrap baseline uses stable expected and skipped observations from the coverage-complete but failed full run `29984209316`, while unexpected and flaky tests retain their prior duration weights from run `29980474263`. It is bootstrap data, not a fabricated successful history; normal planning still uses p75 from the latest three successful full runs when those artifacts exist. This keeps expensive internally parallel suites together instead of multiplying their shared setup across runners. Serial/global behavior stays in one-worker lanes. Large suites listed in `AUDITED_PARALLEL_SUITES` are split at test granularity only after confirming that they are not serial and do not depend on earlier tests. The planner fails when any remaining atomic unit or bounded lane exceeds the 20-minute ceiling. The `Basic` and `chromium` projects share that common 24-runner cap and are balanced together; they are not separate pools of standard hosted runners. Isolated ingestion, search, reindex, permission, and global-state lanes are additional because they cannot safely share mutable server state with the common matrix. diff --git a/.github/scripts/build_playwright_shards.py b/.github/scripts/build_playwright_shards.py index a186ee494eac..8f7297d7f320 100755 --- a/.github/scripts/build_playwright_shards.py +++ b/.github/scripts/build_playwright_shards.py @@ -57,7 +57,12 @@ "search-rbac": 1, } TARGET_MS = 20 * 60 * 1000 -COMMON_SHARD_BUDGET_MS = 19 * 60 * 1000 +# The chromium lane has outgrown a 19-minute shard: at the COMMON_MAX_SHARDS +# ceiling the heaviest shard is predicted at 19.2m, so full-mode planning aborts +# and every merge-queue run fails before a single test runs. 21m keeps the plan +# inside the 25m `timeout` wrapper around `npx playwright test` and the 35m +# playwright-ci-postgresql job clock, which together leave ~4m of headroom. +COMMON_SHARD_BUDGET_MS = 21 * 60 * 1000 EFFICIENCY = 0.85 COMMON_MAX_SHARDS = 24 FALLBACK_TEST_MS = 20_000 diff --git a/.github/scripts/tests/test_playwright_ci_planning.py b/.github/scripts/tests/test_playwright_ci_planning.py index 71d3edc9c1ca..54e5ea605b74 100644 --- a/.github/scripts/tests/test_playwright_ci_planning.py +++ b/.github/scripts/tests/test_playwright_ci_planning.py @@ -61,10 +61,13 @@ def test_full_common_shard_count_is_capped_at_24(): assert planner.shard_count(units, "chromium", "full") == 24 -def test_common_lane_keeps_one_minute_of_allocation_reserve(): +def test_common_lane_carries_its_own_shard_budget(): + # The chromium lane no longer sits a minute under TARGET_MS: the suite grew + # past what COMMON_MAX_SHARDS could hold at 19m, so it now runs a minute + # above the other lanes. Both still fit the 25m playwright timeout wrapper. planner = load_script("build_playwright_shards") - assert planner.shard_budget_ms_for_lane("chromium") == 19 * 60 * 1000 + assert planner.shard_budget_ms_for_lane("chromium") == 21 * 60 * 1000 assert planner.shard_budget_ms_for_lane("search") == 20 * 60 * 1000 @@ -449,7 +452,7 @@ def test_hook_heavy_subsuites_in_audited_suite_stay_atomic(): ] -def test_common_shards_enforce_the_nineteen_minute_budget(tmp_path): +def test_common_shards_enforce_the_twenty_one_minute_budget(tmp_path): planner = load_script("build_playwright_shards") within_budget = planner.Unit( "chromium", @@ -457,7 +460,7 @@ def test_common_shards_enforce_the_nineteen_minute_budget(tmp_path): "within", grep_titles={("chromium", "within.spec.ts", "within")}, test_ids={"within"}, - weight_ms=19 * 60 * 1000, + weight_ms=21 * 60 * 1000, ) above_budget = planner.Unit( "chromium", @@ -465,11 +468,11 @@ def test_common_shards_enforce_the_nineteen_minute_budget(tmp_path): "above", grep_titles={("chromium", "above.spec.ts", "above")}, test_ids={"above"}, - weight_ms=19 * 60 * 1000 + 1, + weight_ms=21 * 60 * 1000 + 1, ) planner.write_plan(tmp_path, "chromium", 0, [within_budget]) - with pytest.raises(SystemExit, match="above the 19-minute plan budget"): + with pytest.raises(SystemExit, match="above the 21-minute plan budget"): planner.write_plan(tmp_path, "chromium", 1, [above_budget]) From 96fe55ac4c17fda3676f5d200e7495c995d1ce9a Mon Sep 17 00:00:00 2001 From: Shailesh Parmar Date: Sat, 1 Aug 2026 12:37:00 +0530 Subject: [PATCH 2/2] test(playwright): cover full-mode allocator convergence at the shard ceiling Addresses review on #30784. assign_lane_within_budget() was only exercised in "targeted" mode, so neither the full-mode convergence path nor the SystemExit at COMMON_MAX_SHARDS had coverage -- the exact code path that took the merge queue down. Add both: - test_full_mode_chromium_converges_at_the_shard_ceiling builds a lane that needs the window above 19m and asserts the allocator converges at or before the ceiling. Verified as a real guard: with the budget reverted to 19m it fails with "needs more than 24 shards ... heaviest shard is predicted at 20.4m". - test_full_mode_chromium_reports_a_lane_the_ceiling_cannot_hold pins the SystemExit path, which had no coverage at all. Also reword the budget comment: ~4m of headroom is relative to the 25m playwright timeout wrapper specifically, not to the 35m job clock, which is looser and additionally absorbs setup/teardown. Co-Authored-By: Claude Opus 5 --- .github/scripts/build_playwright_shards.py | 7 ++-- .../tests/test_playwright_ci_planning.py | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/scripts/build_playwright_shards.py b/.github/scripts/build_playwright_shards.py index 8f7297d7f320..400c2b520fa3 100755 --- a/.github/scripts/build_playwright_shards.py +++ b/.github/scripts/build_playwright_shards.py @@ -59,9 +59,10 @@ TARGET_MS = 20 * 60 * 1000 # The chromium lane has outgrown a 19-minute shard: at the COMMON_MAX_SHARDS # ceiling the heaviest shard is predicted at 19.2m, so full-mode planning aborts -# and every merge-queue run fails before a single test runs. 21m keeps the plan -# inside the 25m `timeout` wrapper around `npx playwright test` and the 35m -# playwright-ci-postgresql job clock, which together leave ~4m of headroom. +# and every merge-queue run fails before a single test runs. The binding limit +# is the 25m `timeout` wrapper around `npx playwright test`, against which 21m +# leaves ~4m; the 35m playwright-ci-postgresql job clock is looser still, since +# it also has to cover ~5-8m of setup and teardown around that wrapper. COMMON_SHARD_BUDGET_MS = 21 * 60 * 1000 EFFICIENCY = 0.85 COMMON_MAX_SHARDS = 24 diff --git a/.github/scripts/tests/test_playwright_ci_planning.py b/.github/scripts/tests/test_playwright_ci_planning.py index 54e5ea605b74..53a14dd811c6 100644 --- a/.github/scripts/tests/test_playwright_ci_planning.py +++ b/.github/scripts/tests/test_playwright_ci_planning.py @@ -98,6 +98,44 @@ def test_common_assignment_stays_within_the_execution_ceiling(): ) +def test_full_mode_chromium_converges_at_the_shard_ceiling(): + # Regression guard for the merge-queue outage: a lane of this shape exhausted + # COMMON_MAX_SHARDS under the old 19m budget and aborted planning outright. + # The allocator must converge at or before the ceiling, and the resulting + # plan genuinely needs the window above 19m -- so quietly reverting the + # budget to 19m fails on the final assertion rather than only in CI. + planner = load_script("build_playwright_shards") + units = [ + planner.Unit( + "chromium", f"heavy-{index}.spec.ts", str(index), weight_ms=13 * 60 * 1000 + ) + for index in range(96) + ] + + shards = planner.assign_lane_within_budget(units, "chromium", "full") + + workers = planner.LANE_WORKERS.get("chromium", 3) + heaviest_ms = max( + planner.predicted_execution_ms(shard, workers) for shard in shards + ) + assert len(shards) <= planner.COMMON_MAX_SHARDS + assert heaviest_ms <= planner.COMMON_SHARD_BUDGET_MS + assert heaviest_ms > 19 * 60 * 1000 + + +def test_full_mode_chromium_reports_a_lane_the_ceiling_cannot_hold(): + planner = load_script("build_playwright_shards") + units = [ + planner.Unit( + "chromium", f"huge-{index}.spec.ts", str(index), weight_ms=19 * 60 * 1000 + ) + for index in range(120) + ] + + with pytest.raises(SystemExit, match=r"needs more than 24 shards"): + planner.assign_lane_within_budget(units, "chromium", "full") + + def test_shard_pattern_includes_project_and_file(): planner = load_script("build_playwright_shards") unit = planner.Unit(