Merge remote-tracking branch 'origin/main' into fix/playwright-guard-api-responses

Author: ShaileshParmar11

.github/scripts/build_playwright_shards.py

@@ -60,15 +60,29 @@
     "search-rbac": 1,
 }
 TARGET_MS = 20 * 60 * 1000
-# The chromium lane has outgrown a 19-minute shard: at the COMMON_MAX_SHARDS
-# ceiling the heaviest shard is predicted at 19.2m, so full-mode planning aborts
-# and every merge-queue run fails before a single test runs. The binding limit
-# is the 25m `timeout` wrapper around `npx playwright test`, against which 21m
-# leaves ~4m; the 35m playwright-ci job clock is looser still, since
-# it also has to cover ~5-8m of setup and teardown around that wrapper.
-COMMON_SHARD_BUDGET_MS = 21 * 60 * 1000
+# Chromium shard budget, derived from the predicted→actual execution tail
+# rather than from the average. Measured on merge_group run 32219260486
+# (23 chromium shards, fresh auto-refreshed baseline): actual/predicted
+# execution ratio has median 0.96 — predictions are well calibrated — but
+# the tail reaches 1.08 on a healthy run and 1.23 on a noisy one
+# (chromium-01 in run 32209040146 was SIGTERM'd by the 25-minute wrapper
+# with 162/164 tests already passed). The binding constraint is that
+# budget × worst-tail-ratio must stay under the 1500-second wrapper:
+#   21 min × 1.23 = 1550 s  → dead shard, all tests green (the outage)
+#   19 min × 1.23 = 1402 s  → 98 s of margin
+#   19 min × 1.32 = 1500 s  → break-even; tolerated tail is 1.32
+# 21 min was a stop-gap (#30784) to fit the lane under the old 24-shard
+# ceiling; the ceiling is now 28 (below), which is what actually makes a
+# 19-minute budget feasible again.
+COMMON_SHARD_BUDGET_MS = 19 * 60 * 1000
 EFFICIENCY = 0.85
-COMMON_MAX_SHARDS = 24
+# Raised 24 → 28 together with the budget revert above. Current chromium
+# content (~71,700 predicted worker-seconds) needs 25 shards at a
+# 19-minute budget — over the old cap, which is exactly why #30784 had to
+# raise the budget instead. 28 leaves ~12% content-growth headroom before
+# planning aborts; if the lane grows past that, split heavy suites (see
+# AUDITED_PARALLEL_SUITES) before considering another cap raise.
+COMMON_MAX_SHARDS = 28
 # Weight assigned to a test that has no timing evidence in `timing-baseline.json`
 # (or in any additional history payloads). Bumped from 20 s → 30 s alongside the
 # all-zero-history fix in `load_history`: a suite re-enabled after being

.github/scripts/tests/test_playwright_ci_planning.py

@@ -40,34 +40,40 @@ def test_basic_and_chromium_share_the_bounded_common_lane():
     planner = load_script("build_playwright_shards")
 
     assert planner.PROJECT_LANES["Basic"] == "chromium"
-    assert planner.lane_bounds("chromium", "full") == (5, 24)
+    assert planner.lane_bounds("chromium", "full") == (
+        5,
+        planner.COMMON_MAX_SHARDS,
+    )
 
 
-def test_full_common_shard_count_is_capped_at_24():
+def test_full_common_shard_count_is_capped_at_the_common_max():
     planner = load_script("build_playwright_shards")
+    cap = planner.COMMON_MAX_SHARDS
+    # Enough content to push the calculated shard count comfortably above
+    # the cap — the exact figure is unimportant; we just need
+    # `calculated > cap` so the min(cap, …) clamp is what returns.
     units = [
         planner.Unit(
             "chromium",
             f"{index}.spec.ts",
             str(index),
             weight_ms=1_000_000,
         )
-        for index in range(81)
+        for index in range(cap * 4)
     ]
-    units.append(
-        planner.Unit("chromium", "remainder.spec.ts", "remainder", weight_ms=363_055)
-    )
 
-    assert planner.shard_count(units, "chromium", "full") == 24
+    assert planner.shard_count(units, "chromium", "full") == cap
 
 
 def test_common_lane_carries_its_own_shard_budget():
-    # The chromium lane no longer sits a minute under TARGET_MS: the suite grew
-    # past what COMMON_MAX_SHARDS could hold at 19m, so it now runs a minute
-    # above the other lanes. Both still fit the 25m playwright timeout wrapper.
+    # Chromium's budget is a minute UNDER the other lanes' TARGET_MS, derived
+    # from the predicted→actual execution tail: actuals run up to 1.23× the
+    # prediction on noisy runs (run 32209040146), and 19 min × 1.32 is the
+    # break-even against the 25-minute playwright wrapper. See the derivation
+    # comment on COMMON_SHARD_BUDGET_MS.
     planner = load_script("build_playwright_shards")
 
-    assert planner.shard_budget_ms_for_lane("chromium") == 21 * 60 * 1000
+    assert planner.shard_budget_ms_for_lane("chromium") == 19 * 60 * 1000
     assert planner.shard_budget_ms_for_lane("search") == 20 * 60 * 1000
 
 
@@ -98,18 +104,26 @@ def test_common_assignment_stays_within_the_execution_ceiling():
     )
 
 
-def test_full_mode_chromium_converges_at_the_shard_ceiling():
-    # Regression guard for the merge-queue outage: a lane of this shape exhausted
-    # COMMON_MAX_SHARDS under the old 19m budget and aborted planning outright.
-    # The allocator must converge at or before the ceiling, and the resulting
-    # plan genuinely needs the window above 19m -- so quietly reverting the
-    # budget to 19m fails on the final assertion rather than only in CI.
+def test_full_mode_chromium_converges_above_the_old_24_shard_ceiling():
+    # Regression guard for BOTH historical outages on this lane:
+    # 1. Under budget=19m × cap=24, content past ~1163 worker-minutes
+    #    exhausted the cap and aborted planning outright (the #30784
+    #    outage — its stop-gap was raising the budget to 21m).
+    # 2. Under budget=21m, packed shards ran into the 25-minute wrapper on
+    #    tail-ratio runs (run 32209040146 — SIGTERM with 162/164 passed).
+    # The durable configuration is budget=19m × cap=28: the allocator must
+    # converge within the ceiling AND genuinely need more than 24 shards —
+    # so quietly reverting COMMON_MAX_SHARDS to 24 fails here rather than
+    # only in the merge queue. The content shape mirrors reality post
+    # audit-splitting: many fine-grained units (1300 × 1 min ≈ today's
+    # ~1200 worker-minutes of chromium content), not a few near-atomic
+    # blocks whose granularity would distort LPT balance.
     planner = load_script("build_playwright_shards")
     units = [
         planner.Unit(
-            "chromium", f"heavy-{index}.spec.ts", str(index), weight_ms=13 * 60 * 1000
+            "chromium", f"fine-{index}.spec.ts", str(index), weight_ms=60_000
         )
-        for index in range(96)
+        for index in range(1300)
     ]
 
     shards = planner.assign_lane_within_budget(units, "chromium", "full")
@@ -120,7 +134,7 @@ def test_full_mode_chromium_converges_at_the_shard_ceiling():
     )
     assert len(shards) <= planner.COMMON_MAX_SHARDS
     assert heaviest_ms <= planner.COMMON_SHARD_BUDGET_MS
-    assert heaviest_ms > 19 * 60 * 1000
+    assert len(shards) > 24
 
 
 def test_full_mode_chromium_reports_a_lane_the_ceiling_cannot_hold():
@@ -132,7 +146,7 @@ def test_full_mode_chromium_reports_a_lane_the_ceiling_cannot_hold():
         for index in range(120)
     ]
 
-    with pytest.raises(SystemExit, match=r"needs more than 24 shards"):
+    with pytest.raises(SystemExit, match=r"needs more than 28 shards"):
         planner.assign_lane_within_budget(units, "chromium", "full")
 
 
@@ -992,27 +1006,27 @@ def test_hook_heavy_subsuites_in_audited_suite_stay_atomic():
     ]
 
 
-def test_common_shards_enforce_the_twenty_one_minute_budget(tmp_path):
+def test_common_shards_enforce_the_nineteen_minute_budget(tmp_path):
     planner = load_script("build_playwright_shards")
     within_budget = planner.Unit(
         "chromium",
         "within.spec.ts",
         "within",
         grep_titles={("chromium", "within.spec.ts", "within")},
         test_ids={"within"},
-        weight_ms=21 * 60 * 1000,
+        weight_ms=19 * 60 * 1000,
     )
     above_budget = planner.Unit(
         "chromium",
         "above.spec.ts",
         "above",
         grep_titles={("chromium", "above.spec.ts", "above")},
         test_ids={"above"},
-        weight_ms=21 * 60 * 1000 + 1,
+        weight_ms=19 * 60 * 1000 + 1,
     )
 
     planner.write_plan(tmp_path, "chromium", 0, [within_budget])
-    with pytest.raises(SystemExit, match="above the 21-minute plan budget"):
+    with pytest.raises(SystemExit, match="above the 19-minute plan budget"):
         planner.write_plan(tmp_path, "chromium", 1, [above_budget])
 
 

.github/workflows/playwright-e2e-reusable.yml

@@ -592,22 +592,53 @@ jobs:
         uses: actions/setup-node@v5
         with:
           node-version-file: openmetadata-ui/src/main/resources/ui/.nvmrc
-          cache: yarn
-          cache-dependency-path: openmetadata-ui/src/main/resources/ui/yarn.lock
+
+      # setup-node's built-in `cache: yarn` is exact-key only — a yarn.lock
+      # change leaves the cache completely cold and yarn re-fetches all
+      # ~2000 tarballs from the registry, maximizing exposure to transient
+      # registry 5xx (run 32222855561: a single 502 on one tarball killed
+      # planning for the whole queue entry). An explicit cache with
+      # restore-keys falls back to the most recent lockfile's cache, so a
+      # lockfile change only fetches the delta.
+      - name: Restore yarn package cache
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/yarn
+          key: yarn-pkg-cache-${{ runner.os }}-${{ hashFiles('openmetadata-ui/src/main/resources/ui/yarn.lock') }}
+          restore-keys: |
+            yarn-pkg-cache-${{ runner.os }}-
 
       - name: Install dependencies
         working-directory: openmetadata-ui/src/main/resources/ui
         run: |
           corepack enable
-          yarn --ignore-scripts --frozen-lockfile
+          # Retry transient registry failures (5xx on tarball fetches) —
+          # yarn v1 does not retry HTTP errors on its own.
+          for attempt in 1 2 3; do
+            yarn --ignore-scripts --frozen-lockfile --network-timeout 100000 && break
+            if [[ "$attempt" -eq 3 ]]; then
+              echo "yarn install failed after 3 attempts" >&2
+              exit 1
+            fi
+            echo "::warning::yarn install attempt $attempt failed (registry blip?); retrying in $((attempt * 15))s"
+            sleep $((attempt * 15))
+          done
 
+      # These two steps SEED the run-scoped browser cache for the 30+ shard
+      # jobs downstream: the plan job finishes before any shard starts, so
+      # on a yarn.lock change the new browser-cache key is populated exactly
+      # once here instead of every shard downloading ~250 MB of Chromium in
+      # parallel. `--list` itself never launches a browser, so a seeding
+      # failure must not fail planning — shards self-install as a fallback.
       - name: Seed Playwright browser cache
+        continue-on-error: true
         uses: actions/cache@v5
         with:
           path: ~/.cache/ms-playwright
           key: ${{ runner.os }}-playwright-${{ hashFiles('openmetadata-ui/src/main/resources/ui/yarn.lock') }}
 
       - name: Install cached Chromium
+        continue-on-error: true
         working-directory: openmetadata-ui/src/main/resources/ui
         run: npx playwright install chromium
 
@@ -865,15 +896,32 @@ jobs:
         uses: actions/setup-node@v5
         with:
           node-version-file: openmetadata-ui/src/main/resources/ui/.nvmrc
-          cache: yarn
-          cache-dependency-path: openmetadata-ui/src/main/resources/ui/yarn.lock
+
+      # restore-keys fallback + retry — see the rationale on the
+      # plan-playwright job's yarn cache step.
+      - name: Restore yarn package cache
+        if: ${{ needs.restore-playwright-fixture.outputs.fixture_cache_hit != 'true' || (needs.plan-playwright.outputs.requires_airflow == 'true' && needs.restore-playwright-fixture.outputs.ingestion_cache_hit != 'true') }}
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/yarn
+          key: yarn-pkg-cache-${{ runner.os }}-${{ hashFiles('openmetadata-ui/src/main/resources/ui/yarn.lock') }}
+          restore-keys: |
+            yarn-pkg-cache-${{ runner.os }}-
 
       - name: Install fixture-state dependencies
         if: ${{ needs.restore-playwright-fixture.outputs.fixture_cache_hit != 'true' || (needs.plan-playwright.outputs.requires_airflow == 'true' && needs.restore-playwright-fixture.outputs.ingestion_cache_hit != 'true') }}
         working-directory: openmetadata-ui/src/main/resources/ui
         run: |
           corepack enable
-          yarn --ignore-scripts --frozen-lockfile
+          for attempt in 1 2 3; do
+            yarn --ignore-scripts --frozen-lockfile --network-timeout 100000 && break
+            if [[ "$attempt" -eq 3 ]]; then
+              echo "yarn install failed after 3 attempts" >&2
+              exit 1
+            fi
+            echo "::warning::yarn install attempt $attempt failed (registry blip?); retrying in $((attempt * 15))s"
+            sleep $((attempt * 15))
+          done
 
       - name: Restore Playwright browser cache
         if: ${{ needs.restore-playwright-fixture.outputs.fixture_cache_hit != 'true' || (needs.plan-playwright.outputs.requires_airflow == 'true' && needs.restore-playwright-fixture.outputs.ingestion_cache_hit != 'true') }}
@@ -1209,15 +1257,31 @@ jobs:
         uses: actions/setup-node@v5
         with:
           node-version-file: "openmetadata-ui/src/main/resources/ui/.nvmrc"
-          cache: yarn
-          cache-dependency-path: openmetadata-ui/src/main/resources/ui/yarn.lock
+
+      # restore-keys fallback + retry — see the rationale on the
+      # plan-playwright job's yarn cache step.
+      - name: Restore yarn package cache
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/yarn
+          key: yarn-pkg-cache-${{ runner.os }}-${{ hashFiles('openmetadata-ui/src/main/resources/ui/yarn.lock') }}
+          restore-keys: |
+            yarn-pkg-cache-${{ runner.os }}-
 
       - name: Install dependencies
         id: install-dependencies
         working-directory: openmetadata-ui/src/main/resources/ui/
         run: |
           corepack enable
-          yarn --ignore-scripts --frozen-lockfile
+          for attempt in 1 2 3; do
+            yarn --ignore-scripts --frozen-lockfile --network-timeout 100000 && break
+            if [[ "$attempt" -eq 3 ]]; then
+              echo "yarn install failed after 3 attempts" >&2
+              exit 1
+            fi
+            echo "::warning::yarn install attempt $attempt failed (registry blip?); retrying in $((attempt * 15))s"
+            sleep $((attempt * 15))
+          done
 
       - name: Restore Playwright browser cache
         id: restore-browser
@@ -1638,14 +1702,30 @@ jobs:
         uses: actions/setup-node@v5
         with:
           node-version-file: openmetadata-ui/src/main/resources/ui/.nvmrc
-          cache: yarn
-          cache-dependency-path: openmetadata-ui/src/main/resources/ui/yarn.lock
+
+      # restore-keys fallback + retry — see the rationale on the
+      # plan-playwright job's yarn cache step.
+      - name: Restore yarn package cache
+        uses: actions/cache@v5
+        with:
+          path: ~/.cache/yarn
+          key: yarn-pkg-cache-${{ runner.os }}-${{ hashFiles('openmetadata-ui/src/main/resources/ui/yarn.lock') }}
+          restore-keys: |
+            yarn-pkg-cache-${{ runner.os }}-
 
       - name: Install dependencies
         working-directory: openmetadata-ui/src/main/resources/ui/
         run: |
           corepack enable
-          yarn --ignore-scripts --frozen-lockfile
+          for attempt in 1 2 3; do
+            yarn --ignore-scripts --frozen-lockfile --network-timeout 100000 && break
+            if [[ "$attempt" -eq 3 ]]; then
+              echo "yarn install failed after 3 attempts" >&2
+              exit 1
+            fi
+            echo "::warning::yarn install attempt $attempt failed (registry blip?); retrying in $((attempt * 15))s"
+            sleep $((attempt * 15))
+          done
 
       - name: Download blob reports
         uses: actions/download-artifact@v7