Merge branch 'main' into perf/ai-context-strip-render-only-markup

Author: pmbrull

.claude/rules/frontend-performance.md

@@ -38,12 +38,31 @@ Applies to UI `*.{ts,tsx}`. Component/hook conventions in `frontend-react.md`; s
   Exception: `@openmetadata/ui-core-components` is consumed by bare package name — keep that.
 - **Never `import _ from 'lodash'` or `import * as _ from 'lodash'`.** Use named members only:
   `import { isEmpty, groupBy } from 'lodash';`.
-- **Lazy-load routes and heavy widgets** with `React.lazy` + `Suspense` (existing pattern:
-  `src/utils/CustomizeMyDataPageWidgetUtils.tsx`, `LineageProvider`). Anything importing a graph,
-  editor, chart, diff, or markdown renderer must be lazy — never in a route's top-level import.
+- **Lazy-load routes and heavy widgets** with `React.lazy`. Every lazy component must be passed to
+  an approved helper from `components/AppRouter/withSuspenseFallback`, or rendered in a real React
+  `Suspense` boundary with an explicit `fallback`. Anything importing a graph, editor, chart, diff,
+  or markdown renderer must be lazy — never in a route's top-level import.
+- **Router modules must not statically import page modules.** Runtime imports whose path contains
+  `pages/` must use `import()` so the page stays in a deferred chunk. Type-only imports are allowed.
+- Keep critical entity headers eager, then defer secondary tabs and expensive visualizations behind
+  the nearest useful loading boundary. Preserve `ref` behavior when wrapping a ref-sensitive lazy
+  component.
+- Prefer direct imports for pure utilities. Code splitting a small utility creates request overhead
+  without reducing meaningful render work.
 - Do not add a new dependency for something the repo already has, and never add a second UI
   component library (see `component-library.md`).
 
+## Loading and prefetching
+
+- Loading fallbacks belong at the smallest boundary that can render independently; avoid replacing
+  an entire page when only a secondary panel is deferred.
+- Prefetch only from a clear user signal or an established high-confidence navigation path. Do not
+  indiscriminately idle-prefetch settings, admin, graph, or editor chunks.
+- React Query prefetching is intentional cache warming: use the same query key and stale-time policy
+  as the destination query so it does not become a duplicate fetch.
+- When deferring extension or plugin requests, validate that no synchronous render path still needs
+  their data; moving the request later must not introduce a render-time waterfall.
+
 ## Re-renders
 
 - **Inline object/array/function props defeat `React.memo`** — a new identity every render makes the
@@ -72,3 +91,38 @@ Applies to UI `*.{ts,tsx}`. Component/hook conventions in `frontend-react.md`; s
 - Never render the full result set and hide overflow with CSS.
 - Debounce/throttle search inputs, `resize`, and `scroll` handlers; passive listeners for scroll.
 - Guard against layout thrash: don't read `getBoundingClientRect` in a loop that also writes styles.
+
+## Caches
+
+- Every module-level `Map` or `Set` used as a cache must have an explicit numeric or named uppercase
+  size limit and an eviction path using `delete` or `clear`. Prefer LRU behavior when stale entries
+  are inexpensive to recreate.
+- Cache hits and misses both consume capacity. Bound negative-result caches as carefully as successful
+  results, and dispose retained browser resources such as textures when evicting them.
+- Before adding a UI cache, check whether the API client, React Query, or a lower layer already owns
+  the same data. Avoid parallel caches with different invalidation rules.
+
+## Mechanically enforced rules
+
+ESLint blocks these zero-backlog invariants:
+
+- `openmetadata-performance/no-eager-page-imports` in AppRouter modules.
+- `openmetadata-performance/require-suspense-fallback` for every React lazy call.
+- `openmetadata-performance/no-unbounded-module-cache` for module-level cache-like `Map` and `Set`
+  bindings.
+
+ESLint also reports these backlog-bearing invariants as non-blocking warnings:
+
+- Pure utilities cannot contain JSX or depend on React, UI/state layers, pages, hooks, or REST.
+- Lower layers cannot import pages; page features cannot import other page features; REST and hooks
+  cannot depend upward on UI layers.
+- Runtime circular imports and app-internal barrel imports are reported; type-only imports do not
+  create runtime cycle/barrel findings.
+- Lodash default/namespace imports, REST calls inside iteration, and potentially sequential REST
+  calls are reported for direct-import, bulk-fetch, or parallelization review.
+
+The exact rule names, measured baselines, and CI visibility are documented in
+`docs/ui-code-quality-gate.md`.
+
+Use a narrow `eslint-disable-next-line` only when a test deliberately exercises the boundary helper
+itself and therefore owns the fallback in the test case. Include the reason beside the suppression.

.github/actions/setup-openmetadata-test-environment/action.yml

@@ -143,7 +143,7 @@ runs:
         source env/bin/activate
         uv pip install "setuptools<81"
         uv pip install --no-build-isolation "cx_Oracle>=8.3.0,<9"
-        uv pip install --no-deps "sqlalchemy-redshift==0.8.14" "sqlalchemy-ibmi==0.9.3" "pydoris-custom==1.1.0"
+        uv pip install --no-deps "sqlalchemy-ibmi==0.9.3" "pydoris-custom==1.1.0"
         uv pip install "${{ github.workspace }}/ingestion[all]"
         uv pip install "${{ github.workspace }}/ingestion[test]"
         uv pip install nox

.github/playwright/impact-map.json

@@ -95,6 +95,23 @@
       "projects": ["SearchRBAC"],
       "specs": ["playwright/e2e/Flow/SearchRBAC.spec.ts"]
     },
+    {
+      "sources": [
+        "openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/**",
+        "openmetadata-ui/src/main/resources/ui/src/utils/FeedUtils.ts",
+        "openmetadata-ui/src/main/resources/ui/src/utils/FeedUtilsPure.ts",
+        "openmetadata-ui/src/main/resources/ui/src/interface/feed.interface.ts",
+        "openmetadata-ui/src/main/resources/ui/src/rest/feedsAPI.ts",
+        "openmetadata-ui/src/main/resources/ui/src/rest/tasksAPI.ts"
+      ],
+      "specs": [
+        "playwright/e2e/Features/ActivityFeed.spec.ts",
+        "playwright/e2e/Features/Tasks/**/*.spec.ts",
+        "playwright/e2e/Pages/Tasks.spec.ts",
+        "playwright/e2e/Pages/TasksUIFlow.spec.ts",
+        "playwright/e2e/Pages/TaskComments.spec.ts"
+      ]
+    },
     {
       "sources": [
         "openmetadata-ui/src/main/resources/ui/src/components/Lineage/**",
@@ -207,6 +224,30 @@
         "playwright/e2e/Pages/TestSuite*.spec.ts"
       ]
     },
+    {
+      "sources": [
+        "openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/IngestionPipelineRepository.java",
+        "openmetadata-service/src/main/java/org/openmetadata/service/logstorage/**",
+        "openmetadata-service/src/main/java/org/openmetadata/service/sse/**",
+        "openmetadata-ui/src/main/resources/ui/src/components/ServiceAgents/**",
+        "openmetadata-ui/src/main/resources/ui/src/components/common/LogViewerModal/**",
+        "openmetadata-ui/src/main/resources/ui/src/hooks/useEntityLogs.ts",
+        "openmetadata-ui/src/main/resources/ui/src/hooks/useIngestionLogSource.ts",
+        "openmetadata-ui/src/main/resources/ui/src/hooks/useLogsModal.tsx",
+        "openmetadata-ui/src/main/resources/ui/src/hooks/usePaginatedLiveLog.ts",
+        "openmetadata-ui/src/main/resources/ui/src/hooks/usePollingEffect.ts",
+        "openmetadata-ui/src/main/resources/ui/src/utils/AgentsProgressPureUtils.ts",
+        "openmetadata-ui/src/main/resources/ui/src/utils/SseStreamUtils.ts",
+        "openmetadata-ui/src/main/resources/ui/src/utils/logsPolling.ts"
+      ],
+      "projects": ["chromium", "Ingestion"],
+      "specs": [
+        "playwright/e2e/Features/AgentLogStreamHandover.spec.ts",
+        "playwright/e2e/Pages/AppRunsHistoryLogs.spec.ts",
+        "playwright/e2e/Pages/IngestionLogStreamLive.spec.ts",
+        "playwright/e2e/Pages/LogsViewer.spec.ts"
+      ]
+    },
     {
       "sources": [
         "openmetadata-ui/src/main/resources/ui/src/components/Glossary/**",
@@ -477,6 +518,41 @@
         "playwright/e2e/Flow/ApiServiceRest.spec.ts",
         "playwright/e2e/Features/AutoPilot.spec.ts"
       ]
+    },
+    {
+      "_comment": "The landing-page Activity Feed widget is driven by ActivityFeedProvider and feedsAPI, and is exercised by specs that none of these paths live in. Without this entry a widget change selected only the specs it edited directly, and a broken filter reached the merge queue before any of these ran.",
+      "sources": [
+        "openmetadata-ui/src/main/resources/ui/src/components/MyData/**",
+        "openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/**",
+        "openmetadata-ui/src/main/resources/ui/src/rest/feedsAPI.ts"
+      ],
+      "projects": ["chromium", "Basic"],
+      "specs": [
+        "playwright/e2e/Features/ActivityFeed.spec.ts",
+        "playwright/e2e/Features/ActivityAPI.spec.ts",
+        "playwright/e2e/Features/ActivityStream.spec.ts",
+        "playwright/e2e/Features/ActivityFeedTabBadge.spec.ts",
+        "playwright/e2e/Flow/CustomizeWidgets.spec.ts"
+      ]
+    },
+    {
+      "_comment": "The customize-landing-page and widget-filter playwright helpers are shared by every landing-page/customization spec. Without this entry a helper-only change (e.g. a wait fix in customizeLandingPage.ts or widgetFilters.ts) counted as an unmapped file, so CustomizeWidgets/CustomizeLandingPage and the other specs that execute the changed helpers never ran on the PR. Specs listed are the exact importers of these modules; DataInsight.spec.ts runs in the Data Insight project.",
+      "sources": [
+        "openmetadata-ui/src/main/resources/ui/playwright/utils/customizeLandingPage.ts",
+        "openmetadata-ui/src/main/resources/ui/playwright/utils/widgetFilters.ts"
+      ],
+      "projects": ["chromium", "Data Insight"],
+      "specs": [
+        "playwright/e2e/Features/ActivityFeed.spec.ts",
+        "playwright/e2e/Features/CuratedAssets.spec.ts",
+        "playwright/e2e/Features/LandingPageWidgets/**/*.spec.ts",
+        "playwright/e2e/Features/NavigationBlocker.spec.ts",
+        "playwright/e2e/Features/SettingsNavigationPage.spec.ts",
+        "playwright/e2e/Flow/CustomizeLandingPage.spec.ts",
+        "playwright/e2e/Flow/CustomizeWidgets.spec.ts",
+        "playwright/e2e/Flow/PersonaFlow.spec.ts",
+        "playwright/e2e/Pages/DataInsight.spec.ts"
+      ]
     }
   ]
 }

.github/playwright/timing-baseline.json

@@ -144,6 +144,13 @@ def main() -> None:
             for entry in impact_map["canary"]:
                 add_selection(selected, entry, repo_root)
 
+        delegated_patterns = impact_map.get("delegatedSpecs", [])
+        selected = {
+            spec: projects
+            for spec, projects in selected.items()
+            if not matches(spec, delegated_patterns)
+        }
+
         plan = {
             "version": 1,
             "mode": "targeted",

.github/scripts/select_playwright_tests.py

@@ -2149,6 +2149,75 @@ def test_dedicated_rdf_specs_are_not_selected_by_the_main_workflow():
     )
 
 
+def test_impact_mapping_excludes_delegated_specs(tmp_path, monkeypatch):
+    selector = load_script("select_playwright_tests")
+    source_path = (
+        tmp_path / selector.UI_ROOT / "src/components/OntologyExplorer/view.ts"
+    )
+    source_path.parent.mkdir(parents=True)
+    source_path.write_text("export const view = {};\n")
+    spec_dir = tmp_path / selector.UI_ROOT / "playwright/e2e/Features"
+    spec_dir.mkdir(parents=True)
+    (spec_dir / "OntologyExplorer.spec.ts").write_text(
+        "test('ontology', () => undefined);\n"
+    )
+    (spec_dir / "OntologyExplorerRdf.spec.ts").write_text(
+        "test('rdf', () => undefined);\n"
+    )
+    impact_map = tmp_path / "impact-map.json"
+    impact_map.write_text(
+        json.dumps(
+            {
+                "smoke": [],
+                "canary": [],
+                "delegatedSpecs": [
+                    "playwright/e2e/Features/OntologyExplorerRdf.spec.ts"
+                ],
+                "sharedInfrastructure": [],
+                "mappings": [
+                    {
+                        "sources": [
+                            f"{selector.UI_ROOT}src/components/OntologyExplorer/**"
+                        ],
+                        "projects": ["chromium"],
+                        "specs": ["playwright/e2e/Features/OntologyExplorer*.spec.ts"],
+                    }
+                ],
+            }
+        )
+    )
+    changed = tmp_path / "changed.txt"
+    changed.write_text(f"{source_path.relative_to(tmp_path)}\n")
+    output = tmp_path / "selection.json"
+    monkeypatch.chdir(tmp_path)
+    monkeypatch.delenv("GITHUB_OUTPUT", raising=False)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            "select_playwright_tests.py",
+            "--event-name",
+            "pull_request_target",
+            "--changed-files",
+            str(changed),
+            "--impact-map",
+            str(impact_map),
+            "--output",
+            str(output),
+        ],
+    )
+
+    selector.main()
+
+    selection = json.loads(output.read_text())
+    assert selection["selectors"] == [
+        {
+            "projects": ["chromium"],
+            "spec": "playwright/e2e/Features/OntologyExplorer.spec.ts",
+        }
+    ]
+
+
 @pytest.mark.parametrize(
     "spec",
     [

.github/scripts/tests/test_playwright_ci_planning.py

@@ -115,8 +115,15 @@ jobs:
 
       - name: Start Keycloak SAML IdP
         if: startsWith(matrix.provider.name, 'keycloak-')
+        env:
+          # Must name the host this leg's browser uses, or Keycloak answers
+          # "Invalid Request". The '-crosssite' leg fronts the IdP on 127.0.0.1.
+          KEYCLOAK_HOSTNAME: ${{ matrix.provider.name == 'keycloak-azure-saml-crosssite' && 'http://127.0.0.1:8080' || 'http://localhost:8080' }}
         run: |
           docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml up -d
+          # The OM container needs to reach Keycloak by name; localhost:8080 is
+          # its own loopback. Tolerates a missing network and a repeat attach.
+          docker network connect ometa_network openmetadata-keycloak-saml || true
           timeout 180 bash -c 'until curl -fsS http://localhost:8080/realms/om-azure-saml >/dev/null; do sleep 2; done'
 
       - name: Run SSO Login Spec