Skip to content

Commit 4d25a6c

Browse files
alkaline-0alkaline-0
authored
feat/3198-add-workspace-info-and-profile-selection
Added a dropdown for profile selection in the dashboard interface and updated the layout to display profile and workspace information inline with pipeline selection.
1 parent 4224e88 commit 4d25a6c

File tree

7 files changed

+378
-115
lines changed

7 files changed

+378
-115
lines changed

‎.github/workflows/test_tools_dashboard.yml‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ jobs:
9696
if: matrix.python-version != '3.14.0-beta.4'
9797

9898
- name: Install playwright & deps
99-
run: playwright install && playwright install-deps
99+
run: playwright install chromium && playwright install-deps
100100
if: matrix.python-version != '3.14.0-beta.4'
101101

102102
# Run workspace dashboard unit tests
@@ -107,16 +107,8 @@ jobs:
107107
# Run workspace dashboard e2e tests (does not pass with python 3.9
108108
- name: Run dashboard e2e
109109
run: |
110-
marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines/ --with_test_identifiers true & pytest --browser chromium tests/e2e
111-
if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os != 'windows-latest'
112-
113-
# note that this test will pass only when running from cmd shell (_storage\.dlt\pipelines\ must stay)
114-
- name: Run dashboard e2e windows
115-
run: |
116-
start marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage\.dlt\pipelines\ --with_test_identifiers true
117-
timeout /t 6 /nobreak
118110
pytest --browser chromium tests/e2e
119-
if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4' && matrix.os == 'windows-latest'
111+
if: matrix.python-version != '3.9' && matrix.python-version != '3.14.0-beta.4'
120112

121113
matrix_job_required_check:
122114
name: common | common tests

‎Makefile‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -173,9 +173,6 @@ test-e2e-dashboard:
173173
test-e2e-dashboard-headed:
174174
uv run pytest --headed --browser chromium tests/e2e
175175

176-
start-dlt-dashboard-e2e:
177-
uv run marimo run --headless dlt/_workspace/helpers/dashboard/dlt_dashboard.py -- -- --pipelines-dir _storage/.dlt/pipelines --with_test_identifiers true
178-
179176
# creates the dashboard test pipelines globally for manual testing of the dashboard app and cli
180177
create-test-pipelines:
181178
uv run python tests/workspace/helpers/dashboard/example_pipelines.py

‎dlt/_workspace/helpers/dashboard/dlt_dashboard.py‎

Lines changed: 158 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,13 @@
1717
import pyarrow
1818
from dlt._workspace.helpers.dashboard import strings, utils, ui_elements as ui
1919
from dlt._workspace.helpers.dashboard.config import DashboardConfiguration
20+
from dlt.common.configuration.specs.pluggable_run_context import ProfilesRunContext
21+
from dlt._workspace.run_context import switch_profile
2022

2123

2224
@app.cell(hide_code=True)
2325
def home(
26+
dlt_profile_select: mo.ui.dropdown,
2427
dlt_all_pipelines: List[Dict[str, Any]],
2528
dlt_pipeline_select: mo.ui.multiselect,
2629
dlt_pipelines_dir: str,
@@ -41,16 +44,44 @@ def home(
4144
dlt_pipeline = utils.get_pipeline(dlt_pipeline_name, dlt_pipelines_dir)
4245

4346
dlt_config = utils.resolve_dashboard_config(dlt_pipeline)
44-
47+
_header_controls = (
48+
[
49+
dlt_profile_select,
50+
mo.md(f"<small> Workspace: {getattr(dlt.current.run_context(), 'name', None)}</small>"),
51+
]
52+
if isinstance(dlt.current.run_context(), ProfilesRunContext)
53+
else None
54+
)
4555
if not dlt_pipeline and not dlt_pipeline_name:
4656
_stack = [
4757
mo.hstack(
4858
[
49-
mo.image(
50-
"https://dlthub.com/docs/img/dlthub-logo.png", width=100, alt="dltHub logo"
59+
mo.hstack(
60+
[
61+
mo.image(
62+
"https://dlthub.com/docs/img/dlthub-logo.png",
63+
width=100,
64+
alt="dltHub logo",
65+
),
66+
_header_controls[0] if _header_controls else "",
67+
],
68+
justify="start",
69+
gap=2,
70+
),
71+
mo.hstack(
72+
[
73+
_header_controls[1] if _header_controls else "",
74+
],
75+
justify="center",
76+
),
77+
mo.hstack(
78+
[
79+
dlt_pipeline_select,
80+
],
81+
justify="end",
5182
),
52-
dlt_pipeline_select,
5383
],
84+
justify="space-between",
5485
),
5586
mo.md(strings.app_title).center(),
5687
mo.md(strings.app_intro).center(),
@@ -91,14 +122,65 @@ def home(
91122
[
92123
mo.hstack(
93124
[
94-
mo.image(
95-
"https://dlthub.com/docs/img/dlthub-logo.png",
96-
width=100,
97-
alt="dltHub logo",
98-
).style(padding_bottom="1em"),
99-
mo.center(mo.md(strings.app_title_pipeline.format(dlt_pipeline_name))),
100-
dlt_pipeline_select,
125+
mo.vstack(
126+
[
127+
mo.hstack(
128+
[
129+
mo.hstack(
130+
[
131+
mo.hstack(
132+
[
133+
mo.image(
134+
"https://dlthub.com/docs/img/dlthub-logo.png",
135+
width=100,
136+
alt="dltHub logo",
137+
),
138+
(
139+
_header_controls[0]
140+
if _header_controls
141+
else ""
142+
),
143+
],
144+
justify="start",
145+
gap=2,
146+
),
147+
mo.hstack(
148+
[
149+
(
150+
_header_controls[1]
151+
if _header_controls
152+
else ""
153+
),
154+
],
155+
justify="center",
156+
),
157+
mo.hstack(
158+
[
159+
dlt_pipeline_select,
160+
],
161+
justify="end",
162+
),
163+
],
164+
justify="center",
165+
),
166+
],
167+
),
168+
mo.center(
169+
mo.hstack(
170+
[
171+
mo.md(
172+
strings.app_title_pipeline.format(
173+
dlt_pipeline_name
174+
)
175+
),
176+
],
177+
align="center",
178+
),
179+
),
180+
]
181+
),
101182
],
183+
justify="space-between",
102184
),
103185
mo.hstack(_buttons, justify="start"),
104186
]
@@ -785,6 +867,7 @@ def section_ibis_backend(
785867

786868
@app.cell(hide_code=True)
787869
def utils_discover_pipelines(
870+
dlt_profile_select: mo.ui.dropdown,
788871
mo_cli_arg_pipelines_dir: str,
789872
mo_cli_arg_pipeline: str,
790873
mo_query_var_pipeline_name: str,
@@ -793,6 +876,13 @@ def utils_discover_pipelines(
793876
Discovers local pipelines and returns a multiselect widget to select one of the pipelines
794877
"""
795878

879+
_run_context = dlt.current.run_context()
880+
if (
881+
isinstance(_run_context, ProfilesRunContext)
882+
and not _run_context.profile == dlt_profile_select.value
883+
):
884+
switch_profile(dlt_profile_select.value)
885+
796886
# discover pipelines and build selector
797887
dlt_pipelines_dir: str = ""
798888
dlt_all_pipelines: List[Dict[str, Any]] = []
@@ -816,6 +906,39 @@ def utils_discover_pipelines(
816906
return dlt_all_pipelines, dlt_pipeline_select, dlt_pipelines_dir
817907

818908

909+
@app.cell(hide_code=True)
910+
def utils_discover_profiles(mo_query_var_profile: str, mo_cli_arg_profile: str):
911+
"""Discover profiles and return a single-select multiselect, similar to pipelines."""
912+
run_context = dlt.current.run_context()
913+
914+
# Default (non-profile-aware) output
915+
dlt_profile_select = mo.ui.dropdown(options=[], value=None, label="Profile: ")
916+
selected_profile = None
917+
918+
if isinstance(run_context, ProfilesRunContext):
919+
options = run_context.available_profiles() or []
920+
current = run_context.profile if options and run_context.profile in options else None
921+
922+
selected_profile = current
923+
if mo_query_var_profile and mo_query_var_profile in options:
924+
selected_profile = mo_query_var_profile
925+
elif mo_cli_arg_profile and mo_cli_arg_profile in options:
926+
selected_profile = mo_cli_arg_profile
927+
928+
def _on_profile_change(v: str) -> None:
929+
mo.query_params().set("profile", v)
930+
931+
dlt_profile_select = mo.ui.dropdown(
932+
options=options,
933+
value=selected_profile,
934+
label="Profile: ",
935+
on_change=_on_profile_change,
936+
searchable=True,
937+
)
938+
939+
return dlt_profile_select, selected_profile
940+
941+
819942
@app.cell(hide_code=True)
820943
def utils_discover_schemas(dlt_pipeline: dlt.Pipeline):
821944
"""
@@ -1040,25 +1163,40 @@ def utils_cli_args_and_query_vars_config():
10401163
"""
10411164
Prepare cli args as globals for the following cells
10421165
"""
1043-
1166+
_run_context = dlt.current.run_context()
1167+
mo_query_var_pipeline_name: str = None
1168+
mo_cli_arg_pipelines_dir: str = None
1169+
mo_cli_arg_with_test_identifiers: bool = False
1170+
mo_cli_arg_pipeline: str = None
1171+
mo_query_var_profile: str = None
1172+
mo_cli_arg_profile: str = None
10441173
try:
1045-
mo_query_var_pipeline_name: str = cast(str, mo.query_params().get("pipeline")) or None
1046-
mo_cli_arg_pipeline: str = cast(str, mo.cli_args().get("pipeline")) or None
1047-
mo_cli_arg_pipelines_dir: str = cast(str, mo.cli_args().get("pipelines-dir")) or None
1048-
mo_cli_arg_with_test_identifiers: bool = (
1174+
mo_query_var_pipeline_name = cast(str, mo.query_params().get("pipeline")) or None
1175+
mo_cli_arg_pipeline = cast(str, mo.cli_args().get("pipeline")) or None
1176+
mo_cli_arg_pipelines_dir = cast(str, mo.cli_args().get("pipelines-dir")) or None
1177+
mo_cli_arg_with_test_identifiers = (
10491178
cast(bool, mo.cli_args().get("with_test_identifiers")) or False
10501179
)
1180+
mo_query_var_profile = (
1181+
cast(str, mo.query_params().get("profile")) or None
1182+
if isinstance(_run_context, ProfilesRunContext)
1183+
else None
1184+
)
1185+
mo_cli_arg_profile = (
1186+
cast(str, mo.cli_args().get("profile")) or None
1187+
if isinstance(_run_context, ProfilesRunContext)
1188+
else None
1189+
)
10511190
except Exception:
1052-
mo_query_var_pipeline_name = None
1053-
mo_cli_arg_pipelines_dir = None
1054-
mo_cli_arg_with_test_identifiers = False
1055-
mo_cli_arg_pipeline = None
1191+
pass
10561192

10571193
return (
10581194
mo_cli_arg_pipelines_dir,
10591195
mo_cli_arg_with_test_identifiers,
10601196
mo_query_var_pipeline_name,
10611197
mo_cli_arg_pipeline,
1198+
mo_query_var_profile,
1199+
mo_cli_arg_profile,
10621200
)
10631201

10641202

‎dlt/_workspace/helpers/dashboard/runner.py‎

Lines changed: 79 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
1+
import contextlib
12
import os
23
import sys
34
import subprocess
45
from importlib.resources import files
5-
from typing import Any
6+
import time
7+
from typing import Any, Iterator, List
68
from pathlib import Path
9+
import urllib
10+
711
from dlt.common.exceptions import MissingDependencyException
812

913

@@ -39,7 +43,73 @@ def run_dashboard(
3943
pipelines_dir: str = None,
4044
port: int = None,
4145
host: str = None,
46+
with_test_identifiers: bool = False,
47+
headless: bool = False,
4248
) -> None:
49+
"""Run dashboard blocked"""
50+
try:
51+
subprocess.run(
52+
run_dashboard_command(
53+
pipeline_name, edit, pipelines_dir, port, host, with_test_identifiers, headless
54+
)
55+
)
56+
except KeyboardInterrupt:
57+
pass
58+
59+
60+
def _wait_http_up(url: str, timeout_s: float = 15.0, wait_on_ok: float = 0.1) -> None:
61+
start = time.time()
62+
while time.time() - start < timeout_s:
63+
try:
64+
with urllib.request.urlopen(url, timeout=1.0):
65+
time.sleep(wait_on_ok)
66+
return
67+
except Exception:
68+
time.sleep(0.1)
69+
raise TimeoutError(f"Server did not become ready: {url}")
70+
71+
72+
@contextlib.contextmanager
73+
def start_dashboard(
74+
pipelines_dir: str = None,
75+
port: int = 2718,
76+
test_identifiers: bool = True,
77+
headless: bool = True,
78+
wait_on_ok: float = 1.0,
79+
) -> Iterator[subprocess.Popen[bytes]]:
80+
"""Launches dashboard in context manager that will kill it after use"""
81+
command = run_dashboard_command(
82+
pipeline_name=None,
83+
edit=False,
84+
pipelines_dir=pipelines_dir,
85+
port=port,
86+
with_test_identifiers=test_identifiers,
87+
headless=headless,
88+
)
89+
# start the dashboard process using subprocess.Popen
90+
proc = subprocess.Popen(command)
91+
try:
92+
_wait_http_up(f"http://localhost:{port}", timeout_s=60.0, wait_on_ok=wait_on_ok)
93+
yield proc
94+
finally:
95+
proc.terminate()
96+
try:
97+
proc.wait(timeout=10)
98+
except subprocess.TimeoutExpired:
99+
proc.kill()
100+
proc.wait()
101+
102+
103+
def run_dashboard_command(
104+
pipeline_name: str = None,
105+
edit: bool = False,
106+
pipelines_dir: str = None,
107+
port: int = None,
108+
host: str = None,
109+
with_test_identifiers: bool = False,
110+
headless: bool = False,
111+
) -> List[str]:
112+
"""Creates cli command to run workspace dashboard"""
43113
from dlt._workspace.helpers.dashboard import dlt_dashboard
44114

45115
ejected_app_path = os.path.join(os.getcwd(), EJECTED_APP_FILE_NAME)
@@ -77,6 +147,9 @@ def run_dashboard(
77147
dashboard_cmd.append("--host")
78148
dashboard_cmd.append(host)
79149

150+
if headless:
151+
dashboard_cmd.append("--headless")
152+
80153
if pipeline_name:
81154
dashboard_cmd.append("--")
82155
dashboard_cmd.append("--pipeline")
@@ -85,8 +158,9 @@ def run_dashboard(
85158
dashboard_cmd.append("--")
86159
dashboard_cmd.append("--pipelines-dir")
87160
dashboard_cmd.append(pipelines_dir)
161+
if with_test_identifiers:
162+
dashboard_cmd.append("--")
163+
dashboard_cmd.append("--with_test_identifiers")
164+
dashboard_cmd.append("true")
88165

89-
try:
90-
subprocess.run(dashboard_cmd)
91-
except KeyboardInterrupt:
92-
pass
166+
return dashboard_cmd

0 commit comments

Comments
 (0)