Add Lakehouse Minio file picker in wizard and clickable file links in job detail

README.md

@@ -63,6 +63,18 @@ uv run jupyter lab
 
 With the watch command running, every saved change will immediately be built locally and available in your running JupyterLab. Refresh JupyterLab to load the change in your browser (you may need to wait several seconds for the extension to be rebuilt).
 
+#### Development commands
+
+| Command                  | Description                    |
+| ------------------------ | ------------------------------ |
+| `uv run jlpm build`      | Build extension (dev mode)     |
+| `uv run jlpm build:prod` | Build extension (production)   |
+| `uv run jlpm watch`      | Watch and rebuild on changes   |
+| `uv run jlpm lint`       | Run all linters with auto-fix  |
+| `uv run jlpm lint:check` | Run all linters without fixing |
+| `uv run jlpm test`       | Run Jest tests with coverage   |
+| `uv run jlpm clean:all`  | Clean all build artifacts      |
+
 #### Environment variables
 
 | Variable               | Description                                    |

berdl_task_browser/__init__.py

@@ -47,4 +47,7 @@ def _load_jupyter_server_extension(server_app):
     if os.environ.get("CTS_MOCK_MODE", "").lower() in ("true", "1", "yes"):
         page_config["ctsMockMode"] = "true"
 
+    from .handlers import setup_handlers
+    setup_handlers(server_app.web_app)
+
     server_app.log.info("Registered berdl_task_browser server extension")

berdl_task_browser/handlers.py

@@ -0,0 +1,77 @@
+"""S3 path mapping endpoint for the BERDL Task Browser server extension."""
+
+import json
+import logging
+import os
+from typing import Any
+
+from jupyter_server.base.handlers import APIHandler
+from jupyter_server.utils import url_path_join
+import tornado.web
+
+logger = logging.getLogger(__name__)
+
+
+class S3PathMappingsHandler(APIHandler):
+    """Exposes S3 path mappings from GroupedS3ContentsManager."""
+
+    @tornado.web.authenticated
+    def get(self) -> None:
+        try:
+            self.finish(json.dumps({"mappings": self._get_mappings()}))
+        except Exception:
+            logger.exception("Error reading S3 path mappings")
+            self.set_status(500)
+            self.finish(json.dumps({"error": "Internal server error"}))
+
+    def _get_mappings(self) -> dict[str, Any]:
+        cm = self.settings.get("contents_manager")
+        if cm is None:
+            return {}
+
+        # HybridContentsManager stores sub-managers in _managers dict.
+        managers = getattr(cm, "_managers", None)
+        if managers is None:
+            managers = getattr(cm, "managers", None)
+        if not isinstance(managers, dict):
+            return {}
+
+        s3_cm = managers.get("lakehouse_minio")
+        if s3_cm is None:
+            return {}
+
+        raw = getattr(s3_cm, "managers", None)
+        return raw if isinstance(raw, dict) else {}
+
+
+class MockServiceWorkerHandler(tornado.web.RequestHandler):
+    """Serve MSW's mockServiceWorker.js at the site root for mock mode."""
+
+    def get(self) -> None:
+        import pathlib
+
+        candidates = list(
+            pathlib.Path(__file__).parent.parent.glob(
+                "node_modules/msw/lib/mockServiceWorker.js"
+            )
+        )
+        if candidates:
+            self.set_header("Content-Type", "application/javascript")
+            self.finish(candidates[0].read_text())
+        else:
+            self.set_status(404)
+            self.finish("mockServiceWorker.js not found")
+
+
+def setup_handlers(web_app: Any) -> None:
+    """Register handlers with the Jupyter server."""
+    base_url = web_app.settings["base_url"]
+    handlers = [
+        (url_path_join(base_url, "api", "task-browser", "s3-path-mappings"), S3PathMappingsHandler),
+    ]
+
+    # Serve MSW service worker in mock mode
+    if os.environ.get("CTS_MOCK_MODE", "").lower() in ("true", "1", "yes"):
+        handlers.append(("/mockServiceWorker.js", MockServiceWorkerHandler))
+
+    web_app.add_handlers(".*$", handlers)

package.json

@@ -63,6 +63,9 @@
         "@fortawesome/react-fontawesome": "^0.2.2",
         "@jupyterlab/application": "^4.0.0",
         "@jupyterlab/apputils": "^4.0.0",
+        "@jupyterlab/coreutils": "^6.0.0",
+        "@jupyterlab/docmanager": "^4.0.0",
+        "@jupyterlab/filebrowser": "^4.0.0",
         "@jupyterlab/services": "^7.0.0",
         "@jupyterlab/statedb": "^4.0.0",
         "@jupyterlab/ui-components": "^4.0.0",

pyproject.toml

@@ -24,6 +24,7 @@ classifiers = [
 ]
 dependencies = [
     "ipywidgets>=8.0.0",
+    "jupyter_server>=2.0.0",
     "requests>=2.25.0",
 ]
 dynamic = ["version", "description", "authors", "urls", "keywords"]

src/api/ctsApi.ts

@@ -3,12 +3,14 @@
  */
 
 import { useQuery, useMutation, useQueryClient } from '@tanstack/react-query';
+import { PageConfig } from '@jupyterlab/coreutils';
 import {
   CTS_API_BASE,
   POLLING_INTERVAL_ACTIVE,
   POLLING_INTERVAL_LIST,
   DEFAULT_JOB_LIMIT
 } from '../config';
+import { S3Mappings, fetchS3Mappings } from '../utils/s3PathResolver';
 import {
   IJob,
   IJobStatus,
@@ -234,3 +236,13 @@ export function useJobLog(
     staleTime: 30000 // Cache logs for 30 seconds
   });
 }
+
+export function useS3Mappings(): S3Mappings | null {
+  const baseUrl = PageConfig.getBaseUrl();
+  const { data } = useQuery({
+    queryKey: ['s3-mappings'],
+    queryFn: () => fetchS3Mappings(baseUrl),
+    staleTime: Infinity
+  });
+  return data ?? null;
+}

src/api/mockData.ts

@@ -65,11 +65,15 @@ export const MOCK_SITES: ISite[] = [
 const createJobInput = (
   cluster: string,
   image: string,
-  params?: Record<string, unknown>
+  opts?: {
+    params?: Record<string, unknown>;
+    input_files?: string[];
+    output_dir?: string;
+  }
 ): IJobInput => ({
   cluster,
   image,
-  params
+  ...opts
 });
 
 export const MOCK_JOBS: IJob[] = [
@@ -88,8 +92,15 @@ export const MOCK_JOBS: IJob[] = [
       registered_on: '2024-01-01T00:00:00Z'
     },
     job_input: createJobInput('perlmutter-jaws', 'kbase/analysis-tool:v1.2.3', {
-      param1: 'value1',
-      param2: 42
+      params: { param1: 'value1', param2: 42 },
+      input_files: [
+        's3a://cdm-lake/users-general-warehouse/testuser/data/sample_001.fastq',
+        's3a://cdm-lake/users-general-warehouse/testuser/data/sample_002.fastq',
+        's3a://cdm-lake/users-general-warehouse/testuser/data/sample_003.fastq',
+        's3a://cdm-lake/tenant-general-warehouse/kbase/shared/reference_genome.fa',
+        's3a://cdm-lake/tenant-sql-warehouse/kbase/exports/metadata.parquet'
+      ],
+      output_dir: '/output'
     }),
     cpu_factor: 1.0,
     max_memory: '8Gi',
@@ -117,12 +128,25 @@ export const MOCK_JOBS: IJob[] = [
     },
     job_input: createJobInput(
       'perlmutter-jaws',
-      'kbase/genome-assembler:v2.0.0'
+      'kbase/genome-assembler:v2.0.0',
+      {
+        input_files: [
+          's3a://cdm-lake/users-general-warehouse/testuser/data/reads_R1.fastq',
+          's3a://cdm-lake/users-general-warehouse/testuser/data/reads_R2.fastq',
+          's3a://cdm-lake/tenant-general-warehouse/kbase/shared/ref.fa'
+        ],
+        output_dir: '/results'
+      }
     ),
     outputs: [
-      { file: 'assembly.fasta', crc64nvme: 'abc123' },
-      { file: 'stats.json' },
-      { file: 'log.txt' }
+      {
+        file: 's3a://cdm-lake/users-general-warehouse/testuser/data/assembly.fasta',
+        crc64nvme: 'abc123'
+      },
+      {
+        file: 's3a://cdm-lake/users-general-warehouse/testuser/data/stats.json'
+      },
+      { file: 's3a://cdm-lake/users-general-warehouse/testuser/data/log.txt' }
     ],
     transition_times: createTransitionTimes([
       { state: 'created', timeAgo: 180 },
@@ -151,7 +175,14 @@ export const MOCK_JOBS: IJob[] = [
     },
     job_input: createJobInput(
       'lawrencium-jaws',
-      'kbase/memory-intensive:latest'
+      'kbase/memory-intensive:latest',
+      {
+        input_files: [
+          's3a://cdm-lake/users-general-warehouse/testuser/data/big_dataset.h5',
+          's3a://other-bucket/external/reference.txt'
+        ],
+        output_dir: '/output'
+      }
     ),
     error:
       'Container exited with non-zero status: OutOfMemoryError - Java heap space exceeded. Consider increasing memory allocation or reducing input size.',
@@ -244,7 +275,14 @@ export const MOCK_JOBS: IJob[] = [
       registered_on: '2024-01-01T00:00:00Z'
     },
     job_input: createJobInput('kbase', 'kbase/legacy-tool:v0.9.0'),
-    outputs: [{ file: 'result1.txt' }, { file: 'result2.txt' }],
+    outputs: [
+      {
+        file: 's3a://cdm-lake/users-general-warehouse/testuser/data/result1.txt'
+      },
+      {
+        file: 's3a://cdm-lake/users-general-warehouse/testuser/data/result2.txt'
+      }
+    ],
     transition_times: createTransitionTimes([
       { state: 'created', timeAgo: 1500 },
       { state: 'download_submitted', timeAgo: 1498 },

src/components/CTSBrowser.tsx

@@ -1,9 +1,11 @@
 import React, { useState, useCallback, useEffect, useMemo } from 'react';
+import { JupyterFrontEnd } from '@jupyterlab/application';
 import { INotebookTracker } from '@jupyterlab/notebook';
+import { IDocumentManager } from '@jupyterlab/docmanager';
 import { Box, Typography, IconButton, Tooltip } from '@mui/material';
 import { FontAwesomeIcon } from '@fortawesome/react-fontawesome';
 import { faWandMagicSparkles } from '@fortawesome/free-solid-svg-icons';
-import { useJobs, useJobDetail } from '../api/ctsApi';
+import { useJobs, useJobDetail, useS3Mappings } from '../api/ctsApi';
 import { IJobFilters } from '../types/jobs';
 import { JobFilters } from './JobFilters';
 import { JobList } from './JobList';
@@ -13,10 +15,16 @@ import { registerSelectJobCallback } from '../index';
 import { getToken } from '../auth/token';
 
 export interface ICTSBrowserProps {
+  app: JupyterFrontEnd;
   notebookTracker: INotebookTracker | null;
+  documentManager: IDocumentManager | null;
 }
 
-export const CTSBrowser: React.FC<ICTSBrowserProps> = ({ notebookTracker }) => {
+export const CTSBrowser: React.FC<ICTSBrowserProps> = ({
+  app,
+  notebookTracker,
+  documentManager
+}) => {
   const [filters, setFilters] = useState<IJobFilters>({});
   const [selectedJobId, setSelectedJobId] = useState<string | null>(null);
 
@@ -29,6 +37,7 @@ export const CTSBrowser: React.FC<ICTSBrowserProps> = ({ notebookTracker }) => {
 
   const jobsQuery = useJobs(filters);
   const jobDetailQuery = useJobDetail(selectedJobId);
+  const s3Mappings = useS3Mappings();
 
   // Handlers
   const handleFiltersChange = useCallback((newFilters: IJobFilters) => {
@@ -48,8 +57,13 @@ export const CTSBrowser: React.FC<ICTSBrowserProps> = ({ notebookTracker }) => {
   }, []);
 
   const handleOpenWizard = useCallback(() => {
-    showJobWizardDialog(notebookTracker);
-  }, [notebookTracker]);
+    showJobWizardDialog(
+      notebookTracker,
+      undefined,
+      documentManager,
+      s3Mappings
+    );
+  }, [notebookTracker, documentManager, s3Mappings]);
 
   const statusSummary = useMemo(() => {
     const jobs = jobsQuery.data;
@@ -141,10 +155,13 @@ export const CTSBrowser: React.FC<ICTSBrowserProps> = ({ notebookTracker }) => {
           }}
         >
           <JobDetail
+            key={selectedJobId}
             job={jobDetailQuery.data}
             isLoading={jobDetailQuery.isLoading}
             error={jobDetailQuery.error}
             onClose={handleCloseDetail}
+            app={app}
+            s3Mappings={s3Mappings}
           />
         </Box>
       )}