[1.x] Package input files (no autodownload, no multiprocessing DownloadManager) (#510)

.github/workflows/build.yml

@@ -27,7 +27,8 @@ jobs:
       - name: Lint the extension
         run: |
           set -eux
-          jlpm
+          jlpm install
+          jlpm run lint:check
 
       - name: Test the extension
         run: |
@@ -94,59 +95,3 @@ jobs:
           jupyter labextension list
           jupyter labextension list 2>&1 | grep -ie "@jupyterlab/scheduler.*OK"
           python -m jupyterlab.browser_check --no-chrome-test
-
-  integration-tests:
-    name: Integration tests
-    needs: build
-    runs-on: ubuntu-latest
-
-    env:
-      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Base Setup
-        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
-
-      - name: Download extension package
-        uses: actions/download-artifact@v2
-        with:
-          name: extension-artifacts
-
-      - name: Install the extension
-        run: |
-          set -eux
-          python -m pip install "jupyterlab~=3.1" jupyter_scheduler*.whl
-
-      - name: Install dependencies
-        working-directory: ui-tests
-        env:
-          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-        run: jlpm install
-
-      - name: Set up browser cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ github.workspace }}/pw-browsers
-          key: ${{ runner.os }}-${{ hashFiles('ui-tests/yarn.lock') }}
-
-      - name: Install browser
-        run: jlpm playwright install chromium
-        working-directory: ui-tests
-
-      - name: Execute integration tests
-        working-directory: ui-tests
-        run: |
-          jlpm playwright test
-
-      - name: Upload Playwright Test report
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: jupyter_scheduler-playwright-tests
-          path: |
-            ui-tests/test-results
-            ui-tests/playwright-report

.github/workflows/e2e-tests.yml

@@ -0,0 +1,71 @@
+name: E2E Tests
+
+# suppress warning raised by https://github.com/jupyter/jupyter_core/pull/292
+env:
+  JUPYTER_PLATFORM_DIRS: '1'
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: '*'
+
+jobs:
+  e2e-tests:
+    name: Linux
+    runs-on: ubuntu-latest
+
+    env:
+      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Install extension dependencies
+        run: python -m pip install -U jupyterlab~=3.1
+
+      - name: Build the extension
+        run: |
+          set -eux
+          python -m pip install .
+
+          jupyter server extension list
+          jupyter server extension list 2>&1 | grep -ie "jupyter_scheduler.*OK"
+
+          jupyter labextension list
+          jupyter labextension list 2>&1 | grep -ie "@jupyterlab/scheduler.*OK"
+          python -m jupyterlab.browser_check
+
+      - name: Install ui-tests dependencies
+        working-directory: ui-tests
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: jlpm install
+
+      - name: Set up browser cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ github.workspace }}/pw-browsers
+          key: ${{ runner.os }}-${{ hashFiles('ui-tests/yarn.lock') }}
+
+      - name: Install browser
+        working-directory: ui-tests
+        run: jlpm playwright install chromium
+
+      - name: Run E2E Playwright tests
+        working-directory: ui-tests
+        run: jlpm test
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: jupyter_scheduler-playwright-tests-linux
+          path: |
+            ui-tests/test-results
+            ui-tests/playwright-report

.github/workflows/update-integration-tests.yml

@@ -2,6 +2,10 @@ name: Update Playwright Snapshots
 
 on:
   workflow_dispatch:
+    inputs:
+      number:
+        description: 'PR number'
+        required: true
   issue_comment:
     types: [created, edited]
 
@@ -10,10 +14,8 @@ permissions:
   pull-requests: write
 
 jobs:
-
-
   update-snapshots:
-    if: ${{ github.event.issue.pull_request && contains(github.event.comment.body, 'please update playwright snapshots') }}
+    if: ${{ github.event.inputs || (github.event.issue.pull_request && contains(github.event.comment.body, 'please update playwright snapshots')) }}
     runs-on: ubuntu-latest
 
     steps:
@@ -26,7 +28,7 @@ jobs:
         run: git config --global hub.protocol https
 
       - name: Checkout the branch from the PR that triggered the job
-        run: hub pr checkout ${{ github.event.issue.number }}
+        run: gh pr checkout ${{ github.event.inputs.number || github.event.issue.number }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
@@ -36,7 +38,7 @@ jobs:
           python_version: '3.11'
 
       - name: Install dependencies
-        run: python -m pip install -U jupyterlab~=3.1 jupyter-archive
+        run: python -m pip install -U jupyterlab~=3.1
 
       - name: Install extension
         run: |
@@ -50,3 +52,5 @@ jobs:
           # Playwright knows how to start JupyterLab server
           start_server_script: 'null'
           test_folder: ui-tests
+          # use jlpm (not default yet)
+          npm_client: jlpm

.gitignore

@@ -123,5 +123,15 @@ dmypy.json
 # job outputs in local development env
 dev/jobs
 
+# Notebook files created in dev folder
+dev/*.ipynb
+
 # jupyter releaser local checkout
 .jupyter_releaser_checkout
+
+# Yarn 3 cache
+.yarn
+
+# Jest coverage reports and a side effect
+coverage
+junit.xml

.readthedocs.yml

@@ -0,0 +1,17 @@
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+    nodejs: "18"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

CHANGELOG.md

@@ -2,6 +2,128 @@
 
 <!-- <START NEW CHANGELOG ENTRY> -->
 
+## 1.8.2
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.8.1...95138f9b4d7aed5a356aee1e03b79f26c8470af0))
+
+### Bugs fixed
+
+- \[1.x\] Changed column header to "Input file" in Notebook Job Definitions (#496) [#501](https://github.com/jupyter-server/jupyter-scheduler/pull/501) ([@srdas](https://github.com/srdas))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2024-02-15&to=2024-04-15&type=c))
+
+[@andrii-i](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Aandrii-i+updated%3A2024-02-15..2024-04-15&type=Issues) | [@github-actions](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Agithub-actions+updated%3A2024-02-15..2024-04-15&type=Issues) | [@srdas](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Asrdas+updated%3A2024-02-15..2024-04-15&type=Issues) | [@welcome](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Awelcome+updated%3A2024-02-15..2024-04-15&type=Issues)
+
+<!-- <END NEW CHANGELOG ENTRY> -->
+
+## 1.8.1
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.8.0...2e3ccf021dba582464fbab33c4bec129c0380cc2))
+
+### Bugs fixed
+
+- \[1.x\] Make server extension verification call during extension startup non-blocking (#480) [#481](https://github.com/jupyter-server/jupyter-scheduler/pull/481) ([@andrii-i](https://github.com/andrii-i))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2024-01-23&to=2024-02-15&type=c))
+
+[@andrii-i](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Aandrii-i+updated%3A2024-01-23..2024-02-15&type=Issues) | [@dlqqq](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Adlqqq+updated%3A2024-01-23..2024-02-15&type=Issues)
+
+## 1.8.0
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.7.0...4447c3ba6d53dd73b13be2b376163935a9ac1d15))
+
+### Enhancements made
+
+- \[1.x\] Emit telemetry events on success and failure of the Create Job, Create Job Definition, Create Job from Job Definition hooks (#472) [#475](https://github.com/jupyter-server/jupyter-scheduler/pull/475) ([@andrii-i](https://github.com/andrii-i))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2023-11-16&to=2024-01-23&type=c))
+
+[@andrii-i](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Aandrii-i+updated%3A2023-11-16..2024-01-23&type=Issues)
+
+## 1.7.0
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.6.0...242ff36d4ba22ebfebf42b9b68a170af16c2e6f1))
+
+### Enhancements made
+
+- \[1.x\] Add `dev-install` script [#467](https://github.com/jupyter-server/jupyter-scheduler/pull/467) ([@dlqqq](https://github.com/dlqqq))
+- Backport 463 to 1.x [#465](https://github.com/jupyter-server/jupyter-scheduler/pull/465) ([@JasonWeill](https://github.com/JasonWeill))
+
+### Bugs fixed
+
+- Backport PR 461 to 1.x [#466](https://github.com/jupyter-server/jupyter-scheduler/pull/466) ([@JasonWeill](https://github.com/JasonWeill))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2023-10-31&to=2023-11-16&type=c))
+
+[@dlqqq](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Adlqqq+updated%3A2023-10-31..2023-11-16&type=Issues) | [@JasonWeill](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3AJasonWeill+updated%3A2023-10-31..2023-11-16&type=Issues)
+
+## 1.6.0
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.5.0...9600bfb41a369181f411b321f9fb51d409465b76))
+
+### Enhancements made
+
+- Backport PRs 448, 457, 459 [#460](https://github.com/jupyter-server/jupyter-scheduler/pull/460) ([@3coins](https://github.com/3coins))
+
+### Bugs fixed
+
+- Backport PRs 448, 457, 459 [#460](https://github.com/jupyter-server/jupyter-scheduler/pull/460) ([@3coins](https://github.com/3coins))
+- Add test:update script to 1.x ui-tests [#456](https://github.com/jupyter-server/jupyter-scheduler/pull/456) ([@dlqqq](https://github.com/dlqqq))
+- \[1.x\] Fix "event loop is already running" bug on Linux [#454](https://github.com/jupyter-server/jupyter-scheduler/pull/454) ([@dlqqq](https://github.com/dlqqq))
+
+### Other merged PRs
+
+- \[1.x\] Migrate from hub to gh in workflows [#455](https://github.com/jupyter-server/jupyter-scheduler/pull/455) ([@dlqqq](https://github.com/dlqqq))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2023-10-13&to=2023-10-31&type=c))
+
+[@3coins](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3A3coins+updated%3A2023-10-13..2023-10-31&type=Issues) | [@dlqqq](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Adlqqq+updated%3A2023-10-13..2023-10-31&type=Issues) | [@Zsailer](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3AZsailer+updated%3A2023-10-13..2023-10-31&type=Issues)
+
+## 1.5.0
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.4.0...ccaa72baeb3e5d135d48166073e415338c7bc1e9))
+
+### Maintenance and upkeep improvements
+
+- \[1.x\] Fix RTD CI workflow [#444](https://github.com/jupyter-server/jupyter-scheduler/pull/444) ([@andrii-i](https://github.com/andrii-i))
+- \[1.x\] Remove unused s3fs dependency [#443](https://github.com/jupyter-server/jupyter-scheduler/pull/443) ([@dlqqq](https://github.com/dlqqq))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2023-08-15&to=2023-10-13&type=c))
+
+[@andrii-i](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Aandrii-i+updated%3A2023-08-15..2023-10-13&type=Issues) | [@dlqqq](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Adlqqq+updated%3A2023-08-15..2023-10-13&type=Issues)
+
+## 1.4.0
+
+([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.3.4...51230951c3d865c6e6ce39c1623cf7d5e41d227d))
+
+### Enhancements made
+
+- \[1.x\] Add version specifiers for all dependencies [#427](https://github.com/jupyter-server/jupyter-scheduler/pull/427) ([@dlqqq](https://github.com/dlqqq))
+- \[1.x\] Fix CI, run lint, reduce end-to-end tests flakiness [#425](https://github.com/jupyter-server/jupyter-scheduler/pull/425) ([@andrii-i](https://github.com/andrii-i))
+- \[1.x\] Archiving scheduler, fix JFM tests [#418](https://github.com/jupyter-server/jupyter-scheduler/pull/418) ([@JasonWeill](https://github.com/JasonWeill))
+
+### Bugs fixed
+
+- \[1.x\] Fix JFM tests [#426](https://github.com/jupyter-server/jupyter-scheduler/pull/426) ([@dlqqq](https://github.com/dlqqq))
+
+### Contributors to this release
+
+([GitHub contributors page for this release](https://github.com/jupyter-server/jupyter-scheduler/graphs/contributors?from=2023-07-03&to=2023-08-15&type=c))
+
+[@andrii-i](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Aandrii-i+updated%3A2023-07-03..2023-08-15&type=Issues) | [@dlqqq](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Adlqqq+updated%3A2023-07-03..2023-08-15&type=Issues) | [@github-actions](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Agithub-actions+updated%3A2023-07-03..2023-08-15&type=Issues) | [@JasonWeill](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3AJasonWeill+updated%3A2023-07-03..2023-08-15&type=Issues)
+
 ## 1.3.4
 
 ([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.3.3...aaa3a8922a3021b1158c668e50b17e2bf708b4e6))
@@ -16,8 +138,6 @@
 
 [@github-actions](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3Agithub-actions+updated%3A2023-06-27..2023-07-03&type=Issues) | [@JasonWeill](https://github.com/search?q=repo%3Ajupyter-server%2Fjupyter-scheduler+involves%3AJasonWeill+updated%3A2023-06-27..2023-07-03&type=Issues)
 
-<!-- <END NEW CHANGELOG ENTRY> -->
-
 ## 1.3.3
 
 ([Full Changelog](https://github.com/jupyter-server/jupyter-scheduler/compare/v1.3.2...5cd14b538a656e8fd2318dbbb687f2bf4da8fd37))

docs/operators/index.md

@@ -86,6 +86,20 @@ jupyter lab --SchedulerApp.job_files_manager_class=jupyter_scheduler.job_files_m
 
 For more information on writing a custom implementation, please see the {doc}`developer's guide </developers/index>`.
 
+### Example: Capturing side effect files
+
+The default scheduler and execution manager classes do not capture
+**side effect files** (files that are created as a side effect of executing
+cells in a notebook) unless “Run job with input folder” is checked. The `ArchivingScheduler` and `ArchivingExecutionManager`
+classes do capture side effect files by default. If you intend to run notebooks that produce
+side effect files, you can use these classes by running:
+
+```
+jupyter lab \
+  --SchedulerApp.scheduler_class=jupyter_scheduler.scheduler.ArchivingScheduler \
+  --Scheduler.execution_manager_class=jupyter_scheduler.executors.ArchivingExecutionManager
+```
+
 ## UI configuration
 
 You can configure the Jupyter Scheduler UI by installing a lab extension that both:

docs/users/index.md

@@ -44,6 +44,10 @@ To create a _job_ or _job definition_ from an open notebook, click on a “Creat
 
 Give your notebook job or job definition a name, choose an environment to run it in, select its output formats, and provide parameters that are set as local variables when your notebook gets executed. This parameterized execution is similar to the one used in [Papermill](https://papermill.readthedocs.io/en/latest/).
 
+If you check "Run job with input folder", the scheduled job will have access to all files within the same folder as the input file.
+The scheduler will copy all files from the input file to a staging directory at runtime, and will make these files and any side effect files created during the job run available for download after the job has finished.
+Use caution with this option if your input file's directory has many large files in it.
+
 To create a _job_ that runs once, select "Run now" in the "Schedule" section, and click "Create".
 !["Create Job Form"](./images/create_job_form.png)
 

jupyter_scheduler/_version.py

@@ -3,5 +3,5 @@
 
 __all__ = ["__version__"]
 
-version_info = (1, 3, 4, "", "")
+version_info = (1, 8, 2, "", "")
 __version__ = ".".join(map(str, version_info[:3])) + "".join(version_info[3:])