Redo the scan grouping so it uses modern BIDS and is easier to understand (#1092)

* initial try at a grouping module/cli

* add some more checks, warnings

* use new groups at the beginning of qsiprep

* docs issue

* update base.py to use new grouping

* first pass into the workflows

* propagate grouping to more workflows

* keep going

* fix summary bug

* fix drbuddi reports

* fix remaining tests

* use extremely cool grouping notebook for docs

* demote intendedfor

* limit full text testing

* switch to myst ipynb

* use real bids fields for the grouping tutorial

* [DATALAD RUNCMD] bash -lc 'if pixi lock --check; then ech...

=== Do not change lines below ===
{
 "chain": [],
 "cmd": "bash -lc 'if pixi lock --check; then echo '\"'\"'Lockfile up to date'\"'\"'; else pixi lock; fi'",
 "exit": 0,
 "extra_inputs": [],
 "inputs": [
  "pixi.lock",
  "pyproject.toml"
 ],
 "outputs": [
  "pixi.lock"
 ],
 "pwd": "."
}
^^^ Do not change lines above ^^^

* start implementing the final default concatenation

* CI failure

* rpe_drbuddi merge

* continue to try to fix CI

* tsv/csv issues when concatenating

* handle tortoise splits and fallback for unpaired

* add support for MultipartID lists

* require unique b0fields across sessions

* rename "benchmarking" to "virtual acquisition"

---------

Co-authored-by: nipreps[bot] <bot@nipreps.org>

Author: mattcieslak

.gitignore

@@ -100,6 +100,10 @@ scratch/
 notebooks/*
 !notebooks/*.ipynb
 
+# The grouping tutorial is committed as a MyST .md notebook; ipynb copies
+# converted with jupytext for interactive use stay local.
+docs/notebooks/*.ipynb
+
 # pixi environments
 .pixi/*
 !.pixi/config.toml

docs/comparisons.md

@@ -47,4 +47,4 @@ their current feature sets. These other
 |                               | QSIPrep           | Tractoflow | PreQual | MRtrix3_connectome |
 | ----------------------------- | :---------------: | :--------: | :-----: | :----------------: |
 | Automated methods boilerplate |         ✔         |     ✘      |    ✘    |         ✘          |
-| HTML Preprocessing Report     | [NiWorkflows-based](preprocessing.html#visual-reports) |     ✘      | Custom  |      EddyQuad      |
+| HTML Preprocessing Report     | {ref}`NiWorkflows-based <visual_reports>` |     ✘      | Custom  |      EddyQuad      |

docs/conf.py

@@ -43,22 +43,27 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'nbsphinx',
+    'myst_nb',  # notebooks and markdown (replaces nbsphinx + recommonmark)
     'nipype.sphinxext.apidoc',
     'nipype.sphinxext.plot_workflow',
-    'recommonmark',
     'sphinx.ext.autodoc',
     'sphinx.ext.coverage',
     'sphinx.ext.doctest',
     'sphinx.ext.intersphinx',
     'sphinx.ext.linkcode',
     'sphinx.ext.mathjax',
-    'sphinx_markdown_tables',
     'sphinxarg.ext',  # argparse extension
     'sphinxcontrib.apidoc',
     'sphinxcontrib.bibtex',
 ]
 
+# Notebooks are MyST markdown (clean git diffs, no committed outputs) and
+# are executed during the docs build, which doubles as a regression test
+# for their demos. 'cache' skips re-execution when a notebook is unchanged.
+nb_execution_mode = 'cache'
+nb_execution_raise_on_error = True
+nb_execution_timeout = 300
+
 # Mock modules in autodoc:
 autodoc_mock_imports = [
     'matplotlib',
@@ -115,7 +120,9 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+# notebooks/*.ipynb are local jupytext conversions of the committed MyST .md
+# notebooks (gitignored); exclude them so sphinx never prefers a stale copy.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'notebooks/*.ipynb']
 
 # The reST default role (used for this markup: `text`) to use for all
 # documents.

docs/index.rst

@@ -17,6 +17,7 @@ Contents
    quickstart
    usage
    preprocessing
+   notebooks/grouping_tutorial
    reconstruction
    contributors
    changes