BodenmillerGroup · Milad4849 · Feb 24, 2023 · Feb 24, 2023 · Feb 24, 2023 · Apr 10, 2023
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -17,7 +17,7 @@ on:
 jobs:
 
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     steps:
 
       - name: Checkout repository
@@ -46,7 +46,7 @@ jobs:
   pypi:
     needs: test
     if: startsWith(github.ref, 'refs/tags')
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     steps:
 
       - name: Checkout repository
@@ -74,8 +74,10 @@ jobs:
   ghcr:
     needs: test
     if: startsWith(github.ref, 'refs/tags')
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
+      - name: Print GitHub context
+        run: echo "GITHUB_REF=$GITHUB_REF"
 
       - name: Checkout repository
         uses: actions/checkout@v3

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -39,12 +39,12 @@ jobs:
           git config user.name github-actions
           git config user.email [email protected]
           git fetch origin gh-pages --verbose
-          mike deploy -p -u --rebase ${GITHUB_REF#refs/tags/}
+          mike deploy -p -u ${GITHUB_REF#refs/tags/}
 
       - name: Build documentation (latest)
         if: github.ref == 'refs/heads/main'
         run: |
           git config user.name github-actions
           git config user.email [email protected]
           git fetch origin gh-pages --verbose
-          mike deploy -p -u --rebase latest
+          mike deploy -p -u latest
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 /data/
 /steinbock/_version.py
+images.csv
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 exclude: ^\.vscode/.*
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: check-added-large-files
       - id: check-case-conflict
@@ -16,25 +16,25 @@ repos:
       - id: requirements-txt-fixer
       - id: trailing-whitespace
   - repo: https://github.com/PyCQA/isort
-    rev: "5.12.0"
+    rev: "5.13.2"
     hooks:
       - id: isort
   - repo: https://github.com/PyCQA/autoflake
-    rev: v2.0.1
+    rev: v2.3.1
     hooks:
       - id: autoflake
         args: [--in-place, --remove-all-unused-imports]
   - repo: https://github.com/psf/black
-    rev: '23.1.0'
+    rev: '24.8.0'
     hooks:
       - id: black
   - repo: https://github.com/PyCQA/flake8
-    rev: "6.0.0"
+    rev: "7.1.1"
     hooks:
       - id: flake8
         additional_dependencies: [flake8-typing-imports]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.0.0
+    rev: v1.11.1
     hooks:
       - id: mypy
         additional_dependencies: [types-requests, types-PyYAML]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,16 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.16.3] - 2024-05-29
+
+### Changed
+
+  - Updated readimc to 0.7.0
+
+### Added
+
+  - Added the option for lenient parsing of corrupted ROIs
+
 ## [0.16.2] - 2023-08-15
 
 ### Changed
@@ -376,7 +386,8 @@ Added:
 
 Initial release for beta testing
 
-
+[0.17.0]: https://github.com/BodenmillerGroup/steinbock/compare/v0.14.2...v0.17.0
+[0.16.2]: https://github.com/BodenmillerGroup/steinbock/compare/v0.14.2...v0.16.2
 [0.16.1]: https://github.com/BodenmillerGroup/steinbock/compare/v0.16.0...v0.16.1
 [0.16.0]: https://github.com/BodenmillerGroup/steinbock/compare/v0.15.0...v0.16.0
 [0.15.0]: https://github.com/BodenmillerGroup/steinbock/compare/v0.14.2...v0.15.0

diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md
@@ -4,6 +4,11 @@
 
 [[email protected]](mailto:[email protected])
 
-  - Logo
-  - Documentation
-  - Testing, feedback & discussion
+- Logo
+- Documentation
+- Testing, feedback & discussion
+
+
+## Milad Adibi
+
+[[email protected]](mailto:[email protected])
diff --git a/README.md b/README.md
@@ -14,3 +14,18 @@
 A toolkit for processing multiplexed tissue images
 
 Documentation is available at https://bodenmillergroup.github.io/steinbock
+
+## Citation
+
+Please cite the following paper when using `steinbock` in your work:
+
+>  Windhager, J., Zanotelli, V.R.T., Schulz, D. et al. An end-to-end workflow for multiplexed image processing and analysis. Nat Protoc (2023). https://doi.org/10.1038/s41596-023-00881-0
+
+    @article{Windhager2023,
+        author = {Windhager, Jonas and Zanotelli, Vito R.T. and Schulz, Daniel and Meyer, Lasse and Daniel, Michelle and Bodenmiller, Bernd and Eling, Nils},
+        title = {An end-to-end workflow for multiplexed image processing and analysis},
+        year = {2023},
+        doi = {10.1038/s41596-023-00881-0},
+        URL = {https://www.nature.com/articles/s41596-023-00881-0},
+        journal = {Nature Protocols}
+    }
diff --git a/docs/cli/preprocessing.md b/docs/cli/preprocessing.md
@@ -76,6 +76,10 @@ After image extraction, if the `--hpf` option is specified, the images are filte
 
 [^1]: Zanotelli et al. ImcSegmentationPipeline: A pixel classification-based multiplexed image segmentation pipeline. Zenodo, 2017. DOI: [10.5281/zenodo.3841961](https://doi.org/10.5281/zenodo.3841961).
 
+### Support for XTI imc machines
+
+The new generation of imc machines do not provide back-up text files of individual ROIs any more. Instead they provide individual mcd files per ROI as back-up. In some instances the downstream analysis pipelines are configured to use the text files as input. Therefore in order to maintain compatibility with these pieplines, we now provide the option to generate text files for ROIs via the command: `steinbock preprocess imc images --xti`.
+
 ## External images
 
 *External images* are images preprocessed externally (i.e., without *steinbock*) that are saved in an image format supported by [imageio](https://imageio.readthedocs.io).

diff --git a/docs/cli/segmentation.md b/docs/cli/segmentation.md
@@ -7,6 +7,12 @@ Various segmentation approaches are supported, each of which is described in the
 !!! note "Pixel classification-based image segmentation vs end-to-end approaches"
     While [pixel classification](classification.md)-based image segmentation using CellProfiler uses probability images to segment objects, end-to-end workflows such as DeepCell/Mesmer and Cellpose directly operate on images without the need for a preceding pixel classification step.
 
+
+!!! note "Large masks containing more than 65535 objects"
+  Please refer to the [installation page](../install-docker.md#Using-the-docker-container-with-large-images-containing-more-than-65535-objects) when analizing large masks with more than 65535 objects.
+
+
+
 ## CellProfiler
 
 [CellProfiler](https://cellprofiler.org) is an open-source software for measuring and analyzing cell images. Here, CellProfiler is used for object detection and region growth-based object segmentation.
@@ -87,7 +93,7 @@ This will create grayscale cell/nuclear masks of the same x and y dimensions as
     Specify `--minmax` to enable min-max normalization and `--zscore` to enable z-score normalization.
 
 !!! note "Preprocessing/postprocessing parameters"
-    Application-dependent preprocessing/postprocessing parameters can be specified in YAML files using the `--preprocess`/`--postprocess` options. For the Mesmer application, this can e.g. be used to control thresholding, histogram normalization and watershed segmentation. Please refer to the DeepCell online documentation for available parameters. For example, one could specify `--preprocess preprocessing.yml`, where `preprocessing.yml` is a file in the steinbock data/working directory containing:
+    Application-dependent preprocessing/postprocessing parameters can be specified in YAML files using the `--preprocess`/`--postprocess` options. For the Mesmer application, this can e.g. be used to control thresholding, histogram normalization and watershed segmentation. Please refer to the DeepCell online documentation for available parameters. For example, one could specify `--preprocess preprocessing.yml`, where `preprocessing.yml` is a file in the *steinbock* data/working directory containing:
 
         threshold: true
         percentile: 99.9
@@ -99,20 +105,23 @@ This will create grayscale cell/nuclear masks of the same x and y dimensions as
 !!! danger "Experimental feature"
     This is an experimental feature and is only available in the `-cellpose` flavors of the *steinbock* Docker container.
 
-    Segmentation using cellpose likely requires fine-tuning of parameters, e.g. using steinbock command-line interface options.
+    Segmentation using cellpose likely requires fine-tuning of parameters, e.g. using steinbock command-line interface options. The steinbock default parameters for the cellpose models (cell probability threshhold, flow threshhold etc) are not the same as the default values of *cellpose* itself. The steinbock default values are the result of manual optimization of cellpose parameters for segmentation of IMC images in the Bodenmiller lab. This default values can be viewed using the command `steinbock segment cellpose run --help`.
 
 [Cellpose](https://www.cellpose.org) is a generalist algorithm for cellular segmentation.
 
 !!! note "End-to-end cell segmentation"
     This approach operates directly on image intensities and does not require a preceding pixel classification step.
 
-To segment cells using the default `cyto2` model:
+To segment cells using the default `tissuenet` model:
 
-    steinbock segment cellpose --minmax
+    steinbock segment cellpose run --minmax
 
 To segment nuclei using the `nuclei` model:
 
-    steinbock segment cellpose --minmax --model nuclei
+    steinbock segment cellpose run --minmax --model nuclei
+
+Models introduced in [cellpose 2.0](https://www.nature.com/articles/s41592-022-01663-4) can also be chosen, bringing the total number of available models to 14, these are 'nuclei',
+            'cyto', 'cyto2', 'tissuenet', 'livecell', 'CP', 'CPx', 'TN1', 'TN2', 'TN3', 'LC1', 'LC2', 'LC3' and 'LC4'. The user can utilize any of these models by specifying them using the `--model` argument as demonstrated above. The default is set to `tissuenet` model, please note that `tissuenet` is also the default model for training a cellpose model (See training a cellpose model below). Furthermore the path to a user-trained model (see [`Training a cellpose model` below](### Training a cellpose model)) can be used via the `-pretrained-model` argument.
 
 !!! note "Cellpose image data"
     Cellpose expects two-channel images as input, where the first channel must be a nuclear channel (e.g. DAPI) and the second channel must be a cytoplasmic channel (e.g. E-Cadherin). The nuclear channel is optional and only the cytoplasmic channel ("channel to segment") is required. Note that - compared to the original cellpose implementation - the channel order is reversed for compatibility with DeepCell/Mesmer.
@@ -121,6 +130,21 @@ To segment nuclei using the `nuclei` model:
 
     If no `cellpose` column is present, images are expected to be in the correct format already.
 
+### Training a cellpose model
+Training a cellpose model is performed using two commands
+
+      steinbock segment cellpose preapre
+      steinbock segment cellpose train
+
+By default the first command generates crops of images in `img` and stores them in `cellpose_crops` folder. These crops consist of a nuclear and a Cytoplasmic channels as described above. By default crops are then segmented using the `tissuenet` pre-trained model and the resulting masks are placed in the folder `cellpose_labels`. Any model from the  model zoo can be specified using the`--model_type` argument. User trained models can be specified by providing the full path to the model using the `pretrained-model` option (when `pretrained-model` is provided, the `--model_type` argument is set to `None`, see above). The user can optionally specify a list of files for cropping and training. After running the first command, the user should open and inspect the generated masks and correct them as needed. This can be done by starting the `cellpose` gui via *steinbock*: `steinbock apps cellpose` (See `Apps`in this document. For details on how to use cellpose to correct gui segmentations, see its relevant [documentation](https://cellpose.readthedocs.io/en/latest/gui.html).
+
+The second command runs the cellpose training module. Unless specified via the `--train-data` and `--train-labels` options, the command looks in `cellpose_crops` and `cellpose_labels` for images and masks that are used as ground truth. The resulting model is by default saved in `training_out`. The trained model can subsequently be used for segmentation via the `--pretrained-model` argument, for example as following;
+
+
+
+	  steinbock segment cellpose run --pretrained-model ./training_out/models/myModel
+
+
 !!! note "GPU support"
     Currently. steinbock does not support cellpose segmentation with GPU support.
 

diff --git a/docs/cli/utils.md b/docs/cli/utils.md
@@ -40,3 +40,10 @@ The created image tiles will have the following file name, where `{IMG}` is the
 The following command will stitch all mask tiles in `masks` (following the file conventions above) to assemble masks of original size and save them to `masks_full`:
 
     steinbock utils mosaics stitch masks -o masks_full
+
+### Expansion
+
+This utility incorporates the `expand_labels` from [`skimage.segmentation`](https://scikit-image.org/docs/stable/api/skimage.segmentation.html#skimage.segmentation.expand_labels). It expands mask objects outwards by up to a distance specified in pixels without overflowing into neighboring regions.
+The following command expands the masks in each fils contained in `masks` directory by a maximum of 5 pixels and saves the resulting masks in `expanded_masks`:
+
+    steinbock utils expand masks 5 -o expanded_masks
diff --git a/docs/index.md b/docs/index.md
@@ -51,15 +51,17 @@ Workshop 2023: [https://github.com/BodenmillerGroup/ImagingWorkshop2023](https:/
 Please cite the following paper when using *steinbock* in your work:
 
 !!! quote
-    Windhager J, Bodenmiller B, Eling N (2021). An end-to-end workflow for multiplexed image processing and analysis. bioRxiv. doi: https://doi.org/10.1101/2021.11.12.468357.
+    Windhager, J., Zanotelli, V.R.T., Schulz, D. et al. An end-to-end workflow for multiplexed image processing and analysis. Nat Protoc (2023). https://doi.org/10.1038/s41596-023-00881-0
 
 ```
-@article{Windhager2021,
-  author = {Windhager, Jonas and Bodenmiller, Bernd and Eling, Nils},
+@article{Windhager2023,
+  author = {Windhager, Jonas and Zanotelli, Vito R.T. and Schulz, Daniel and Meyer, Lasse and Daniel, Michelle and Bodenmiller, Bernd and Eling, Nils},
   title = {An end-to-end workflow for multiplexed image processing and analysis},
-  year = {2021},
-  doi = {10.1101/2021.11.12.468357},
-  URL = {https://www.biorxiv.org/content/early/2021/11/13/2021.11.12.468357},
-  journal = {bioRxiv}
+  year = {2023},
+  doi = {10.1038/s41596-023-00881-0},
+  URL = {https://www.nature.com/articles/s41596-023-00881-0},
+  journal = {Nature Protocols}
 }
 ```
+
+If you have issues accessing the manuscript, please reach out to us and we can share the PDF version.