Release 3.3.5, Merge pull request #334 from sentinel-hub/develop

Release 3.3.5

Author: zigaLuksic

.pre-commit-config.yaml

@@ -16,22 +16,22 @@ repos:
     rev: "v4.0.0-alpha.8"
     hooks:
       - id: prettier
-        exclude: "tests/(test_stats|test_project)/"
+        exclude: "tests/test_project/"
         types_or: [json]
 
   - repo: https://github.com/psf/black
-    rev: 24.1.1
+    rev: 24.2.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.1.14"
+    rev: "v0.3.2"
     hooks:
       - id: ruff
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.7.1
+    rev: 1.8.4
     hooks:
       - id: nbqa-black
       - id: nbqa-ruff

CHANGELOG.md

@@ -1,3 +1,8 @@
+## [Version 1.7.7] - 2024-03-11
+
+- When a Pipeline raises an error it now saves the stack-trace to the `failure.log` file in the logs folder.
+
+
 ## [Version 1.7.6] - 2024-01-29
 
 - Pipelines that are run as part of a pipeline-chain execution will now no longer be retried by ray in the case when an exception occurs.

eogrow/__init__.py

@@ -1,3 +1,3 @@
 """The main module of the eo-grow package."""
 
-__version__ = "1.7.6"
+__version__ = "1.7.7"

eogrow/core/pipeline.py

@@ -4,9 +4,11 @@
 
 import logging
 import time
+import traceback
 import uuid
 from typing import Any, TypeVar
 
+import fs
 import ray
 
 from eolearn.core import CreateEOPatchTask, EOExecutor, EONode, EOWorkflow, LoadTask, SaveTask, WorkflowResults
@@ -255,6 +257,14 @@ def run(self) -> None:
 
             if failed and self.config.raise_on_failure:
                 raise PipelineExecutionError(f"Pipeline failed some executions. Check {log_folder}.")
+        except Exception as e:
+            # store any exception info you can
+            failure_log_path = fs.path.join(
+                self.logging_manager.get_pipeline_logs_folder(self.current_execution_name), "failure.log"
+            )
+            with self.storage.filesystem.open(failure_log_path, mode="w") as log_file:
+                log_file.writelines(traceback.format_exc())
+            raise e
         finally:
             self.logging_manager.stop_logging(root_logger, handlers)
 

eogrow/tasks/batch_to_eopatch.py

@@ -54,7 +54,7 @@ def _parse_timestamps(userdata: dict, userdata_timestamp_reader: str) -> list:
             reader = import_object(userdata_timestamp_reader)
             time_strings = reader(userdata)
         except (ImportError, ValueError):
-            time_strings = eval(userdata_timestamp_reader)  # pylint: disable=eval-used  # noqa: PGH001
+            time_strings = eval(userdata_timestamp_reader)  # pylint: disable=eval-used
 
         return [parse_time(time_string, force_datetime=True, ignoretz=True) for time_string in time_strings]
 

pyproject.toml

@@ -116,7 +116,7 @@ preview = true
 [tool.ruff]
 line-length = 120
 target-version = "py38"
-select = [
+lint.select = [
     "F",    # pyflakes
     "E",    # pycodestyle
     "W",    # pycodestyle
@@ -147,11 +147,11 @@ select = [
     "RUF",  # ruff rules
 ]
 fix = true
-fixable = [
+lint.fixable = [
     "I",    # sort imports
     "F401", # remove redundant imports
 ]
-ignore = [
+lint.ignore = [
     "C408",   # complains about `dict()` calls, we use them to avoid too many " in the code
     "SIM117", # wants to always combine `with` statements, gets ugly for us
     "SIM108", # tries to aggresively inline `if`, not always readable
@@ -164,11 +164,11 @@ ignore = [
     "B028",  # always demands a stacklevel argument when warning
     "PT011", # complains for `pytest.raises(ValueError)` but we use it a lot
 ]
-per-file-ignores = { "__init__.py" = ["F401"] }
+lint.per-file-ignores = { "__init__.py" = ["F401"] }
 exclude = [".git", "__pycache__", "build", "dist"]
 
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 section-order = [
     "future",
     "standard-library",

tests/core/test_pipeline.py

@@ -1,5 +1,7 @@
 import logging
 import os
+from contextlib import suppress
+from pathlib import Path
 from typing import List, Tuple, Union
 
 import pytest
@@ -98,6 +100,23 @@ def test_get_patch_list_filtration_error(test_subset: List[Union[int, str]], sim
         pipeline.get_patch_list()
 
 
+def test_pipeline_logs_exception(simple_config_filename: str) -> None:
+    def cookie_alarm():
+        raise RuntimeError("Oh no, someone stole my cookie! :(")
+
+    config = interpret_config_from_path(simple_config_filename)
+    pipeline = SimplePipeline.from_raw_config(config)
+    pipeline.run_procedure = cookie_alarm
+
+    with suppress(RuntimeError):
+        pipeline.run()
+
+    log_folder = pipeline.logging_manager.get_pipeline_logs_folder(pipeline.current_execution_name, full_path=True)
+    log_path = Path(log_folder) / "failure.log"
+    assert log_path.exists()
+    assert "Oh no, someone stole my cookie! :(" in log_path.read_text()
+
+
 @pytest.mark.parametrize("fail", [True, False])
 @pytest.mark.parametrize("raise_on_failure", [True, False])
 def test_pipeline_raises_on_failure(fail: bool, raise_on_failure: bool, simple_config_filename: str):

tests/test_stats/download_and_batch/batch_to_eopatch.json

@@ -3,12 +3,7 @@
     "bbox": "BBox(((729480.0, 4390045.0), (732120.0, 4391255.0)), crs=CRS('32638'))",
     "data": {
       "bands": {
-        "array_shape": [
-          7,
-          300,
-          200,
-          3
-        ],
+        "array_shape": [7, 300, 200, 3],
         "basic_stats": {
           "max": 1234.38,
           "mean": 617.189,
@@ -23,36 +18,15 @@
         "dtype": "float32",
         "histogram": {
           "counts": [
-            157086,
-            157534,
-            157504,
-            158241,
-            157351,
-            157494,
-            157391,
-            157399
+            157086, 157534, 157504, 158241, 157351, 157494, 157391, 157399
           ],
           "edges": [
-            0.0,
-            154.297,
-            308.594,
-            462.891,
-            617.188,
-            771.485,
-            925.782,
-            1080.08,
+            0.0, 154.297, 308.594, 462.891, 617.188, 771.485, 925.782, 1080.08,
             1234.38
           ]
         },
         "random_values": [
-          88.3902,
-          949.824,
-          1209.32,
-          265.664,
-          939.578,
-          357.388,
-          668.235,
-          609.102
+          88.3902, 949.824, 1209.32, 265.664, 939.578, 357.388, 668.235, 609.102
         ],
         "subsample_basic_stats": {
           "max": 1234.38,
@@ -63,12 +37,7 @@
         }
       },
       "bands_repeated": {
-        "array_shape": [
-          7,
-          300,
-          200,
-          3
-        ],
+        "array_shape": [7, 300, 200, 3],
         "basic_stats": {
           "max": 1234.38,
           "mean": 617.189,
@@ -83,36 +52,15 @@
         "dtype": "float32",
         "histogram": {
           "counts": [
-            157086,
-            157534,
-            157504,
-            158241,
-            157351,
-            157494,
-            157391,
-            157399
+            157086, 157534, 157504, 158241, 157351, 157494, 157391, 157399
           ],
           "edges": [
-            0.0,
-            154.297,
-            308.594,
-            462.891,
-            617.188,
-            771.485,
-            925.782,
-            1080.08,
+            0.0, 154.297, 308.594, 462.891, 617.188, 771.485, 925.782, 1080.08,
             1234.38
           ]
         },
         "random_values": [
-          88.3902,
-          949.824,
-          1209.32,
-          265.664,
-          939.578,
-          357.388,
-          668.235,
-          609.102
+          88.3902, 949.824, 1209.32, 265.664, 939.578, 357.388, 668.235, 609.102
         ],
         "subsample_basic_stats": {
           "max": 1234.38,
@@ -140,12 +88,7 @@
     "bbox": "BBox(((729480.0, 4391145.0), (732120.0, 4392355.0)), crs=CRS('32638'))",
     "data": {
       "bands": {
-        "array_shape": [
-          7,
-          300,
-          200,
-          3
-        ],
+        "array_shape": [7, 300, 200, 3],
         "basic_stats": {
           "max": 1234.38,
           "mean": 617.189,
@@ -160,36 +103,15 @@
         "dtype": "float32",
         "histogram": {
           "counts": [
-            157086,
-            157534,
-            157504,
-            158241,
-            157351,
-            157494,
-            157391,
-            157399
+            157086, 157534, 157504, 158241, 157351, 157494, 157391, 157399
           ],
           "edges": [
-            0.0,
-            154.297,
-            308.594,
-            462.891,
-            617.188,
-            771.485,
-            925.782,
-            1080.08,
+            0.0, 154.297, 308.594, 462.891, 617.188, 771.485, 925.782, 1080.08,
             1234.38
           ]
         },
         "random_values": [
-          88.3902,
-          949.824,
-          1209.32,
-          265.664,
-          939.578,
-          357.388,
-          668.235,
-          609.102
+          88.3902, 949.824, 1209.32, 265.664, 939.578, 357.388, 668.235, 609.102
         ],
         "subsample_basic_stats": {
           "max": 1234.38,
@@ -200,12 +122,7 @@
         }
       },
       "bands_repeated": {
-        "array_shape": [
-          7,
-          300,
-          200,
-          3
-        ],
+        "array_shape": [7, 300, 200, 3],
         "basic_stats": {
           "max": 1234.38,
           "mean": 617.189,
@@ -220,36 +137,15 @@
         "dtype": "float32",
         "histogram": {
           "counts": [
-            157086,
-            157534,
-            157504,
-            158241,
-            157351,
-            157494,
-            157391,
-            157399
+            157086, 157534, 157504, 158241, 157351, 157494, 157391, 157399
           ],
           "edges": [
-            0.0,
-            154.297,
-            308.594,
-            462.891,
-            617.188,
-            771.485,
-            925.782,
-            1080.08,
+            0.0, 154.297, 308.594, 462.891, 617.188, 771.485, 925.782, 1080.08,
             1234.38
           ]
         },
         "random_values": [
-          88.3902,
-          949.824,
-          1209.32,
-          265.664,
-          939.578,
-          357.388,
-          668.235,
-          609.102
+          88.3902, 949.824, 1209.32, 265.664, 939.578, 357.388, 668.235, 609.102
         ],
         "subsample_basic_stats": {
           "max": 1234.38,