Release 3.10.3, Merge pull request #539 from sentinel-hub/develop

Release 3.10.3

Author: zigaLuksic

.github/workflows/ci_action.yml

@@ -107,13 +107,13 @@ jobs:
         run: |
           pytest -m "not sh_integration and not aws_integration"
 
-      - name: Upload code coverage
-        if: ${{ matrix.full_test_suite && github.event_name == 'push' }}
-        uses: codecov/codecov-action@v2
-        with:
-          files: coverage.xml
-          fail_ci_if_error: true
-          verbose: false
+      # - name: Upload code coverage
+      #   if: ${{ matrix.full_test_suite && github.event_name == 'push' }}
+      #   uses: codecov/codecov-action@v2
+      #   with:
+      #     files: coverage.xml
+      #     fail_ci_if_error: true
+      #     verbose: false
 
   mirror-and-integration-test-on-gitlab:
     if: github.event_name == 'push'
@@ -123,11 +123,11 @@ jobs:
       - name: Mirror + trigger CI
         uses: SvanBoxel/gitlab-mirror-and-ci-action@master
         with:
-          args: "https://git.sinergise.com/eo/code/sentinelhub-py-dev/"
+          args: "https://hello.planet.com/code/eo/code/sentinelhub-py-dev/"
         env:
           FOLLOW_TAGS: "true"
-          GITLAB_HOSTNAME: "git.sinergise.com"
+          GITLAB_HOSTNAME: "hello.planet.com/code"
           GITLAB_USERNAME: "github-action"
           GITLAB_PASSWORD: ${{ secrets.GITLAB_PASSWORD }}
-          GITLAB_PROJECT_ID: "354"
+          GITLAB_PROJECT_ID: "9711"
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_trigger.yml

@@ -16,4 +16,4 @@ jobs:
             -F ref=main \
             -F variables[CUSTOM_RUN_TAG]=auto \
             -F variables[LAYER_NAME]=dotai-eo \
-            https://git.sinergise.com/api/v4/projects/1031/trigger/pipeline
+            https://hello.planet.com/code/api/v4/projects/9723/trigger/pipeline

.gitlab-ci.yml

@@ -1,3 +1,10 @@
+default:
+  tags:
+    - sinergise-lju
+
+variables:
+  BASE_IMAGE: $CI_REGISTRY_IMAGE/python-3.9-base:latest
+
 stages:
   - update
   - test
@@ -6,20 +13,21 @@ update_base_image:
   stage: update
   image: docker:latest
   rules:
-    - if: '$UPDATE_BASE_IMAGE == "true"'
+    - if: $UPDATE_BASE_IMAGE == "true"
+
   script:
-    - docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY"
-    - docker build --file=".gitlab/docker/Dockerfile" --tag="$BASE_IMAGE" .
-    - docker push "$BASE_IMAGE"
+    - echo $CI_JOB_TOKEN | docker login $CI_REGISTRY -u $CI_REGISTRY_USER --password-stdin
+    - docker build -f docker/Dockerfile -t $BASE_IMAGE .
+    - docker push $BASE_IMAGE
 
 run_sh_integration_tests:
   stage: test
-  image: "$BASE_IMAGE"
+  image: $BASE_IMAGE
   dependencies: []
   needs: []
   rules:
     - when: always
-      if: '$UPDATE_BASE_IMAGE != "true"'
+      if: $UPDATE_BASE_IMAGE != "true"
   script:
     - pip install -e .[AWS,DEV]
     - pytest -m "sh_integration"

.pre-commit-config.yaml

@@ -13,18 +13,18 @@ repos:
       - id: debug-statements
 
   - repo: https://github.com/psf/black
-    rev: 24.4.0
+    rev: 24.8.0
     hooks:
       - id: black
         language_version: python3
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.4.1"
+    rev: "v0.5.7"
     hooks:
       - id: ruff
 
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.5
+    rev: 1.8.7
     hooks:
       - id: nbqa-black
       - id: nbqa-ruff

CHANGELOG.MD

@@ -1,3 +1,9 @@
+## [Version 3.10.3] - 2024-13-08
+
+- Monitoring a batch job now concludes if the batch job is cancelled.
+- The monitoring function now waits for the status of the batch job to change (even if all tiles have completed) before finalization in order to return the correct status.
+
+
 ## [Version 3.10.2] - 2024-24-04
 
 - Added `max_retries` parameter to `SHConfig` class. It controls how many times the client will attempt to re-download before raising `OutOfRequestsException`. It is set to `None` by default, in which case it never stops trying. Contributed by @Regan-Koopmans.

.gitlab/docker/Dockerfile docker/Dockerfile

@@ -110,7 +110,7 @@ preview = true
 [tool.ruff]
 line-length = 120
 target-version = "py38"
-select = [
+lint.select = [
     "F",    # pyflakes
     "E",    # pycodestyle
     "W",    # pycodestyle
@@ -143,15 +143,15 @@ select = [
     "RUF",  # ruff rules
 ]
 fix = true
-fixable = [
+lint.fixable = [
     "I",     # sort imports
     "F401",  # remove redundant imports
     "UP007", # use new-style union type annotations
     "UP006", # use new-style built-in type annotations
     "UP037", # remove quotes around types when not necessary
     "FA100", # import future annotations where necessary (not autofixable ATM)
 ]
-ignore = [
+lint.ignore = [
     "C408",   # complains about `dict()` calls, we use them to avoid too many " in the code
     "SIM108", # tries to aggresively inline `if`, not always readable
     "COM812", # trailing comma missing, fights with black
@@ -162,11 +162,11 @@ ignore = [
     "B028",  # always demands a stacklevel argument when warning
     "PT011", # complains for `pytest.raises(ValueError)` but we use it a lot
 ]
-per-file-ignores = { "__init__.py" = ["F401"], "conf.py" = ["FA100"] }
+lint.per-file-ignores = { "__init__.py" = ["F401"], "conf.py" = ["FA100"] }
 exclude = [".git", "__pycache__", "build", "dist", "sentinelhub/aws/*"]
 
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 section-order = [
     "future",
     "standard-library",

pyproject.toml

@@ -1,3 +1,3 @@
 """Version of the sentinelhub package."""
 
-__version__ = "3.10.2"
+__version__ = "3.10.3"

sentinelhub/_version.py

@@ -73,9 +73,12 @@ def monitor_batch_job(
 
     progress_bar = tqdm(total=batch_request.tile_count, initial=finished_count, desc="Progress rate")
     success_bar = tqdm(total=finished_count, initial=success_count, desc="Success rate")
+
+    monitoring_status = [BatchRequestStatus.ANALYSIS_DONE, BatchRequestStatus.PROCESSING]
     with progress_bar, success_bar:
-        while finished_count < batch_request.tile_count:
+        while finished_count < batch_request.tile_count and batch_request.status in monitoring_status:
             time.sleep(sleep_time)
+            batch_request = batch_client.get_request(batch_request)
 
             tiles_per_status = _get_batch_tiles_per_status(batch_request, batch_client)
             new_success_count = len(tiles_per_status[BatchTileStatus.PROCESSED])
@@ -93,6 +96,13 @@ def monitor_batch_job(
     failed_tiles_num = finished_count - success_count
     if failed_tiles_num:
         LOGGER.info("Batch job failed for %d tiles", failed_tiles_num)
+
+    while batch_request.status is BatchRequestStatus.PROCESSING:
+        LOGGER.info("Waiting on batch job status update.")
+        time.sleep(sleep_time)
+        batch_request = batch_client.get_request(batch_request)
+
+    LOGGER.info("Batch job finished with status %s", batch_request.status.value)
     return tiles_per_status
 
 

sentinelhub/api/batch/utils.py

@@ -81,7 +81,7 @@ def __new__(mcs, cls, bases, classdict):  # type: ignore[no-untyped-def] # noqa:
 
         return super().__new__(mcs, cls, bases, classdict)
 
-    def __call__(cls, crs_value, *args, **kwargs):  # type: ignore[no-untyped-def] # noqa: N805
+    def __call__(cls, crs_value, *args, **kwargs):  # type: ignore[no-untyped-def]
         """This is executed whenever CRS('something') is called"""
         # pylint: disable=signature-differs
         crs_value = cls._parse_crs(crs_value)
@@ -201,7 +201,7 @@ def is_utm(self) -> bool:
         """
         return self.name.startswith("UTM")
 
-    @functools.lru_cache(maxsize=128)  # noqa: B019
+    @functools.lru_cache(maxsize=128)
     def projection(self) -> pyproj.Proj:
         """Returns a projection in form of pyproj class.
 
@@ -212,7 +212,7 @@ def projection(self) -> pyproj.Proj:
         """
         return pyproj.Proj(self._get_pyproj_projection_def(), preserve_units=True)
 
-    @functools.lru_cache(maxsize=128)  # noqa: B019
+    @functools.lru_cache(maxsize=128)
     def pyproj_crs(self) -> pyproj.CRS:
         """Returns a pyproj CRS class.
 
@@ -223,7 +223,7 @@ def pyproj_crs(self) -> pyproj.CRS:
         """
         return pyproj.CRS(self._get_pyproj_projection_def())
 
-    @functools.lru_cache(maxsize=512)  # noqa: B019
+    @functools.lru_cache(maxsize=512)
     def get_transform_function(self, other: CRS, always_xy: bool = True) -> Callable[..., tuple]:
         """Returns a function for transforming geometrical objects from one CRS to another. The function will support
         transformations between any objects that pyproj supports.

sentinelhub/constants.py

@@ -67,12 +67,18 @@ def test_monitor_batch_process_job(
         len(tiles) == tile_count for tiles in tiles_sequence
     ), "There should be the same number of tiles in each step. Fix tile_status_sequence parameter of this test."
 
-    batch_request = BatchRequest(
-        request_id="mocked-request", process_request={}, tile_count=tile_count, status=batch_status
-    )
+    batch_kwargs = dict(request_id="mocked-request", process_request={}, tile_count=tile_count)
+    batch_request = BatchRequest(**batch_kwargs, status=batch_status)
+    updated_batch_request = BatchRequest(**batch_kwargs, status=BatchRequestStatus.DONE)
+
     monitor_analysis_mock = mocker.patch("sentinelhub.api.batch.utils.monitor_batch_analysis")
     monitor_analysis_mock.return_value = batch_request
 
+    # keep returning the same batch request until the last step (simulate batch job status update)
+    progress_loop_counts = len(tile_status_sequence) - 1
+    batch_request_update_mock = mocker.patch("sentinelhub.SentinelHubBatch.get_request")
+    batch_request_update_mock.side_effect = progress_loop_counts * [batch_request] + [updated_batch_request]
+
     batch_tiles_mock = mocker.patch("sentinelhub.SentinelHubBatch.iter_tiles")
     batch_tiles_mock.side_effect = tiles_sequence
 
@@ -88,18 +94,16 @@ def test_monitor_batch_process_job(
         assert len(results[tile_status]) == tile_status_sequence[-1].get(tile_status, 0)
 
     assert monitor_analysis_mock.call_count == 1
-
-    progress_loop_counts = len(tile_status_sequence) - 1
-
     assert batch_tiles_mock.call_count == progress_loop_counts + 1
     assert all(call.args == (batch_request,) and call.kwargs == {} for call in batch_tiles_mock.mock_calls)
 
-    assert sleep_mock.call_count == progress_loop_counts
+    additional_calls = 1 if batch_status is BatchRequestStatus.PROCESSING else 0
+    assert sleep_mock.call_count == progress_loop_counts + additional_calls
     assert all(call.args == (sleep_time,) and call.kwargs == {} for call in sleep_mock.mock_calls)
 
     is_processing_logged = batch_status is BatchRequestStatus.PROCESSING
     is_failure_logged = BatchTileStatus.FAILED in tile_status_sequence[-1]
-    assert logging_mock.call_count == int(is_processing_logged) + int(is_failure_logged)
+    assert logging_mock.call_count == int(is_processing_logged) + int(is_failure_logged) + additional_calls + 1
 
 
 def _tile_status_counts_to_tiles(tile_status_counts: dict[BatchTileStatus, int]) -> list[dict[str, str]]:

tests/api/batch/test_utils.py

@@ -57,14 +57,14 @@ def test_session_caching_and_clearing(
     client_object: SentinelHubDownloadClient | type[SentinelHubDownloadClient], session: SentinelHubSession
 ) -> None:
     client_object.clear_cache()
-    assert {} == SentinelHubDownloadClient._CACHED_SESSIONS
+    assert SentinelHubDownloadClient._CACHED_SESSIONS == {}
 
     client_object.cache_session(session)
     assert len(SentinelHubDownloadClient._CACHED_SESSIONS) == 1
     assert list(SentinelHubDownloadClient._CACHED_SESSIONS.values()) == [session]
 
     client_object.clear_cache()
-    assert {} == SentinelHubDownloadClient._CACHED_SESSIONS
+    assert SentinelHubDownloadClient._CACHED_SESSIONS == {}
 
 
 @pytest.mark.sh_integration()