Skip to content

Commit f9f29c9

Browse files
zigaLuksiczigaLuksic
authored
Release 3.10.2, Merge pull request #529 from sentinel-hub/develop
Release 3.10.2
2 parents c7f5912 + 13a71ab commit f9f29c9

27 files changed

+172
-124
lines changed

‎.pre-commit-config.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: https://github.com/pre-commit/pre-commit-hooks
3-
rev: v4.5.0
3+
rev: v4.6.0
44
hooks:
55
- id: end-of-file-fixer
66
- id: requirements-txt-fixer
@@ -13,18 +13,18 @@ repos:
1313
- id: debug-statements
1414

1515
- repo: https://github.com/psf/black
16-
rev: 23.12.1
16+
rev: 24.4.0
1717
hooks:
1818
- id: black
1919
language_version: python3
2020

2121
- repo: https://github.com/charliermarsh/ruff-pre-commit
22-
rev: "v0.1.11"
22+
rev: "v0.4.1"
2323
hooks:
2424
- id: ruff
2525

2626
- repo: https://github.com/nbQA-dev/nbQA
27-
rev: 1.7.1
27+
rev: 1.8.5
2828
hooks:
2929
- id: nbqa-black
3030
- id: nbqa-ruff

‎CHANGELOG.MD

+5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
## [Version 3.10.2] - 2024-24-04
2+
3+
- Added `max_retries` parameter to `SHConfig` class. It controls how many times the client will attempt to re-download before raising `OutOfRequestsException`. It is set to `None` by default, in which case it never stops trying. Contributed by @Regan-Koopmans.
4+
5+
16
## [Version 3.10.1] - 2024-01-10
27

38
- Improved documentation for Copernicus Data Space Ecosystem.

‎examples/fis_request.ipynb

+9-7
Original file line numberDiff line numberDiff line change
@@ -432,13 +432,15 @@
432432
"\n",
433433
"geometry1 = Geometry(Polygon([(-5.13, 48), (-5.23, 48.09), (-5.13, 48.17), (-5.03, 48.08), (-5.13, 48)]), CRS.WGS84)\n",
434434
"geometry2 = Geometry(\n",
435-
" Polygon([\n",
436-
" (1292344.0, 5205055.5),\n",
437-
" (1301479.5, 5195920.0),\n",
438-
" (1310615.0, 5205055.5),\n",
439-
" (1301479.5, 5214191.0),\n",
440-
" (1292344.0, 5205055.5),\n",
441-
" ]),\n",
435+
" Polygon(\n",
436+
" [\n",
437+
" (1292344.0, 5205055.5),\n",
438+
" (1301479.5, 5195920.0),\n",
439+
" (1310615.0, 5205055.5),\n",
440+
" (1301479.5, 5214191.0),\n",
441+
" (1292344.0, 5205055.5),\n",
442+
" ]\n",
443+
" ),\n",
442444
" CRS.POP_WEB,\n",
443445
")"
444446
]

‎examples/process_request.ipynb

+9-7
Original file line numberDiff line numberDiff line change
@@ -989,13 +989,15 @@
989989
"request_raw_dict = {\n",
990990
" \"input\": {\n",
991991
" \"bounds\": {\"properties\": {\"crs\": betsiboka_bbox.crs.opengis_string}, \"bbox\": list(betsiboka_bbox)},\n",
992-
" \"data\": [{\n",
993-
" \"type\": \"S2L1C\",\n",
994-
" \"dataFilter\": {\n",
995-
" \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
996-
" \"mosaickingOrder\": \"leastCC\",\n",
997-
" },\n",
998-
" }],\n",
992+
" \"data\": [\n",
993+
" {\n",
994+
" \"type\": \"S2L1C\",\n",
995+
" \"dataFilter\": {\n",
996+
" \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
997+
" \"mosaickingOrder\": \"leastCC\",\n",
998+
" },\n",
999+
" }\n",
1000+
" ],\n",
9991001
" },\n",
10001002
" \"output\": {\n",
10011003
" \"width\": betsiboka_size[0],\n",

‎examples/process_request_cdse.ipynb

+9-7
Original file line numberDiff line numberDiff line change
@@ -941,13 +941,15 @@
941941
"request_raw_dict = {\n",
942942
" \"input\": {\n",
943943
" \"bounds\": {\"properties\": {\"crs\": betsiboka_bbox.crs.opengis_string}, \"bbox\": list(betsiboka_bbox)},\n",
944-
" \"data\": [{\n",
945-
" \"type\": \"S2L1C\",\n",
946-
" \"dataFilter\": {\n",
947-
" \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
948-
" \"mosaickingOrder\": \"leastCC\",\n",
949-
" },\n",
950-
" }],\n",
944+
" \"data\": [\n",
945+
" {\n",
946+
" \"type\": \"S2L1C\",\n",
947+
" \"dataFilter\": {\n",
948+
" \"timeRange\": {\"from\": \"2020-06-01T00:00:00Z\", \"to\": \"2020-06-30T00:00:00Z\"},\n",
949+
" \"mosaickingOrder\": \"leastCC\",\n",
950+
" },\n",
951+
" }\n",
952+
" ],\n",
951953
" },\n",
952954
" \"output\": {\n",
953955
" \"width\": betsiboka_size[0],\n",

‎pyproject.toml

+2-3
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ docs = [
6363
"matplotlib",
6464
"nbsphinx",
6565
"sphinx==7.1.2",
66-
"sphinx_mdinclude",
66+
"sphinx_mdinclude==0.5.4", # version fixed because 0.6.0 didnt work at release time of 3.10.2
6767
"sphinx_rtd_theme==1.3.0",
6868
]
6969
dev = [
@@ -73,14 +73,13 @@ dev = [
7373
"click>=8.0.0",
7474
"fs",
7575
"mypy>=0.990",
76-
"moto",
76+
"moto[s3]>=5.0.0",
7777
"pandas",
7878
"pre-commit",
7979
"pylint>=2.14.0",
8080
"pytest>=4.0.0",
8181
"pytest-cov",
8282
"pytest-dependency",
83-
"pytest-lazy-fixture",
8483
"pytest-mock",
8584
"ray[default]",
8685
"requests-mock",

‎sentinelhub/_version.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
"""Version of the sentinelhub package."""
22

3-
__version__ = "3.10.1"
3+
__version__ = "3.10.2"

‎sentinelhub/api/base_request.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -134,11 +134,13 @@ def _get_base_url(self) -> str:
134134
settings from config object. In case different collections have different restrictions then
135135
`SHConfig.sh_base_url` breaks the tie in case it matches one of the data collection URLs.
136136
"""
137-
data_collection_urls = tuple({
138-
input_data_dict.service_url.rstrip("/")
139-
for input_data_dict in self.payload["input"]["data"]
140-
if isinstance(input_data_dict, InputDataDict) and input_data_dict.service_url is not None
141-
})
137+
data_collection_urls = tuple(
138+
{
139+
input_data_dict.service_url.rstrip("/")
140+
for input_data_dict in self.payload["input"]["data"]
141+
if isinstance(input_data_dict, InputDataDict) and input_data_dict.service_url is not None
142+
}
143+
)
142144
config_base_url = self.config.sh_base_url.rstrip("/")
143145

144146
if not data_collection_urls:

‎sentinelhub/api/batch/process.py

+13-11
Original file line numberDiff line numberDiff line change
@@ -150,17 +150,19 @@ def output(
150150
:param kwargs: Any other arguments to be added to a dictionary of parameters
151151
:return: A dictionary of output parameters
152152
"""
153-
return remove_undefined({
154-
"defaultTilePath": default_tile_path,
155-
"overwrite": overwrite,
156-
"skipExisting": skip_existing,
157-
"cogOutput": cog_output,
158-
"cogParameters": cog_parameters,
159-
"createCollection": create_collection,
160-
"collectionId": collection_id,
161-
"responses": responses,
162-
**kwargs,
163-
})
153+
return remove_undefined(
154+
{
155+
"defaultTilePath": default_tile_path,
156+
"overwrite": overwrite,
157+
"skipExisting": skip_existing,
158+
"cogOutput": cog_output,
159+
"cogParameters": cog_parameters,
160+
"createCollection": create_collection,
161+
"collectionId": collection_id,
162+
"responses": responses,
163+
**kwargs,
164+
}
165+
)
164166

165167
def iter_tiling_grids(self, **kwargs: Any) -> SentinelHubFeatureIterator:
166168
"""An iterator over tiling grids

‎sentinelhub/api/byoc.py

+7-5
Original file line numberDiff line numberDiff line change
@@ -220,11 +220,13 @@ def update_tile(self, collection: CollectionType, tile: TileType) -> Json:
220220
headers = {"Content-Type": MimeType.JSON.get_string()}
221221

222222
_tile = self._to_dict(tile)
223-
updates = remove_undefined({
224-
"path": _tile["path"],
225-
"coverGeometry": _tile.get("coverGeometry"),
226-
"sensingTime": _tile.get("sensingTime"),
227-
})
223+
updates = remove_undefined(
224+
{
225+
"path": _tile["path"],
226+
"coverGeometry": _tile.get("coverGeometry"),
227+
"sensingTime": _tile.get("sensingTime"),
228+
}
229+
)
228230

229231
return self.client.get_json(
230232
url=url, request_type=RequestType.PUT, post_values=updates, headers=headers, use_session=True

‎sentinelhub/api/catalog.py

+16-14
Original file line numberDiff line numberDiff line change
@@ -140,20 +140,22 @@ def search(
140140
if geometry and geometry.crs is not CRS.WGS84:
141141
geometry = geometry.transform(CRS.WGS84)
142142

143-
payload = remove_undefined({
144-
"collections": [collection_id],
145-
"datetime": f"{start_time}/{end_time}" if time else None,
146-
"bbox": list(bbox) if bbox else None,
147-
"intersects": geometry.get_geojson(with_crs=False) if geometry else None,
148-
"ids": ids,
149-
"filter": self._prepare_filters(filter, collection, filter_lang),
150-
"filter-lang": filter_lang,
151-
"filter-crs": filter_crs,
152-
"fields": fields,
153-
"distinct": distinct,
154-
"limit": limit,
155-
**kwargs,
156-
})
143+
payload = remove_undefined(
144+
{
145+
"collections": [collection_id],
146+
"datetime": f"{start_time}/{end_time}" if time else None,
147+
"bbox": list(bbox) if bbox else None,
148+
"intersects": geometry.get_geojson(with_crs=False) if geometry else None,
149+
"ids": ids,
150+
"filter": self._prepare_filters(filter, collection, filter_lang),
151+
"filter-lang": filter_lang,
152+
"filter-crs": filter_crs,
153+
"fields": fields,
154+
"distinct": distinct,
155+
"limit": limit,
156+
**kwargs,
157+
}
158+
)
157159

158160
return CatalogSearchIterator(self.client, url, payload)
159161

‎sentinelhub/config.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424

2525

2626
@dataclass(repr=False)
27-
class _SHConfig:
27+
class _SHConfig: # pylint: disable=too-many-instance-attributes
2828
instance_id: str = ""
2929
sh_client_id: str = ""
3030
sh_client_secret: str = ""
@@ -46,6 +46,7 @@ class _SHConfig:
4646
download_sleep_time: float = 5.0
4747
download_timeout_seconds: float = 120.0
4848
number_of_download_processes: int = 1
49+
max_retries: int | None = None
4950

5051
def __post_init__(self) -> None:
5152
if self.sh_auth_base_url is not None:
@@ -94,6 +95,7 @@ class SHConfig(_SHConfig):
9495
attempt this number exponentially increases with factor `3`.
9596
- `download_timeout_seconds`: Maximum number of seconds before download attempt is canceled.
9697
- `number_of_download_processes`: Number of download processes, used to calculate rate-limit sleep time.
98+
- `max_retries`: Maximum number of retries until an exception is raised.
9799
98100
The location of `config.toml` for manual modification can be found with `SHConfig.get_config_location()`.
99101
"""

‎sentinelhub/data_collections_bands.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
""" Contains information about data collections used by SH """
1+
"""Contains information about data collections used by SH"""
22

33
from __future__ import annotations
44

‎sentinelhub/download/handlers.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def new_download_func(self: Self, request: DownloadRequest) -> T:
5252

5353

5454
def retry_temporary_errors(
55-
download_func: Callable[[SelfWithConfig, DownloadRequest], T]
55+
download_func: Callable[[SelfWithConfig, DownloadRequest], T],
5656
) -> Callable[[SelfWithConfig, DownloadRequest], T]:
5757
"""Decorator function for handling server and connection errors"""
5858
backoff_coefficient = 3

‎sentinelhub/download/sentinelhub_client.py

+6-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
from ..config import SHConfig
1717
from ..constants import SHConstants
18-
from ..exceptions import SHRateLimitWarning, SHRuntimeWarning
18+
from ..exceptions import OutOfRequestsException, SHRateLimitWarning, SHRuntimeWarning
1919
from ..types import JsonDict
2020
from .client import DownloadClient
2121
from .handlers import fail_user_errors, retry_temporary_errors
@@ -75,10 +75,12 @@ def _execute_download(self, request: DownloadRequest) -> DownloadResponse:
7575
"""
7676
Executes the download with a single thread and uses a rate limit object, which is shared between all threads
7777
"""
78+
download_attempts = 0
7879
while True:
7980
sleep_time = self._execute_thread_safe(self.rate_limit.register_next)
8081

8182
if sleep_time == 0:
83+
download_attempts += 1
8284
LOGGER.debug(
8385
"Sending %s request to %s. Hash of sent request is %s",
8486
request.request_type.value,
@@ -89,6 +91,9 @@ def _execute_download(self, request: DownloadRequest) -> DownloadResponse:
8991

9092
if response.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS:
9193
warnings.warn("Download rate limit hit", category=SHRateLimitWarning)
94+
if self.config.max_retries is not None and download_attempts >= self.config.max_retries:
95+
raise OutOfRequestsException("Maximum number of download attempts reached")
96+
9297
self._execute_thread_safe(self.rate_limit.update, response.headers, default=self.default_retry_time)
9398
continue
9499

‎sentinelhub/geometry.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ def _to_tuple(cls, bbox: BBoxInputType) -> tuple[float, float, float, float]:
150150

151151
@staticmethod
152152
def _tuple_from_list_or_tuple(
153-
bbox: tuple[float, float, float, float] | tuple[tuple[float, float], tuple[float, float]]
153+
bbox: tuple[float, float, float, float] | tuple[tuple[float, float], tuple[float, float]],
154154
) -> tuple[float, float, float, float]:
155155
"""Converts a list or tuple representation of a bbox into a flat tuple representation.
156156

‎tests/api/batch/test_process.py

+15-13
Original file line numberDiff line numberDiff line change
@@ -67,21 +67,23 @@ def test_create_and_run_batch_request(batch_client: SentinelHubBatch, requests_m
6767
request_id = "mocked-id"
6868
requests_mock.post(
6969
"/api/v1/batch/process",
70-
[{
71-
"json": {
72-
"id": request_id,
73-
"processRequest": {
74-
"input": {
75-
"bounds": {
76-
"bbox": list(bbox),
77-
"properties": {"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"},
70+
[
71+
{
72+
"json": {
73+
"id": request_id,
74+
"processRequest": {
75+
"input": {
76+
"bounds": {
77+
"bbox": list(bbox),
78+
"properties": {"crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84"},
79+
}
7880
}
79-
}
80-
},
81-
"tileCount": 42,
82-
"status": "CREATED",
81+
},
82+
"tileCount": 42,
83+
"status": "CREATED",
84+
}
8385
}
84-
}],
86+
],
8587
)
8688

8789
batch_request = batch_client.create(

0 commit comments

Comments
 (0)