Skip to content

Commit 3d8b387

Browse files
committed
fix: sync CAMS review fixes from upstream PR branch
Cherry-pick atomic download, tz-aware available(), and coordinate-based lead-time selection fixes from feat/cams-datasource (NVIDIA#780). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 2acb5da commit 3d8b387

3 files changed

Lines changed: 44 additions & 15 deletions

File tree

CLAUDE.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,15 @@
33
## Repository
44

55
Fork von [NVIDIA/earth2studio](https://github.com/NVIDIA/earth2studio). Upstream-Sync via `git fetch upstream && git merge upstream/main`.
6+
Upstream-Version: 0.14.0a0 (gemergt 2026-03-29). Upstream-PR: NVIDIA/earth2studio#780 (CAMS DataSource).
67

78
**Fork-Strategie:** Upstream-Files NICHT modifizieren. Eigene Additions in separaten Files halten. Nur `__init__.py` Imports und `pyproject.toml` Extras minimal patchen.
89

910
Eigene Erweiterungen:
1011
- CAMS DataSource/Lexicon (`earth2studio/data/cams.py`, `earth2studio/lexicon/cams.py`)
1112
- Serve Workflows (`serve/server/example_workflows/`)
1213
- Client SDK (`serve/client/earth2studio_client/`)
14+
- Remote Examples (`examples/remote/`)
1315
- Windows Build Scripts (`scripts/`)
1416

1517
## Project Rules
@@ -74,6 +76,9 @@ Config: `serve/server/conf/config.yaml`. Custom Workflows via `WORKFLOW_DIR` env
7476
- **Zarr IO erwartet Tensors**: `io.write()` braucht `torch.Tensor`, nicht numpy arrays.
7577
- **PyTorch SHMEM**: Container braucht `ipc: host` und `ulimits` (memlock, stack).
7678
- **fetch_data -> map_coords**: `fetch_data()` liefert GFS-Koordinaten (721 lat), aber FCN erwartet 720. Bei custom Workflows nach `fetch_data` immer `map_coords(x, coords, model.input_coords())` aufrufen.
79+
- **pyproject.toml workspace**: `earth2studio-client` braucht SOWOHL `[tool.uv.workspace] members` ALS AUCH `{ workspace = true }` in `[tool.uv.sources]`.
80+
- **Examples-Struktur**: Upstream nutzt Unterordner (`examples/01_getting_started/`, etc.). Eigene Examples NUR in `examples/remote/`.
81+
- **Upstream-PR Branch**: Immer von `upstream/main` aus erstellen, nur eigene Files cherry-picken.
7782

7883
## Client-SDK
7984

earth2studio/data/cams.py

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,10 @@
1616

1717
import asyncio
1818
import hashlib
19+
import os
1920
import pathlib
2021
import shutil
22+
import tempfile
2123
from dataclasses import dataclass
2224
from datetime import datetime, timedelta
2325
from time import sleep
@@ -104,12 +106,22 @@ def _download_cams_netcdf(
104106
)
105107
else:
106108
sleep(2.0)
107-
r.download(str(cache_path))
109+
tmp_fd, tmp_name = tempfile.mkstemp(dir=cache_path.parent, suffix=".nc.tmp")
110+
try:
111+
os.close(tmp_fd)
112+
r.download(tmp_name)
113+
os.replace(tmp_name, cache_path)
114+
except Exception:
115+
pathlib.Path(tmp_name).unlink(missing_ok=True)
116+
raise
108117
return cache_path
109118

110119

111120
def _extract_field(
112-
ds: xr.Dataset, nc_key: str, level: str = "0", time_index: int = 0
121+
ds: xr.Dataset,
122+
nc_key: str,
123+
level: str = "0",
124+
lead_time_hours: int | None = None,
113125
) -> np.ndarray:
114126
if nc_key not in ds:
115127
raise ValueError(
@@ -118,21 +130,22 @@ def _extract_field(
118130
)
119131
field = ds[nc_key]
120132
non_spatial = [d for d in field.dims if d not in ("latitude", "longitude")]
121-
sel: dict[str, int] = {}
133+
isel: dict[str, int] = {}
122134
for d in non_spatial:
123135
if d == "level" and level:
124136
level_val = float(level) if level else 0.0
125137
level_coords = field.coords["level"].values
126138
nearest_idx = int(np.argmin(np.abs(level_coords - level_val)))
127-
sel[d] = nearest_idx
128-
elif d in ("time", "forecast_period", "forecast_reference_time"):
129-
sel[d] = time_index
130-
elif field.sizes[d] > 1:
131-
sel[d] = 0
139+
isel[d] = nearest_idx
140+
elif d == "forecast_period" and lead_time_hours is not None:
141+
fp_vals = field.coords["forecast_period"].values.astype(float)
142+
target = float(lead_time_hours)
143+
nearest_idx = int(np.argmin(np.abs(fp_vals - target)))
144+
isel[d] = nearest_idx
132145
else:
133-
sel[d] = 0
134-
if sel:
135-
field = field.isel(sel)
146+
isel[d] = 0
147+
if isel:
148+
field = field.isel(isel)
136149
return field.values
137150

138151

@@ -288,7 +301,10 @@ def available(
288301
"""
289302
if isinstance(time, datetime):
290303
time = [time]
291-
return all(t >= _CAMS_EU_MIN_TIME for t in time)
304+
return all(
305+
(t.replace(tzinfo=None) if t.tzinfo else t) >= _CAMS_EU_MIN_TIME
306+
for t in time
307+
)
292308

293309
@staticmethod
294310
def _validate_time(times: list[datetime]) -> None:
@@ -509,7 +525,10 @@ def available(
509525
"""
510526
if isinstance(time, datetime):
511527
time = [time]
512-
return all(t >= _CAMS_GLOBAL_MIN_TIME for t in time)
528+
return all(
529+
(t.replace(tzinfo=None) if t.tzinfo else t) >= _CAMS_GLOBAL_MIN_TIME
530+
for t in time
531+
)
513532

514533
@staticmethod
515534
def _validate_time(times: list[datetime]) -> None:
@@ -568,12 +587,13 @@ def _fetch_forecast(
568587
},
569588
)
570589

571-
for lt_idx in range(len(lead_hours)):
590+
for lt_idx, lt_h in enumerate(lead_hours):
572591
for info in var_infos:
573592
_, modifier = CAMSLexicon[info.e2s_name]
574593
da[0, lt_idx, info.index] = modifier(
575594
_extract_field(
576-
ds, info.nc_key, level=info.level, time_index=lt_idx
595+
ds, info.nc_key, level=info.level,
596+
lead_time_hours=int(lt_h),
577597
)
578598
)
579599

test/data/test_cams.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ def test_cams_time_validation():
9696
def test_cams_available():
9797
assert CAMS.available(datetime.datetime(2024, 1, 1))
9898
assert not CAMS.available(datetime.datetime(2015, 1, 1))
99+
# timezone-aware datetimes must not raise TypeError
100+
assert CAMS.available(datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC))
99101

100102

101103
# ---- CAMS_FX tests ----
@@ -134,3 +136,5 @@ def test_cams_fx_fetch(variable, lead_time):
134136
def test_cams_fx_available():
135137
assert CAMS_FX.available(datetime.datetime(2024, 1, 1))
136138
assert not CAMS_FX.available(datetime.datetime(2010, 1, 1))
139+
# timezone-aware datetimes must not raise TypeError
140+
assert CAMS_FX.available(datetime.datetime(2024, 1, 1, tzinfo=datetime.UTC))

0 commit comments

Comments
 (0)