diff --git a/python/pymetkit/src/pymetkit/metkit_c.h b/python/pymetkit/src/pymetkit/metkit_c.h index 42d88946b..41364ee9e 100644 --- a/python/pymetkit/src/pymetkit/metkit_c.h +++ b/python/pymetkit/src/pymetkit/metkit_c.h @@ -27,6 +27,8 @@ const char* metkit_git_sha1(); metkit_error_t metkit_initialise(); metkit_error_t metkit_parse_marsrequests(const char* str, metkit_requestiterator_t** requests, bool strict); +metkit_error_t metkit_expand_key(const char* verb, const char* keyword, const char* value, + const metkit_marsrequest_t* context, bool strict, metkit_paramiterator_t** values); metkit_error_t metkit_marsrequest_new(metkit_marsrequest_t** request); metkit_error_t metkit_marsrequest_delete(const metkit_marsrequest_t* request); metkit_error_t metkit_marsrequest_set(metkit_marsrequest_t* request, const char* param, const char* values[], diff --git a/python/pymetkit/src/pymetkit/pymetkit.py b/python/pymetkit/src/pymetkit/pymetkit.py index 7c0594fa8..3381a99dc 100644 --- a/python/pymetkit/src/pymetkit/pymetkit.py +++ b/python/pymetkit/src/pymetkit/pymetkit.py @@ -219,6 +219,121 @@ def parse_mars_request(file_or_str: IO | str, strict: bool = False) -> list[Mars return requests +def _expand_param( + value: str | int | list, + verb: str, + context: "MarsRequest | dict | None", + strict: bool, +) -> list[str]: + """Resolve the ``param`` key via a full (multi-pass) request expansion. + + ``param`` cannot be expanded in isolation because its result depends on other + keys. A scoped :class:`MarsRequest` is built from ``context`` (which supplies + the neighbouring keys such as ``class``/``stream``/``type``/``levtype``), the + param value is set, the request is expanded, and the resolved ``param`` values + are returned. + """ + if isinstance(value, (list, tuple)): + values = [str(v) for v in value] + else: + values = str(value).split("/") + + request = MarsRequest(verb) + if context is not None: + if isinstance(context, MarsRequest): + for key, vals in context: + request[key] = vals + else: # dict-like + for key, vals in context.items(): + request[key.rstrip("_")] = vals + request["param"] = values + + expanded = request.expand(inherit=False, strict=strict) + resolved = expanded["param"] + return [resolved] if isinstance(resolved, str) else resolved + + +def expand_key( + keyword: str, + value: str | int | list, + verb: str = "retrieve", + context: "dict | MarsRequest | None" = None, + strict: bool = False, +) -> list[str]: + """Expand and normalise the values of a single MARS key, without building a full request. + + Applies the MARS language rules for ``keyword``: range syntax such as + ``"1/to/10/by/1"`` is expanded, and per-key normalisation is performed (e.g. + ``date="-1"`` resolves to a ``yyyymmdd`` date, ``time="6"`` to ``"0600"``). + + Context-sensitive keys (e.g. those whose interpretation depends on other + keys) consult ``context``. Supply the relevant neighbouring keys there, e.g. + ``expand_key("levelist", "1/to/10", context={"levtype": "ml"})``. + + Note: this performs single-pass expansion for most keys. ``param`` is a + special case: its value depends on other keys (``class``/``stream``/``type``/ + ``levtype``) and requires a full multi-pass expansion, so ``expand_key`` + transparently builds a scoped request from ``context``, expands it, and + returns the resolved ``param`` values, e.g. + ``expand_key("param", "t", context={"levtype": "sfc"})``. + + Params + ------ + keyword: name of the MARS key to expand (canonical, alias or unambiguous prefix) + value: values to expand, as a ``"a/b/c"`` string or a list of tokens + verb: MARS verb whose language defines the key (defaults to "retrieve") + context: optional MarsRequest or dict of neighbouring keys for context-sensitive keys + strict: if True, raise an error on invalid values + + Returns + ------- + list of expanded/normalised string values + + Examples + -------- + >>> expand_key("step", "1/to/10/by/1") + ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10'] + >>> expand_key("param", "t", context={"levtype": "pl"}) + ['130'] + """ + if keyword == "param": + # 'param' cannot be resolved by single-pass expansion: its value depends + # on other keys, so it needs the full multi-pass request expansion. We + # hide that from the caller by building a scoped request from the given + # context, expanding it, and returning just the resolved param values. + return _expand_param(value, verb, context, strict) + + if isinstance(value, (list, tuple)): + value = "/".join(str(v) for v in value) + else: + value = str(value) + + context_c = ffi.NULL + if context is not None: + if isinstance(context, dict): + context = MarsRequest(**context) # @maby: If we're going to build a mars request anyway, why not just pass it all in? + context_c = context.ctype() + + it_c = ffi.new("metkit_paramiterator_t **") + lib.metkit_expand_key( + ffi_encode(verb), + ffi_encode(keyword), + ffi_encode(value), + context_c, + strict, + it_c, + ) + it = ffi.gc(it_c[0], lib.metkit_paramiterator_delete) + + values = [] + while lib.metkit_paramiterator_next(it) == lib.METKIT_ITERATOR_SUCCESS: + cvalue = ffi.new("const char **") + lib.metkit_paramiterator_current(it, cvalue) + values.append(ffi_decode(cvalue[0])) + + return values + + class MetKitException(RuntimeError): """Raised when MetKit library throws exception""" @@ -281,26 +396,37 @@ def __check_error(self, fn, name: str): """ If calls into the MetKit library return errors, ensure that they get detected and reported by throwing an appropriate python exception. + + Two distinct return-code domains exist in the C API and must not share a + single success-set, otherwise their overlapping integer values collide + (e.g. metkit_error_t METKIT_ERROR == metkit_iterator_status_t + METKIT_ITERATOR_COMPLETE == 1): + + * iterator functions (``*_next`` / ``*_current``) return + metkit_iterator_status_t, where SUCCESS and COMPLETE are both non-error + outcomes and only ITERATOR_ERROR indicates failure; + * every other function returns metkit_error_t, where only METKIT_SUCCESS + indicates success. """ - def wrapped_fn(*args, **kwargs): + # Functions that do not return an error/status code at all. + if name in ("metkit_version", "metkit_git_sha1"): + return fn - # debug + is_iterator = name.endswith("_next") or name.endswith("_current") + + def wrapped_fn(*args, **kwargs): retval = fn(*args, **kwargs) - # Some functions dont return error codes. Ignore these. - if name in ["metkit_version", "metkit_git_sha1"]: + if is_iterator: + if retval == self.__lib.METKIT_ITERATOR_ERROR: + err = ffi_decode(self.__lib.metkit_get_error_string(retval)) + raise MetKitException("Error in function '{}': {}".format(name, err)) return retval - # error codes: - if retval not in ( - self.__lib.METKIT_SUCCESS, - self.__lib.METKIT_ITERATOR_SUCCESS, - self.__lib.METKIT_ITERATOR_COMPLETE, - ): + if retval != self.__lib.METKIT_SUCCESS: err = ffi_decode(self.__lib.metkit_get_error_string(retval)) - msg = "Error in function '{}': {}".format(name, err) - raise MetKitException(msg) + raise MetKitException("Error in function '{}': {}".format(name, err)) return retval return wrapped_fn diff --git a/python/pymetkit/tests/test_expand.py b/python/pymetkit/tests/test_expand.py new file mode 100644 index 000000000..40bdb58a3 --- /dev/null +++ b/python/pymetkit/tests/test_expand.py @@ -0,0 +1,262 @@ +"""Tests for :func:`pymetkit.expand_key` covering the more esoteric corners of +the MARS language: per-keyword default ``by`` steps, descending ranges, range +synonyms, date arithmetic (relative/leap/day-of-year), value normalisation and +context-sensitive keywords. + +These complement the basic ``expand_key`` cases in ``test_marsrequest.py``. +""" + +from datetime import datetime, timedelta + +import pytest + +from pymetkit import MarsRequest, MetKitException, expand_key + + +def _relative_date(offset_days: int) -> str: + return (datetime.today() + timedelta(days=offset_days)).strftime("%Y%m%d") + + +yesterday = _relative_date(-1) +today = _relative_date(0) + + +# --------------------------------------------------------------------------- +# Per-keyword default "by" step +# --------------------------------------------------------------------------- +# The default increment of a "to" range is defined per keyword by the language, +# not globally. "step" defaults to by=12, while plain numeric keywords default +# to by=1. A range whose span is smaller than the default step collapses to just +# the start value. +@pytest.mark.parametrize( + "keyword, value, expected", + [ + ("step", "0/to/24", ["0", "12", "24"]), # step default by == 12 + ("step", "0/to/6", ["0"]), # span < default step -> only start + ("step", "0/to/24/by/6", ["0", "6", "12", "18", "24"]), # explicit by + ("number", "1/to/5", ["1", "2", "3", "4", "5"]), # numeric default by == 1 + ], +) +def test_default_by_is_per_keyword(keyword, value, expected): + assert expand_key(keyword, value) == expected + + +# --------------------------------------------------------------------------- +# Descending ranges and invalid steps +# --------------------------------------------------------------------------- +# A range where the end is below the start is descending. The direction can be +# implied (positive by) or given explicitly with a negative by. +@pytest.mark.parametrize( + "value, expected", + [ + ("10/to/1/by/1", ["10", "9", "8", "7", "6", "5", "4", "3", "2", "1"]), + ("10/to/1/by/-1", ["10", "9", "8", "7", "6", "5", "4", "3", "2", "1"]), + ("5/to/5/by/1", ["5"]), # degenerate range + ], +) +def test_descending_ranges(value, expected): + assert expand_key("number", value) == expected + + +def test_zero_step_raises(): + with pytest.raises(MetKitException): + expand_key("number", "1/to/10/by/0") + + +# --------------------------------------------------------------------------- +# Range keyword synonyms: "to"/"t0" and case-insensitivity +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("sep", ["to", "TO", "To", "t0", "T0"]) +def test_to_synonyms_are_case_insensitive(sep): + assert expand_key("number", f"1/{sep}/5") == ["1", "2", "3", "4", "5"] + + +def test_by_keyword_is_case_insensitive(): + assert expand_key("number", "0/to/9/BY/3") == ["0", "3", "6", "9"] + + +# --------------------------------------------------------------------------- +# Time normalisation +# --------------------------------------------------------------------------- +# Times normalise to HHMM. Bare hour values are widened, ':' separators are +# stripped, and ranges may step in minutes. +@pytest.mark.parametrize( + "value, expected", + [ + ("6", ["0600"]), + ("06:30", ["0630"]), + ("0/to/12/by/6", ["0000", "0600", "1200"]), + ("0000/to/0100/by/0030", ["0000", "0030", "0100"]), + ], +) +def test_time_normalisation(value, expected): + assert expand_key("time", value) == expected + + +@pytest.mark.parametrize("value", ["notatime", "2500"]) +def test_invalid_time_raises(value): + with pytest.raises(MetKitException): + expand_key("time", value) + + +# --------------------------------------------------------------------------- +# Date arithmetic +# --------------------------------------------------------------------------- +def test_relative_dates(): + assert expand_key("date", "-1") == [yesterday] + assert expand_key("date", "0") == [today] + + +def test_date_year_day_of_year(): + # yyyy-ddd expands to the yyyymmdd of the ddd-th day of the year. + assert expand_key("date", "2018-23") == ["20180123"] + + +def test_date_range_crosses_leap_day(): + # 2020 is a leap year, so 29 Feb is included. + assert expand_key("date", "20200228/to/20200302") == [ + "20200228", + "20200229", + "20200301", + "20200302", + ] + + +def test_date_range_with_step(): + assert expand_key("date", "20200101/to/20200110/by/2") == [ + "20200101", + "20200103", + "20200105", + "20200107", + "20200109", + ] + + +@pytest.mark.parametrize("value", ["jan", "feb", "dec"]) +def test_month_name_climatology_passthrough(value): + # Bare month names are climatology values and are kept as-is (lower-cased). + assert expand_key("date", value) == [value] + + +# --------------------------------------------------------------------------- +# Numeric / expver value normalisation +# --------------------------------------------------------------------------- +# Float keywords use metkit's canonical form: leading/trailing zeros stripped. +@pytest.mark.parametrize( + "value, expected", + [ + ("0.5/to/2.0/by/0.5", [".5", "1", "1.5", "2"]), + ("2.0", ["2"]), + ("-0.5", ["-0.5"]), + ("1000", ["1000"]), + ], +) +def test_float_normalisation(value, expected): + assert expand_key("levelist", value) == expected + + +# expver is zero-padded to four characters but left untouched when alphanumeric. +@pytest.mark.parametrize( + "value, expected", + [("1", "0001"), ("0001", "0001"), ("abcd", "abcd")], +) +def test_expver_normalisation(value, expected): + assert expand_key("expver", value) == [expected] + + +# --------------------------------------------------------------------------- +# Duplicates, "all", and lists without ranges +# --------------------------------------------------------------------------- +def test_duplicates_preserved(): + # 'step' allows duplicate values; they are not de-duplicated. + assert expand_key("step", "1/1/2") == ["1", "1", "2"] + + +def test_all_passthrough(): + assert expand_key("step", "all") == ["all"] + + +def test_plain_list_without_to(): + assert expand_key("step", "0/6/12/18") == ["0", "6", "12", "18"] + + +def test_empty_value_yields_empty_list(): + assert expand_key("step", "") == [] + + +# --------------------------------------------------------------------------- +# Keyword resolution: aliases, prefixes, unknown / ambiguous keys +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("alias", ["levelist", "level", "levellist", "leve"]) +def test_keyword_aliases_resolve(alias): + assert expand_key(alias, "1/to/3") == ["1", "2", "3"] + + +def test_param_resolves_with_context(): + # 'param' is resolved transparently via a full multi-pass expansion. + assert expand_key("param", "130", context={"levtype": "pl"}) == ["130"] + assert expand_key("param", "t", context={"levtype": "pl"}) == ["130"] + # context changes the result: 't' is 130 on pressure levels, 164 on surface + sfc = {"class": "od", "stream": "oper", "type": "an", "levtype": "sfc"} + assert expand_key("param", "t", context=sfc) == ["164"] + + +def test_unknown_keyword_raises(): + with pytest.raises(MetKitException): + expand_key("notakeyword", "1") + + +def test_ambiguous_prefix_raises(): + # 'l' matches several keywords and cannot be resolved unambiguously. + with pytest.raises(MetKitException): + expand_key("l", "1") + + +# --------------------------------------------------------------------------- +# Verb selection +# --------------------------------------------------------------------------- +def test_verb_selects_language(): + # step's default by==12 holds under the 'archive' verb too. + assert expand_key("step", "0/to/24", verb="archive") == ["0", "12", "24"] + + +# --------------------------------------------------------------------------- +# Context-sensitive keywords +# --------------------------------------------------------------------------- +def test_context_as_dict(): + assert expand_key("levelist", "1000/to/850/by/50", context={"levtype": "pl"}) == [ + "1000", + "950", + "900", + "850", + ] + + +def test_context_as_marsrequest(): + context = MarsRequest("retrieve", levtype="ml") + assert expand_key("levelist", "1/to/3", context=context) == ["1", "2", "3"] + + +# --------------------------------------------------------------------------- +# Input shapes: strings, lists, ints +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "value, expected", + [ + ("1/to/5", ["1", "2", "3", "4", "5"]), + (["1", "to", "5"], ["1", "2", "3", "4", "5"]), + ([1, "to", 5], ["1", "2", "3", "4", "5"]), + ([0, 6, 12], ["0", "6", "12"]), + (5, ["5"]), + ], +) +def test_input_shapes(value, expected): + assert expand_key("number", value) == expected + + +# --------------------------------------------------------------------------- +# Strict validation +# --------------------------------------------------------------------------- +def test_strict_rejects_invalid_value(): + with pytest.raises(MetKitException): + expand_key("time", "notatime", strict=True) diff --git a/python/pymetkit/tests/test_expand_context.py b/python/pymetkit/tests/test_expand_context.py new file mode 100644 index 000000000..618ec8a9a --- /dev/null +++ b/python/pymetkit/tests/test_expand_context.py @@ -0,0 +1,198 @@ +"""Tests for :func:`pymetkit.expand_key` that specifically demonstrate +*context dependence*: keywords whose expansion depends on the values of other +keys in the request. + +``expand_key`` performs single-pass expansion, so context matters here in two +ways: + +* keywords backed by a context-switched (``mixed``) type pick a different value + set / parser depending on the supplied ``context`` (e.g. ``model``, ``domain``, + ``bcmodel``, ``tilescheme``); +* a value can be valid in one context and rejected in another, and every rule of + a context (all of its keys) must match for that context to apply. + +The final section documents the deliberate limitation that second-pass, +rule-based resolution (e.g. ``param``) is *not* applied by ``expand_key`` -- that +still requires a full (scoped) request expansion. +""" + +import pytest + +from pymetkit import MarsRequest, MetKitException, expand_key + + +# Contexts reused across the tests. +MS_IT = {"class": "ms", "country": "it"} # selects the Italy model +DT_EXTREMES = {"class": "d1", "dataset": "on-demand-extremes-dt"} + + +# --------------------------------------------------------------------------- +# A value can be valid only within a matching context +# --------------------------------------------------------------------------- +# 'model' is a context-switched keyword. "wam" is only a member of the +# class=ms/country=it value set; with no context it falls back to the generic +# model enum, which does not contain it, so expansion fails. +@pytest.mark.parametrize( + "keyword, value, context", + [ + ("model", "wam", MS_IT), + ("domain", "italy", MS_IT), + ("domain", "euroatl", MS_IT), + ("bcmodel", "gme", MS_IT), + ("icmodel", "ifs", MS_IT), + ], +) +def test_value_valid_only_with_matching_context(keyword, value, context): + # Valid when the context matches ... + assert expand_key(keyword, value, context=context) == [value] + # ... and rejected when the context is absent. + with pytest.raises(MetKitException): + expand_key(keyword, value) + + +# --------------------------------------------------------------------------- +# The same value expands differently per context +# --------------------------------------------------------------------------- +# Without the class=ms/country=it context, "mediterranean"/"europe" resolve +# against the generic domain enum to their single-letter codes; within that +# context they resolve to the named Italy domains. +@pytest.mark.parametrize( + "value, generic, in_context", + [ + ("mediterranean", "m", "mediterranean"), + ("europe", "e", "europe"), + ], +) +def test_same_value_expands_differently_per_context(value, generic, in_context): + assert expand_key("domain", value) == [generic] + assert expand_key("domain", value, context=MS_IT) == [in_context] + + +# --------------------------------------------------------------------------- +# Every rule of a context must match (partial context is not enough) +# --------------------------------------------------------------------------- +# The class=ms/country=it context requires BOTH keys; supplying only one does not +# activate its value set. +@pytest.mark.parametrize( + "partial_context", + [ + {"class": "ms"}, # missing country + {"country": "it"}, # missing class + {"class": "od", "country": "it"}, # wrong class + None, # no context at all + ], +) +def test_partial_context_does_not_activate_rule(partial_context): + with pytest.raises(MetKitException): + expand_key("model", "wam", context=partial_context) + + +def test_full_context_activates_rule(): + assert expand_key("model", "wam", context=MS_IT) == ["wam"] + + +# --------------------------------------------------------------------------- +# Context-independent values for a context-sensitive keyword (contrast) +# --------------------------------------------------------------------------- +# Even though 'model'/'domain' are context-sensitive, some values live in the +# generic fallback set and therefore expand the same way regardless of context. +@pytest.mark.parametrize( + "keyword, value, expected", + [ + ("model", "glob", "global"), # 'glob' is an alias for 'global' + ("model", "ecmf", "ecmf"), + ("domain", "g", "g"), + ], +) +@pytest.mark.parametrize("context", [None, MS_IT, {"class": "ti"}]) +def test_fallback_values_are_context_independent(keyword, value, expected, context): + assert expand_key(keyword, value, context=context) == [expected] + + +# --------------------------------------------------------------------------- +# Aliases / normalisation available only within a context +# --------------------------------------------------------------------------- +# 'tilescheme' only resolves inside the class=d1 / dataset=on-demand-extremes-dt +# context. There the numeric form is an alias for the named value. +def test_alias_resolves_only_in_context(): + assert expand_key("tilescheme", "simple", context=DT_EXTREMES) == ["simple"] + # numeric alias maps to the named value ... + assert expand_key("tilescheme", "1", context=DT_EXTREMES) == ["simple"] + assert expand_key("tilescheme", "2", context=DT_EXTREMES) == ["granular"] + # ... but the same values are meaningless without the context. + with pytest.raises(MetKitException): + expand_key("tilescheme", "simple") + + +# --------------------------------------------------------------------------- +# Context can itself depend on a context-sensitive keyword +# --------------------------------------------------------------------------- +# 'tilescheme' needs class=d1 AND dataset=on-demand-extremes-dt. 'dataset' is +# itself a context-switched keyword, so this exercises a two-level context. +@pytest.mark.parametrize( + "context", + [ + {"class": "d1"}, # missing dataset + {"dataset": "on-demand-extremes-dt"}, # missing class + ], +) +def test_tilescheme_requires_both_context_keys(context): + with pytest.raises(MetKitException): + expand_key("tilescheme", "granular", context=context) + + +# --------------------------------------------------------------------------- +# Context accepted as both a dict and a MarsRequest +# --------------------------------------------------------------------------- +def test_context_dict_and_marsrequest_agree(): + from_dict = expand_key("domain", "euroatl", context=MS_IT) + + req = MarsRequest("retrieve", class_="ms", country="it") + from_request = expand_key("domain", "euroatl", context=req) + + assert from_dict == from_request == ["euroatl"] + + +# --------------------------------------------------------------------------- +# 'param' is resolved transparently via a full multi-pass expansion +# --------------------------------------------------------------------------- +# 'param' depends on other keys, so expand_key builds a scoped request from the +# context, expands it, and returns the resolved param. Context changes the +# result: 't' is 130 on pressure levels but 164 on the surface. +def test_param_resolved_with_context(): + pl = {"class": "od", "stream": "oper", "type": "an", "levtype": "pl"} + sfc = {"class": "od", "stream": "oper", "type": "an", "levtype": "sfc"} + assert expand_key("param", "t", context=pl) == ["130"] + assert expand_key("param", "t", context=sfc) == ["164"] + assert expand_key("param", "2t", context=sfc) == ["167"] + + +# Context may be a dict or a MarsRequest, and both agree. +def test_param_context_dict_and_marsrequest_agree(): + sfc = {"class": "od", "stream": "oper", "type": "an", "levtype": "sfc"} + from_dict = expand_key("param", "t", context=sfc) + + req = MarsRequest( + "retrieve", class_="od", stream="oper", type="an", levtype="sfc" + ) + from_request = expand_key("param", "t", context=req) + + assert from_dict == from_request == ["164"] + + +# expand_key's transparent param resolution matches a full request expansion. +def test_param_expand_key_matches_full_request_expansion(): + req = MarsRequest( + "retrieve", + class_="od", + stream="oper", + type="an", + levtype="pl", + param="t", + date="-1", + expver="1", + step="0", + ) + assert expand_key("param", "t", context={ + "class": "od", "stream": "oper", "type": "an", "levtype": "pl", + }) == [req.expand()["param"]] diff --git a/python/pymetkit/tests/test_expand_param.py b/python/pymetkit/tests/test_expand_param.py new file mode 100644 index 000000000..f279930a6 --- /dev/null +++ b/python/pymetkit/tests/test_expand_param.py @@ -0,0 +1,226 @@ +"""Esoteric, context-dependent tests for ``expand_key("param", ...)``. + +Unlike single-pass keys, ``param`` is resolved by a full multi-pass expansion: +its numeric id depends on the *other* keys in the request (``class``, ``stream``, +``type``, ``levtype``). ``expand_key`` hides that by building a scoped request +from the supplied ``context``, expanding it (``inherit=False`` -- only the given +context drives resolution), and returning the resolved ``param`` values. + +These tests exercise the interesting corners: shortname->id lookup, the four +context axes that flip the result, the GRIB ``param.table`` numeric encoding, +multi-value lists, ``all``/``off`` handling, insufficient-context fallback, and +error cases. +""" + +import pytest + +from pymetkit import MarsRequest, MetKitException, expand_key + + +# Fully-specified operational contexts (all four matcher keys present). +OD_PL = {"class": "od", "stream": "oper", "type": "an", "levtype": "pl"} +OD_SFC = {"class": "od", "stream": "oper", "type": "an", "levtype": "sfc"} + + +def _param(value, context): + return expand_key("param", value, context=context) + + +# --------------------------------------------------------------------------- +# Shortname -> paramid lookup on pressure levels +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "shortname, expected", + [ + ("t", "130"), + ("z", "129"), + ("q", "133"), + ("u", "131"), + ("v", "132"), + ("w", "135"), + ("r", "157"), + ("vo", "138"), + ("d", "155"), + ("gh", "156"), # geopotential *height*, distinct from z (geopotential) + ], +) +def test_pressure_level_shortnames(shortname, expected): + assert _param(shortname, OD_PL) == [expected] + + +# --------------------------------------------------------------------------- +# Shortname -> paramid lookup on the surface +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "shortname, expected", + [ + ("2t", "167"), + ("2d", "168"), + ("10u", "165"), + ("10v", "166"), + ("msl", "151"), + ("sp", "134"), + ("skt", "235"), + ("lsm", "172"), + ("z", "129"), + ], +) +def test_surface_shortnames(shortname, expected): + assert _param(shortname, OD_SFC) == [expected] + + +# --------------------------------------------------------------------------- +# The four context axes that flip the resolved id +# --------------------------------------------------------------------------- +def test_levtype_flips_result(): + # 't' is geopotential-level temperature (130) on pressure levels but + # 2m temperature (164) on the operational surface. + assert _param("t", OD_PL) == ["130"] + assert _param("t", OD_SFC) == ["164"] + + +def test_class_flips_result(): + # Same surface 't', different class: only 'od' maps to the 2m id. + assert _param("t", {**OD_SFC, "class": "od"}) == ["164"] + assert _param("t", {**OD_SFC, "class": "ei"}) == ["130"] + + +def test_type_flips_result(): + # Same surface 't', different type: analysis vs forecast resolve differently. + assert _param("t", {**OD_SFC, "type": "an"}) == ["164"] + assert _param("t", {**OD_SFC, "type": "fc"}) == ["130"] + + +def test_stream_flips_result(): + # 'swh' (significant wave height) resolves differently in the wave stream. + assert _param("swh", {**OD_SFC, "stream": "wave"}) == ["140229"] + assert _param("swh", {**OD_SFC, "stream": "oper"}) == ["3100"] + + +# --------------------------------------------------------------------------- +# Numeric ids pass through unchanged (still validated against the context) +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("value", ["130", "129", "62", "260048"]) +def test_numeric_passthrough(value): + assert _param(value, OD_SFC) == [value] + + +# --------------------------------------------------------------------------- +# GRIB 'param.table' encoding: 'PP.TT' -> str(TT * 1000 + PP), table 128 -> 0 +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "encoded, expected", + [ + ("151.128", "151"), # table 128 collapses to 0 -> bare param + ("228.128", "228"), + ("228.228", "228228"), # 228*1000 + 228 + ("129.228", "228129"), # 228*1000 + 129 + ("8.171", "171008"), # 171*1000 + 8, zero-padded param + ], +) +def test_param_table_encoding(encoded, expected): + assert _param(encoded, OD_SFC) == [expected] + + +# --------------------------------------------------------------------------- +# Multi-value lists: string ("a/b/c"), python list, mixed shortname/numeric, +# and preserved duplicates +# --------------------------------------------------------------------------- +def test_multivalue_string_list(): + assert _param("t/z/u", OD_PL) == ["130", "129", "131"] + + +def test_multivalue_python_list(): + assert _param(["2t", "msl"], OD_SFC) == ["167", "151"] + + +def test_multivalue_mixed_shortname_and_numeric(): + assert _param("t/130/z", OD_PL) == ["130", "130", "129"] + + +def test_multivalue_duplicates_preserved(): + assert _param("t/t/z", OD_PL) == ["130", "130", "129"] + + +# --------------------------------------------------------------------------- +# Shortname matching is case-insensitive +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("shortname, expected", [("2T", "167"), ("MSL", "151")]) +def test_shortname_case_insensitive(shortname, expected): + assert _param(shortname, OD_SFC) == [expected] + + +# --------------------------------------------------------------------------- +# 'all' passes through; 'off' unsets (yields no values) +# --------------------------------------------------------------------------- +def test_all_passthrough(): + assert _param("all", OD_PL) == ["all"] + + +def test_off_yields_no_values(): + assert _param("off", OD_PL) == [] + + +# --------------------------------------------------------------------------- +# Insufficient context: a sparse context that matches no specific rule +# resolves via metkit's default rule +# --------------------------------------------------------------------------- +def test_insufficient_context_uses_default_rule(): + # Only levtype=sfc is supplied; class/stream/type are NOT invented, so the + # surface-specific rule (which would give 164) does not fire -- 't' falls + # back to the default 130 rather than to 164. + assert expand_key("param", "t", context={"levtype": "sfc"}) == ["130"] + # No context at all: same default-rule fallback. + assert expand_key("param", "t") == ["130"] + + +# --------------------------------------------------------------------------- +# Error cases +# --------------------------------------------------------------------------- +def test_unknown_shortname_raises(): + with pytest.raises(MetKitException): + _param("notaparam", OD_SFC) + + +def test_out_of_range_numeric_raises(): + with pytest.raises(MetKitException): + _param("999999", OD_PL) + + +def test_unmatched_param_table_raises(): + # '174.228' encodes 228*1000+174 = 228174, which is not a known id here. + with pytest.raises(MetKitException): + _param("174.228", OD_SFC) + + +# --------------------------------------------------------------------------- +# Context may be a dict or a MarsRequest, and both agree +# --------------------------------------------------------------------------- +def test_context_dict_and_marsrequest_agree(): + from_dict = _param("t", OD_SFC) + + req = MarsRequest( + "retrieve", class_="od", stream="oper", type="an", levtype="sfc" + ) + from_request = expand_key("param", "t", context=req) + + assert from_dict == from_request == ["164"] + + +# --------------------------------------------------------------------------- +# expand_key matches a full request expansion +# --------------------------------------------------------------------------- +@pytest.mark.parametrize("shortname", ["t", "z", "q", "u", "v"]) +def test_parity_with_full_request_expansion(shortname): + req = MarsRequest( + "retrieve", + class_="od", + stream="oper", + type="an", + levtype="pl", + param=shortname, + date="-1", + expver="1", + step="0", + ) + assert expand_key("param", shortname, context=OD_PL) == [req.expand()["param"]] diff --git a/python/pymetkit/tests/test_marsrequest.py b/python/pymetkit/tests/test_marsrequest.py index 9d4ce6f46..2459c43c3 100644 --- a/python/pymetkit/tests/test_marsrequest.py +++ b/python/pymetkit/tests/test_marsrequest.py @@ -2,7 +2,7 @@ from contextlib import nullcontext as does_not_raise import pytest -from pymetkit import parse_mars_request, MarsRequest, MetKitException +from pymetkit import parse_mars_request, expand_key, MarsRequest, MetKitException request = """ retrieve, @@ -81,6 +81,48 @@ def test_empty_request(tmpdir): assert len(requests) == 0 +@pytest.mark.parametrize( + "keyword, value, kwargs, expected", + [ + ["step", "1/to/10/by/1", {}, [str(i) for i in range(1, 11)]], + ["step", [0, 6, 12], {}, ["0", "6", "12"]], + ["time", "6/to/18/by/6", {}, ["0600", "1200", "1800"]], + ["date", "-1", {}, [yesterday]], + # param resolves via full multi-pass expansion using context + ["param", "t", {"context": {"levtype": "pl"}}, ["130"]], + # context-sensitive keyword: levelist depends on levtype + [ + "levelist", + "1000/to/850/by/50", + {"context": {"levtype": "pl"}}, + ["1000", "950", "900", "850"], + ], + ], +) +def test_expand_key(keyword, value, kwargs, expected): + assert expand_key(keyword, value, **kwargs) == expected + + +def test_expand_key_param_resolves_with_context(): + # 'param' is resolved transparently; context drives the result. + sfc = {"class": "od", "stream": "oper", "type": "an", "levtype": "sfc"} + assert expand_key("param", "t", context=sfc) == ["164"] + + +def test_expand_key_context_marsrequest(): + context = MarsRequest("retrieve", levtype="pl") + assert expand_key("levelist", "500/to/300/by/100", context=context) == [ + "500", + "400", + "300", + ] + + +def test_expand_key_strict_raises(): + with pytest.raises(MetKitException): + expand_key("time", "notatime", strict=True) + + def test_new_request(): req = MarsRequest("retrieve") assert req.verb() == "retrieve" diff --git a/src/metkit/api/metkit_c.cc b/src/metkit/api/metkit_c.cc index 0778e04ab..610bfe7be 100644 --- a/src/metkit/api/metkit_c.cc +++ b/src/metkit/api/metkit_c.cc @@ -1,6 +1,7 @@ #include "metkit_c.h" #include #include "eckit/runtime/Main.h" +#include "eckit/utils/StringTools.h" #include "metkit/mars/MarsExpansion.h" #include "metkit/mars/MarsRequest.h" #include "metkit/metkit_version.h" @@ -183,6 +184,30 @@ metkit_error_t metkit_parse_marsrequest(const char* str, metkit_marsrequest_t* r }); } +metkit_error_t metkit_expand_key(const char* verb, const char* keyword, const char* value, + const metkit_marsrequest_t* context, bool strict, metkit_paramiterator_t** values) { + return tryCatch([verb, keyword, value, context, strict, values] { + ASSERT(keyword); + ASSERT(value); + ASSERT(values); + + std::vector tokens = eckit::StringTools::split("/", value); + + metkit::mars::MarsExpansion expansion(false, strict); + + std::vector expanded; + if (context) { + expanded = expansion.expandKey(verb ? verb : "retrieve", keyword, std::move(tokens), *context); + } + else { + expanded = + expansion.expandKey(verb ? verb : "retrieve", keyword, std::move(tokens), metkit::mars::MarsRequest{}); + } + + *values = new metkit_paramiterator_t(std::move(expanded)); + }); +} + // ----------------------------------------------------------------------------- // REQUEST // ----------------------------------------------------------------------------- diff --git a/src/metkit/api/metkit_c.h b/src/metkit/api/metkit_c.h index 947a490e9..b69c2b4d7 100644 --- a/src/metkit/api/metkit_c.h +++ b/src/metkit/api/metkit_c.h @@ -92,6 +92,34 @@ metkit_error_t metkit_parse_marsrequests(const char* str, metkit_requestiterator * @return metkit_error_t Error code */ metkit_error_t metkit_parse_marsrequest(const char* str, metkit_marsrequest_t* request, bool strict); +/* --------------------------------------------------------------------------------------------------------------------- + * KEY PARSING + * --- */ + +/** + * Parse/normalise the values of a single MARS keyword in isolation, without building a full request. + * + * Applies the MARS language rules for the keyword: range syntax such as "1/to/10/by/1" is expanded and + * per-key normalisation is performed (e.g. date "-1" resolves to a yyyymmdd date, time "6" to "0600"). + * The value string is split on '/' before expansion. + * + * Context-sensitive keywords (e.g. those whose interpretation depends on other keys) consult @p context: + * pass a Request populated with the relevant neighbouring keys, or NULL/empty when not needed. + * + * @note This performs single-pass expansion only. Second-pass, rule-based resolution (e.g. 'param') and + * default inheritance are NOT applied; use metkit_marsrequest_expand on a (scoped) request for those. + * + * @param verb MARS verb whose language defines the keyword (NULL defaults to "retrieve") + * @param keyword keyword to parse (canonical, alias or unambiguous prefix) + * @param value values to expand, as a '/'-separated string (e.g. "1/to/10/by/1") + * @param context Request providing context for context-sensitive keywords, or NULL + * @param strict if true, validate expanded values and raise an error on invalid values + * @param[out] values ParamIterator over the expanded values. Must be deallocated with + * metkit_paramiterator_delete + * @return metkit_error_t Error code + */ +metkit_error_t metkit_expand_key(const char* verb, const char* keyword, const char* value, + const metkit_marsrequest_t* context, bool strict, metkit_paramiterator_t** values); /* --------------------------------------------------------------------------------------------------------------------- * REQUEST * --- */ diff --git a/src/metkit/mars/MarsExpansion.cc b/src/metkit/mars/MarsExpansion.cc index f879d9c44..1d1ecfe00 100644 --- a/src/metkit/mars/MarsExpansion.cc +++ b/src/metkit/mars/MarsExpansion.cc @@ -60,6 +60,11 @@ MarsRequest MarsExpansion::expand(const MarsRequest& request) { return language(request.verb()).expand(request, inherit_, strict_); } +std::vector MarsExpansion::expandKey(const std::string& verb, const std::string& keyword, + std::vector values, const MarsRequest& context) { + return language(verb).expandKey(keyword, std::move(values), context, strict_); +} + void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) { callback(expand(request)); } diff --git a/src/metkit/mars/MarsExpansion.h b/src/metkit/mars/MarsExpansion.h index 70f3ff851..a2c5f4f22 100644 --- a/src/metkit/mars/MarsExpansion.h +++ b/src/metkit/mars/MarsExpansion.h @@ -65,6 +65,16 @@ class MarsExpansion : public eckit::NonCopyable { MarsRequest expand(const MarsRequest&); std::vector expand(const std::vector&); + /// @brief Parse/normalise the values of a single keyword in isolation, using (and caching) the + /// language for @p verb. See MarsLanguage::expandKey for semantics and limitations. + /// @param verb MARS verb whose language defines the keyword (e.g. "retrieve") + /// @param keyword keyword to parse (canonical, alias or unambiguous prefix) + /// @param values values to expand (already split on '/') + /// @param context other keys providing context for context-sensitive types + /// @return expanded/normalised values + std::vector expandKey(const std::string& verb, const std::string& keyword, + std::vector values, const MarsRequest& context = {}); + void expand(const MarsRequest&, ExpandCallback&); void flatten(const MarsRequest&, FlattenCallback&); diff --git a/src/metkit/mars/MarsLanguage.cc b/src/metkit/mars/MarsLanguage.cc index ce6f2bc5a..5632d9d34 100644 --- a/src/metkit/mars/MarsLanguage.cc +++ b/src/metkit/mars/MarsLanguage.cc @@ -439,6 +439,35 @@ Type* MarsLanguage::type(const std::string& name) const { } +std::vector MarsLanguage::expandKey(const std::string& keyword, std::vector values, + const MarsRequest& context, bool strict) { + std::string p = eckit::StringTools::lower(keyword); + + std::string canonical; + if (auto c = cache_.find(p); c != cache_.end()) { + canonical = c->second; + } + else { + canonical = cache_[p] = bestMatch(p, keywords_, true, false, true, aliases_); + } + + Type* t = type(canonical); + + if (values.size() == 1) { + const std::string& s = eckit::StringTools::lower(values[0]); + if (s == "all" && t->multiple()) { + return {"all"}; + } + } + + t->expand(values, context); + if (strict) { + t->check(values); + } + return values; +} + + MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict) { MarsRequest result(verb_); diff --git a/src/metkit/mars/MarsLanguage.h b/src/metkit/mars/MarsLanguage.h index 7e4310bea..a438869cd 100644 --- a/src/metkit/mars/MarsLanguage.h +++ b/src/metkit/mars/MarsLanguage.h @@ -63,6 +63,22 @@ class MarsLanguage : private eckit::NonCopyable { Type* type(const std::string& name) const; + /// @brief Parse/normalise the values of a single keyword, in isolation from a full request. + /// Resolves @p keyword against this verb's language (aliases and partial matches allowed), + /// then expands @p values using the keyword's Type. Range syntax (e.g. "1/to/10/by/1") and + /// per-key normalisation (e.g. date/time) are applied. Context-sensitive keys (e.g. those + /// backed by TypeMixed) consult @p context; supply the relevant neighbouring keys there. + /// @note This performs the single-pass expansion only. Second-pass resolution (pass2/finalise, + /// e.g. rule-based 'param' expansion) and default inheritance are not applied here; use a + /// (scoped) MarsRequest expansion for those. + /// @param keyword keyword to parse (canonical, alias or unambiguous prefix) + /// @param values values to expand (already split on '/') + /// @param context other keys providing context for context-sensitive types + /// @param strict if true, validate expanded values and throw on invalid values + /// @return expanded/normalised values + std::vector expandKey(const std::string& keyword, std::vector values, + const MarsRequest& context = {}, bool strict = false); + bool isData(const std::string& keyword) const; bool isPostProc(const std::string& keyword) const;