METK-171: Add API for mars key parsing

Author: caraghbiner

python/pymetkit/src/pymetkit/metkit_c.h

@@ -27,6 +27,8 @@ const char* metkit_git_sha1();
 metkit_error_t metkit_initialise();
 
 metkit_error_t metkit_parse_marsrequests(const char* str, metkit_requestiterator_t** requests, bool strict);
+metkit_error_t metkit_parse_key(const char* verb, const char* keyword, const char* value,
+                                const metkit_marsrequest_t* context, bool strict, metkit_paramiterator_t** values);
 metkit_error_t metkit_marsrequest_new(metkit_marsrequest_t** request);
 metkit_error_t metkit_marsrequest_delete(const metkit_marsrequest_t* request);
 metkit_error_t metkit_marsrequest_set(metkit_marsrequest_t* request, const char* param, const char* values[],

python/pymetkit/src/pymetkit/pymetkit.py

@@ -219,6 +219,75 @@ def parse_mars_request(file_or_str: IO | str, strict: bool = False) -> list[Mars
     return requests
 
 
+def parse_key(
+    keyword: str,
+    value: str | int | list,
+    verb: str = "retrieve",
+    context: "MarsRequest | dict | None" = None,
+    strict: bool = False,
+) -> list[str]:
+    """Parse/normalise the values of a single MARS key, without building a full request.
+
+    Applies the MARS language rules for ``keyword``: range syntax such as
+    ``"1/to/10/by/1"`` is expanded, and per-key normalisation is performed (e.g.
+    ``date="-1"`` resolves to a ``yyyymmdd`` date, ``time="6"`` to ``"0600"``).
+
+    Context-sensitive keys (e.g. those whose interpretation depends on other
+    keys) consult ``context``. Supply the relevant neighbouring keys there, e.g.
+    ``parse_key("levelist", "1/to/10", context={"levtype": "ml"})``.
+
+    Note: this performs single-pass expansion only. Second-pass, rule-based
+    resolution (e.g. ``param``) and default inheritance are not applied; use
+    :meth:`MarsRequest.expand` on a (scoped) request for those.
+
+    Params
+    ------
+    keyword: name of the MARS key to parse (canonical, alias or unambiguous prefix)
+    value: values to expand, as a ``"a/b/c"`` string or a list of tokens
+    verb: MARS verb whose language defines the key (defaults to "retrieve")
+    context: optional MarsRequest or dict of neighbouring keys for context-sensitive keys
+    strict: if True, raise an error on invalid values
+
+    Returns
+    -------
+    list of expanded/normalised string values
+
+    Examples
+    --------
+    >>> parse_key("step", "1/to/10/by/1")
+    ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10']
+    """
+    if isinstance(value, (list, tuple)):
+        value = "/".join(str(v) for v in value)
+    else:
+        value = str(value)
+
+    context_c = ffi.NULL
+    if context is not None:
+        if isinstance(context, dict):
+            context = MarsRequest(**context)
+        context_c = context.ctype()
+
+    it_c = ffi.new("metkit_paramiterator_t **")
+    lib.metkit_parse_key(
+        ffi_encode(verb),
+        ffi_encode(keyword),
+        ffi_encode(value),
+        context_c,
+        strict,
+        it_c,
+    )
+    it = ffi.gc(it_c[0], lib.metkit_paramiterator_delete)
+
+    values = []
+    while lib.metkit_paramiterator_next(it) == lib.METKIT_ITERATOR_SUCCESS:
+        cvalue = ffi.new("const char **")
+        lib.metkit_paramiterator_current(it, cvalue)
+        values.append(ffi_decode(cvalue[0]))
+
+    return values
+
+
 class MetKitException(RuntimeError):
     """Raised when MetKit library throws exception"""
 

python/pymetkit/tests/test_marsrequest.py

@@ -2,7 +2,7 @@
 from contextlib import nullcontext as does_not_raise
 import pytest
 
-from pymetkit import parse_mars_request, MarsRequest, MetKitException
+from pymetkit import parse_mars_request, parse_key, MarsRequest, MetKitException
 
 request = """
 retrieve,
@@ -81,6 +81,42 @@ def test_empty_request(tmpdir):
     assert len(requests) == 0
 
 
+@pytest.mark.parametrize(
+    "keyword, value, kwargs, expected",
+    [
+        ["step", "1/to/10/by/1", {}, [str(i) for i in range(1, 11)]],
+        ["step", [0, 6, 12], {}, ["0", "6", "12"]],
+        ["time", "6/to/18/by/6", {}, ["0600", "1200", "1800"]],
+        ["date", "-1", {}, [yesterday]],
+        # alias resolves to canonical keyword
+        ["parameter", "130", {}, ["130"]],
+        # context-sensitive keyword: levelist depends on levtype
+        [
+            "levelist",
+            "1000/to/850/by/50",
+            {"context": {"levtype": "pl"}},
+            ["1000", "950", "900", "850"],
+        ],
+    ],
+)
+def test_parse_key(keyword, value, kwargs, expected):
+    assert parse_key(keyword, value, **kwargs) == expected
+
+
+def test_parse_key_context_marsrequest():
+    context = MarsRequest("retrieve", levtype="pl")
+    assert parse_key("levelist", "500/to/300/by/100", context=context) == [
+        "500",
+        "400",
+        "300",
+    ]
+
+
+def test_parse_key_strict_raises():
+    with pytest.raises(MetKitException):
+        parse_key("time", "notatime", strict=True)
+
+
 def test_new_request():
     req = MarsRequest("retrieve")
     assert req.verb() == "retrieve"

src/metkit/api/metkit_c.cc

@@ -1,6 +1,7 @@
 #include "metkit_c.h"
 #include <functional>
 #include "eckit/runtime/Main.h"
+#include "eckit/utils/StringTools.h"
 #include "metkit/mars/MarsExpansion.h"
 #include "metkit/mars/MarsRequest.h"
 #include "metkit/metkit_version.h"
@@ -183,6 +184,30 @@ metkit_error_t metkit_parse_marsrequest(const char* str, metkit_marsrequest_t* r
     });
 }
 
+metkit_error_t metkit_parse_key(const char* verb, const char* keyword, const char* value,
+                                const metkit_marsrequest_t* context, bool strict, metkit_paramiterator_t** values) {
+    return tryCatch([verb, keyword, value, context, strict, values] {
+        ASSERT(keyword);
+        ASSERT(value);
+        ASSERT(values);
+
+        std::vector<std::string> tokens = eckit::StringTools::split("/", value);
+
+        metkit::mars::MarsExpansion expansion(false, strict);
+
+        std::vector<std::string> expanded;
+        if (context) {
+            expanded = expansion.parseKey(verb ? verb : "retrieve", keyword, std::move(tokens), *context);
+        }
+        else {
+            expanded =
+                expansion.parseKey(verb ? verb : "retrieve", keyword, std::move(tokens), metkit::mars::MarsRequest{});
+        }
+
+        *values = new metkit_paramiterator_t(std::move(expanded));
+    });
+}
+
 // -----------------------------------------------------------------------------
 //                           REQUEST
 // -----------------------------------------------------------------------------

src/metkit/api/metkit_c.h

@@ -92,6 +92,34 @@ metkit_error_t metkit_parse_marsrequests(const char* str, metkit_requestiterator
  * @return metkit_error_t Error code
  */
 metkit_error_t metkit_parse_marsrequest(const char* str, metkit_marsrequest_t* request, bool strict);
+/* ---------------------------------------------------------------------------------------------------------------------
+ * KEY PARSING
+ * --- */
+
+/**
+ * Parse/normalise the values of a single MARS keyword in isolation, without building a full request.
+ *
+ * Applies the MARS language rules for the keyword: range syntax such as "1/to/10/by/1" is expanded and
+ * per-key normalisation is performed (e.g. date "-1" resolves to a yyyymmdd date, time "6" to "0600").
+ * The value string is split on '/' before expansion.
+ *
+ * Context-sensitive keywords (e.g. those whose interpretation depends on other keys) consult @p context:
+ * pass a Request populated with the relevant neighbouring keys, or NULL/empty when not needed.
+ *
+ * @note This performs single-pass expansion only. Second-pass, rule-based resolution (e.g. 'param') and
+ * default inheritance are NOT applied; use metkit_marsrequest_expand on a (scoped) request for those.
+ *
+ * @param verb MARS verb whose language defines the keyword (NULL defaults to "retrieve")
+ * @param keyword keyword to parse (canonical, alias or unambiguous prefix)
+ * @param value values to expand, as a '/'-separated string (e.g. "1/to/10/by/1")
+ * @param context Request providing context for context-sensitive keywords, or NULL
+ * @param strict if true, validate expanded values and raise an error on invalid values
+ * @param[out] values ParamIterator over the expanded values. Must be deallocated with
+ *                    metkit_paramiterator_delete
+ * @return metkit_error_t Error code
+ */
+metkit_error_t metkit_parse_key(const char* verb, const char* keyword, const char* value,
+                                const metkit_marsrequest_t* context, bool strict, metkit_paramiterator_t** values);
 /* ---------------------------------------------------------------------------------------------------------------------
  * REQUEST
  * --- */

src/metkit/mars/MarsExpansion.cc

@@ -60,6 +60,11 @@ MarsRequest MarsExpansion::expand(const MarsRequest& request) {
     return language(request.verb()).expand(request, inherit_, strict_);
 }
 
+std::vector<std::string> MarsExpansion::parseKey(const std::string& verb, const std::string& keyword,
+                                                 std::vector<std::string> values, const MarsRequest& context) {
+    return language(verb).parseKey(keyword, std::move(values), context, strict_);
+}
+
 void MarsExpansion::expand(const MarsRequest& request, ExpandCallback& callback) {
     callback(expand(request));
 }

src/metkit/mars/MarsExpansion.h

@@ -65,6 +65,16 @@ class MarsExpansion : public eckit::NonCopyable {
     MarsRequest expand(const MarsRequest&);
     std::vector<MarsRequest> expand(const std::vector<MarsParsedRequest>&);
 
+    /// @brief Parse/normalise the values of a single keyword in isolation, using (and caching) the
+    /// language for @p verb. See MarsLanguage::parseKey for semantics and limitations.
+    /// @param verb MARS verb whose language defines the keyword (e.g. "retrieve")
+    /// @param keyword keyword to parse (canonical, alias or unambiguous prefix)
+    /// @param values values to expand (already split on '/')
+    /// @param context other keys providing context for context-sensitive types
+    /// @return expanded/normalised values
+    std::vector<std::string> parseKey(const std::string& verb, const std::string& keyword,
+                                      std::vector<std::string> values, const MarsRequest& context = {});
+
     void expand(const MarsRequest&, ExpandCallback&);
     void flatten(const MarsRequest&, FlattenCallback&);
 

src/metkit/mars/MarsLanguage.cc

@@ -439,6 +439,35 @@ Type* MarsLanguage::type(const std::string& name) const {
 }
 
 
+std::vector<std::string> MarsLanguage::parseKey(const std::string& keyword, std::vector<std::string> values,
+                                                const MarsRequest& context, bool strict) {
+    std::string p = eckit::StringTools::lower(keyword);
+
+    std::string canonical;
+    if (auto c = cache_.find(p); c != cache_.end()) {
+        canonical = c->second;
+    }
+    else {
+        canonical = cache_[p] = bestMatch(p, keywords_, true, false, true, aliases_);
+    }
+
+    Type* t = type(canonical);
+
+    if (values.size() == 1) {
+        const std::string& s = eckit::StringTools::lower(values[0]);
+        if (s == "all" && t->multiple()) {
+            return {"all"};
+        }
+    }
+
+    t->expand(values, context);
+    if (strict) {
+        t->check(values);
+    }
+    return values;
+}
+
+
 MarsRequest MarsLanguage::expand(const MarsRequest& r, bool inherit, bool strict) {
     MarsRequest result(verb_);
 

src/metkit/mars/MarsLanguage.h

@@ -63,6 +63,22 @@ class MarsLanguage : private eckit::NonCopyable {
 
     Type* type(const std::string& name) const;
 
+    /// @brief Parse/normalise the values of a single keyword, in isolation from a full request.
+    /// Resolves @p keyword against this verb's language (aliases and partial matches allowed),
+    /// then expands @p values using the keyword's Type. Range syntax (e.g. "1/to/10/by/1") and
+    /// per-key normalisation (e.g. date/time) are applied. Context-sensitive keys (e.g. those
+    /// backed by TypeMixed) consult @p context; supply the relevant neighbouring keys there.
+    /// @note This performs the single-pass expansion only. Second-pass resolution (pass2/finalise,
+    /// e.g. rule-based 'param' expansion) and default inheritance are not applied here; use a
+    /// (scoped) MarsRequest expansion for those.
+    /// @param keyword keyword to parse (canonical, alias or unambiguous prefix)
+    /// @param values values to expand (already split on '/')
+    /// @param context other keys providing context for context-sensitive types
+    /// @param strict if true, validate expanded values and throw on invalid values
+    /// @return expanded/normalised values
+    std::vector<std::string> parseKey(const std::string& keyword, std::vector<std::string> values,
+                                      const MarsRequest& context = {}, bool strict = false);
+
     bool isData(const std::string& keyword) const;
 
     bool isPostProc(const std::string& keyword) const;