Merge pull request #325 from iiasa/issue/305

Derive aviation final energy share from solved MESSAGEix-Transport scenarios

Author: khaeru

.github/workflows/pytest.yaml

@@ -137,7 +137,7 @@ jobs:
 
         version_string = "${{ matrix.version.upstream }}"
 
-        if version_string != "main": 
+        if version_string != "main":
             v, result = parse(version_string), []
             for condition, dependency in (
                 (v <= Version("v3.7.0"), "genno < 1.25"),  # Upstream versions < 3.8.0 import genno.computations, removed in 1.25.0 (#156)
@@ -174,7 +174,7 @@ jobs:
           -m "not (ece_db or snapshot)" \
           --color=yes --durations=20 -rA --verbose \
           --cov-report=term-missing --cov-report=xml \
-          --numprocesses=auto \
+          --numprocesses=auto --dist=loadscope \
           --local-cache --jvmargs="-Xmx6G"
 
     - name: Upload test coverage to Codecov.io

doc/project/ssp.rst

@@ -72,18 +72,21 @@ Transport
    1. To process data from file, use :program:`mix-models ssp transport --help in.xlsx out.xlsx` to invoke :func:`.process_file`.
       Data are read from PATH_IN, in :file:`.xlsx` or :file:`.csv` format.
       If :file:`.xlsx`, the data are first temporarily converted to :file:`.csv`.
-      Data are written to PATH_OUT; if not given, this defaults to the same path and suffix as PATH_IN, with "_out" added to the stem.
+      Data are written to PATH_OUT;
+      if not given, this defaults to the same path and suffix as PATH_IN with "_out" added to the stem.
 
       For example:
 
       .. code-block:: shell
 
-         mix-models ssp transport --method=B \
+         mix-models --platform=ixmp-dev \
+           ssp transport --method=C \
            SSP_SSP2_v2.1_baseline.xlsx
 
       …produces a file :file:`SSP_SSP2_v2.1_baseline_out.xlsx` in the same directory.
 
-   2. To process an existing :class:`pandas.DataFrame` from other code, call :func:`.process_df`, passing the input dataframe and the `method` parameter.
+   2. To process an existing :class:`pandas.DataFrame` from other code, call :func:`.process_df`,
+      passing the input dataframe and the `method` parameter.
 
    As of 2025-03-07 / :pull:`309`, the set of required "variable" codes handled includes::
 
@@ -92,15 +95,13 @@ Transport
        Emissions|.*|Energy|Demand|Transportation
        Emissions|.*|Energy|Demand|Transportation|Road Rail and Domestic Shipping
 
-   The previous set is no longer supported.
-
-   As of 2025-01-25:
-
-   - The set of "variable" codes modified includes::
+   The previous set, supported as of 2025-01-25 but no longer supported, included::
 
        Emissions|.*|Energy|Demand|Transportation|Aviation
        Emissions|.*|Energy|Demand|Transportation|Aviation|International
        Emissions|.*|Energy|Demand|Transportation|Road Rail and Domestic Shipping
 
-   - Method 'B' (that is, :func:`.prepare_method_B`; see its documentation) is the preferred method.
-   - The code is tested on :file:`.xlsx` files in the (internal) directories under `SharePoint > ECE > Documents > SharedSocioeconomicPathways2023 > Scenario_Vetting <https://iiasahub.sharepoint.com/sites/eceprog/Shared%20Documents/Forms/AllItems.aspx?csf=1&web=1&e=APKv0Z&CID=23fa0a51%2Dc303%2D4381%2D8c6d%2D143305cbc5a1&FolderCTID=0x012000AA9481BF7BE9264E85B14105F7F082FF&id=%2Fsites%2Feceprog%2FShared%20Documents%2FSharedSocioEconomicPathways2023%2FScenario%5FVetting&viewid=956acd8a%2De1e7%2D4ae9%2Dab1b%2D0506911bae11>`_, for example :file:`v2.1_Internal_version_Dec13_2024/Reporting_output/SSP_SSP2_v2.1_baseline.xlsx`.
+   - Method 'C' (that is, :func:`.method_C`; see its documentation) is the preferred method.
+   - The code is tested on :file:`.xlsx` files in the (internal) directories under `SharePoint > ECE > Documents > SharedSocioeconomicPathways2023 > Scenario_Vetting <https://iiasahub.sharepoint.com/sites/eceprog/Shared%20Documents/Forms/AllItems.aspx?csf=1&web=1&e=APKv0Z&CID=23fa0a51%2Dc303%2D4381%2D8c6d%2D143305cbc5a1&FolderCTID=0x012000AA9481BF7BE9264E85B14105F7F082FF&id=%2Fsites%2Feceprog%2FShared%20Documents%2FSharedSocioEconomicPathways2023%2FScenario%5FVetting&viewid=956acd8a%2De1e7%2D4ae9%2Dab1b%2D0506911bae11>`_,
+     for example :file:`v2.1_Internal_version_Dec13_2024/Reporting_output/SSP_SSP2_v2.1_baseline.xlsx`
+     or :file:`v2.3_v2.4_Submission_Mar01_2025/Scenario_Reporting_Files/SSP_LED_v2.3.1_baseline.xlsx`

doc/whatsnew.rst

@@ -44,7 +44,8 @@ SSP :ref:`ssp-2024`/ScenarioMIP
 
 Improve :mod:`.ssp.transport`:
 
-- Add :func:`.prepare_method_B` and make this the default (:pull:`259`).
+- Add :func:`~.ssp.transport.method_B` and make this the default (:pull:`259`).
+- Add :func:`~.ssp.transport.method_C` (:issue:`305`, :pull:`325`).
 - Add :func:`~.ssp.transport.process_df` (:pull:`303`).
 - Adapt to revised ‘variable’ codes (:pull:`309`, :issue:`304`).
 

message_ix_models/cli.py

@@ -74,7 +74,7 @@ def main(click_ctx, **kwargs):
     # NB this can't be Context.only(). When click.testing.CliRunner is used, there may
     #    already be ≥2 Context instances created elsewhere in the test session before
     #    this function is called to run CLI commands within the test session.
-    click_ctx.obj = Context.get_instance(-1)
+    click_ctx.obj = Context()
 
     # Handle command-line parameters
     click_ctx.obj.core.handle_cli_args(**kwargs)

message_ix_models/model/transport/base.py

@@ -8,7 +8,7 @@
 import genno
 import numpy as np
 import pandas as pd
-from genno import Computer, Key, KeySeq, quote
+from genno import Computer, Key, quote
 from genno.core.key import single_key
 
 from message_ix_models.util import minimum_version
@@ -80,7 +80,7 @@ def smooth(c: Computer, key: "genno.Key", *, dim: str = "ya") -> "genno.Key":
     from itertools import pairwise
 
     assert key.tag != "2"
-    ks = KeySeq(key.remove_tag(key.tag or ""))
+    ks = Key(key.remove_tag(key.tag or ""))
 
     def first_block_false(column: pd.Series) -> pd.Series:
         """Modify `column` to contain at most one contiguous block of :data:`.False`."""
@@ -181,9 +181,9 @@ def prepare_reporter(rep: "message_ix.Reporter") -> str:
 
     # Keys for starting quantities
     e_iea = Key("energy:n-y-product-flow:iea")
-    e_fnp = KeySeq(e_iea.drop("y"))
+    e_fnp = Key(e_iea.drop("y"))
     e_cnlt = Key("energy:c-nl-t:iea+0")
-    k = KeySeq("in:nl-t-ya-c-l-h:transport+units")  # MESSAGE solution values
+    k = Key("in:nl-t-ya-c-l-h:transport+units")  # MESSAGE solution values
 
     # First period
     y0 = rep.get("y0")
@@ -205,18 +205,18 @@ def prepare_reporter(rep: "message_ix.Reporter") -> str:
     )
 
     # Transport outputs for comparison
-    rep.add(k[0], "select", k.base, indexers=dict(ya=y0), drop=True)
-    rep.add(k[1], "aggregate", k[0], "groups::transport to iea", keep=False)
+    rep.add(k[0], "aggregate", k, "groups::transport to iea", keep=False, sums=True)
+    rep.add(k[1] / "ya", "select", k[0], indexers=dict(ya=y0), drop=True)
 
     # Scaling factor 1: ratio of MESSAGEix-Transport outputs to IEA data
-    tmp = rep.add("scale 1", "div", k[1], e_cnlt)
-    s1 = KeySeq(tmp)
-    rep.add(s1[1], "convert_units", s1.base, units="1 / a")
+    tmp = rep.add("scale 1", "div", k[1] / "ya", e_cnlt)
+    s1 = Key(single_key(tmp))
+    rep.add(s1[1], "convert_units", s1, units="1 / a")
     rep.add(s1[2], "mul", s1[1], genno.Quantity(1.0, units="a"))
     # Replace ~0 and ∞ values with 1.0; this avoids x / 0 = inf
     rep.add(s1[3], "where", s1[2], cond=lambda v: (v > 1e-3) & (v != np.inf), other=1.0)
     # Ensure no values are dropped versus the numerator (= MESSAGE outputs)
-    rep.add(s1[4], align_and_fill, s1[3], k[1])
+    rep.add(s1[4], align_and_fill, s1[3], k[1] / "ya")
 
     rep.apply(to_csv, s1[4], name=s1.name, header_comment=SCALE_1_HEADER)
 
@@ -241,7 +241,7 @@ def prepare_reporter(rep: "message_ix.Reporter") -> str:
 
     # Correct MESSAGEix-Transport outputs for the MESSAGEix-base model using the high-
     # resolution scaling factor
-    rep.add(k["s1"], "div", k.base, s1[11])
+    rep.add(k["s1"], "div", k, s1[11])
 
     # Scaling factor 2: ratio of total of scaled data to IEA total
     rep.add(k[2] / "ya", "select", k["s1"], indexers=dict(ya=y0), drop=True, sums=True)
@@ -253,8 +253,8 @@ def prepare_reporter(rep: "message_ix.Reporter") -> str:
         drop=True,
     )
     tmp = rep.add("scale 2", "div", k[2] / ("c", "t", "ya"), "energy:nl:iea+transport")
-    s2 = KeySeq(tmp)
-    rep.add(s2[1], "convert_units", s2.base, units="1 / a")
+    s2 = Key(single_key(tmp))
+    rep.add(s2[1], "convert_units", s2, units="1 / a")
     rep.add(s2[2], "mul", s2[1], genno.Quantity(1.0, units="a"))
 
     rep.apply(to_csv, s2[2], name=s2.name, header_comment=SCALE_2_HEADER)
@@ -283,13 +283,13 @@ def prepare_reporter(rep: "message_ix.Reporter") -> str:
     # - Sum across the "t" dimension of `k` to avoid conflict with "t" labels introduced
     #   by the data from file.
     tmp = rep.add("ue", "div", k["s2"] / "t", "input:t-c-h:base")
-    ue = KeySeq(tmp)
+    ue = Key(single_key(tmp))
 
-    rep.apply(share_constraints, k["s2"], ue.base)
+    rep.apply(share_constraints, k["s2"], ue)
 
     # Ensure units: in::transport+units [=] GWa/a and input::base [=] GWa; their ratio
     # gives units 1/a. The base model expects "GWa" for all 3 parameters.
-    rep.add(ue[1], "mul", ue.base, genno.Quantity(1.0, units="GWa * a"))
+    rep.add(ue[1], "mul", ue, genno.Quantity(1.0, units="GWa * a"))
     rep.apply(to_csv, ue[1] / ("c", "t"), name="demand no fill")
 
     # 'Smooth' ue[1] data by interpolating any dip below the base year value

message_ix_models/model/transport/build.py

@@ -570,7 +570,7 @@ def get_computer(
     if visualize and HAS_GRAPHVIZ:
         path = context.get_local_path("transport", "build.svg")
         path.parent.mkdir(exist_ok=True)
-        c.visualize(filename=path, key="add transport data")
+        c.visualize(filename=path, key="add transport data", rankdir="LR")
         log.info(f"Visualization written to {path}")
 
     return c

message_ix_models/model/transport/config.py

@@ -405,8 +405,21 @@ def set_navigate_scenario(self, value: Optional[str]) -> None:
         self.project.update(navigate=s)
         self.check()
 
-    def use_scenario_code(self, code: "common.Code") -> None:
-        """Update settings given a `code` with :class:`ScenarioCodeAnnotations`."""
+    def use_scenario_code(self, code: "common.Code") -> tuple[str, str]:
+        """Update settings given a `code` with :class:`ScenarioCodeAnnotations`.
+
+        Returns
+        -------
+        tuple of str
+            The entries are:
+
+            1. A short label suitable for a :class:`.Workflow` step name, for instance
+               "SSP3 policy" or "SSP5", where the first part is :py:`code.id`. See
+               :func:`.transport.workflow.generate`.
+            2. A longer, more explicity label suitable for (part of) a
+               :attr:`message_ix.Scenario.scenario` name in an :mod:`ixmp` database, for
+               instance "SSP_2024.3".
+        """
         sca = ScenarioCodeAnnotations.from_obj(code)
 
         # Look up the SSP_2024 Enum
@@ -418,6 +431,17 @@ def use_scenario_code(self, code: "common.Code") -> None:
 
         self.base_scenario_url = sca.base_scenario_URL
 
+        # Construct labels including the SSP code and policy identifier
+        # ‘Short’ label used for workflow steps
+        label = f"{code.id}{' policy' if self.policy else ''}"
+        # ‘Full’ label used in the scenario name
+        if not sca.is_LED_scenario and sca.EDITS_activity_id is None:
+            label_full = f"SSP_2024.{self.ssp.name}"
+        else:
+            label_full = label
+
+        return label, label_full
+
 
 @dataclass
 class ScenarioCodeAnnotations(AnnotationsMixIn):

message_ix_models/model/transport/structure.py

@@ -62,6 +62,7 @@ def _leaf_ids(node) -> list[str]:
     for tech in filter(lambda t: len(t.child), t_list):
         result[tech.id] = _leaf_ids(tech)
         # Store non-LDV technologies
+        # FIXME This is actually "P ex LDV"; label as such
         if tech.id != "LDV":
             result["non-ldv"].extend(result[tech.id])
 

message_ix_models/model/transport/workflow.py

@@ -210,22 +210,13 @@ def generate(
         config = deepcopy(context.transport)
 
         # Update the .transport.Config from the `scenario_code` and `policy`
-        config.use_scenario_code(scenario_code)
         config.policy = policy
+        label, label_full = config.use_scenario_code(scenario_code)
 
         # Retrieve updated values
         is_LED = config.project["LED"]
         EDITS_activity = config.project["EDITS"]["activity"]
 
-        # Construct labels including the SSP code and policy identifier
-        # ‘Short’ label used for workflow steps
-        label = f"{scenario_code.id}{' policy' if policy else ''}"
-        # ‘Full’ label used in the scenario name
-        if not is_LED and EDITS_activity is None:
-            label_full = f"SSP_2024.{config.ssp.name}"
-        else:
-            label_full = label
-
         if config.policy and (is_LED or EDITS_activity is not None):  # TEMPORARY
             log.info(f"({label_full}, {config.policy=}) → skip")
             continue

message_ix_models/project/ssp/cli.py

@@ -25,7 +25,7 @@ def gen_structures(context, **kwargs):
 
 
 @cli.command("transport")
-@click.option("--method", type=click.Choice(["A", "B"]), required=True)
+@click.option("--method", type=click.Choice(["A", "B", "C"]), required=True)
 @click.argument("path_in", type=click.Path(exists=True, dir_okay=False, path_type=Path))
 @click.argument(
     "path_out",
@@ -39,11 +39,14 @@ def transport_cmd(context: "Context", method, path_in: Path, path_out: Optional[
     Data are read from PATH_IN, in .xlsx or .csv format. If .xlsx, the data are first
     temporarily converted to .csv. Data are written to PATH_OUT; if not given, this
     defaults to the same path and suffix as PATH_IN, with "_out" added to the stem.
+
+    For --method=C, the top-level option --platform=ixmp-dev (for example) may be used
+    to specify the Platform on which to locate solved MESSAGEix-Transport scenarios.
     """
     import pandas as pd
     from platformdirs import user_cache_path
 
-    from .transport import process_file
+    from .transport import METHOD, process_file
 
     if path_in.suffix == ".xlsx":
         path_in_user = path_in
@@ -65,7 +68,12 @@ def transport_cmd(context: "Context", method, path_in: Path, path_out: Optional[
     else:
         path_out_user = path_out
 
-    process_file(path_in, path_out, method=method)
+    process_file(
+        path_in,
+        path_out,
+        method=METHOD[method],
+        platform_name=context.core.platform_info.get("name", None),
+    )
 
     if path_out_user != path_out:
         print(f"Convert CSV output to {path_out_user}")