Transport improvements from 2025-W05

.github/workflows/pytest.yaml

@@ -75,8 +75,7 @@ jobs:
         # - Latest supported Python version for those or other dependencies.
         # Minimum version given in pyproject.toml + earlier version of Python
         # For this job only, the oldest version of Python supported by message-ix-models
-        - { upstream: v3.6.0, python: "3.9" }  # Released 2022-08-18
-        - { upstream: v3.7.0, python: "3.11" }  # 2023-05-17
+        - { upstream: v3.7.0, python: "3.9" }  # Released 2023-05-17
         - { upstream: v3.8.0, python: "3.12" }  # 2024-01-12
         # Latest released version + latest released Python
         - { upstream: v3.9.0, python: "3.13" }  # 2024-06-04
@@ -86,7 +85,6 @@ jobs:
         exclude:
         # Specific version combinations that are invalid / not to be used
         # These versions of ixmp are not able locate the arm64 GAMS API binaries
-        - { os: macos-latest, version: { upstream: v3.6.0 }}
         - { os: macos-latest, version: { upstream: v3.7.0 }}
         - { os: macos-latest, version: { upstream: v3.8.0 }}
         - { os: macos-latest, version: { upstream: v3.9.0 }}
@@ -133,9 +131,6 @@ jobs:
 
         v, result = "${{ matrix.version.upstream }}".replace("main", "vmain"), []
         for condition, dependency in (
-            (v <= "v3.6.0", "dask < 2024.3.0"),  # dask[dataframe] >= 2024.3.0 requires dask-expr and in turn pandas >= 2.0 (#156)
-            (v <= "v3.6.0", "numpy < 2.0"),
-            (v <= "v3.6.0", "pandas < 2.0"),
             (v >= "v3.7.0", "dask[dataframe] < 2024.11.0"),  # dask >= 2024.11.0 changes handling of dict (will be addressed in #225)
             (v <= "v3.7.0", "genno < 1.25"),  # Upstream versions < 3.8.0 import genno.computations, removed in 1.25.0 (#156)
             (v < "v3.9.0", "pytest == 8.0.0"),  # Upstream versions < 3.9.0 use a hook argument removed in pytest 8.1.0 (#155)

.github/workflows/transport.yaml

@@ -7,15 +7,46 @@ env:
   target-workflow: transport
 
   # Starting point of the workflow.
+  #
   # Use this value to build from a certain scenario:
   # base: --url="ixmp://ixmp-dev/MESSAGEix-GLOBIOM 1.1-R12/baseline_DEFAULT#21"
+  #
   # Use this value to allow the workflow to determine model & scenario names
   # and versions:
   base: --platform=ixmp-dev
 
-  # Set this to a particular step to truncate the workflow
+  # Set this to a particular step to truncate the workflow.
   from-step: ""
 
+  # Workflow steps/labels to run. These correspond to the 'TARGET' argument to
+  # 'mix-models transport run'. Each label triggers 1 job in the target-repo/
+  # target-workflow.
+  #
+  # - Delete lines to disable some runs.
+  # - Ensure there is NO trailing comma on the last line.
+  labels: >-
+    [
+      "SSP1",
+      "SSP1 policy",
+      "SSP2",
+      "SSP2 policy",
+      "SSP3",
+      "SSP3 policy",
+      "SSP4",
+      "SSP4 policy",
+      "SSP5",
+      "SSP5 policy",
+      "EDITS-CA",
+      "EDITS-HA",
+      "LED-SSP1",
+      "LED-SSP2"
+    ]
+
+  # Currently disabled:
+  # [
+  #
+  # ]
+
 on:
   # Uncomment these lines for debugging, but leave them commented on 'main'
   # pull_request:
@@ -36,22 +67,28 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: Invoke "${{ env.target-workflow }}" workflow in ${{ env.target-repo }}
-        env:
-          GH_TOKEN: ${{ secrets.MESSAGE_DATA_DISPATCH_TOKEN }}
-        run: |
-          gh workflow run \
-            ${{ env.target-workflow }}.yaml \
-            --repo=${{ env.target-repo }} \
-            --ref=${{ env.target-ref }} \
-            --field ref=${{ github.ref }} \
-            --field sha=${{ github.sha }} \
-            --field base=${{ env.base }} \
-            --field from-step=${{ env.from-step }}
-
-          sleep 5
-
-          gh run list \
-            --workflow=${{ env.target-workflow.yaml }} --repo=${{ env.target-repo }} \
-            --json url,status \
-            --jq 'map(select(.status != "completed"))[0].url' >> $GITHUB_STEP_SUMMARY
+    - name: Assemble JSON payload
+      run: |
+        echo '{
+          "ref": "${{ github.ref }}",
+          "sha": "${{ github.sha }}",
+          "base": "${{ env.base }}",
+          "from-step": "${{ env.from-step }}",
+          "labels": ${{ env.labels }}
+        }' | jq -r -c '.labels = (.labels | tostring)' | tee payload.json
+
+    - name: Invoke "${{ env.target-workflow }}" workflow in ${{ env.target-repo }}
+      env:
+        GH_TOKEN: ${{ secrets.MESSAGE_DATA_DISPATCH_TOKEN }}
+      run: |
+        cat payload.json | gh workflow run --json \
+          ${{ env.target-workflow }}.yaml \
+          --repo=${{ env.target-repo }} \
+          --ref=${{ env.target-ref }}
+
+        sleep 5
+
+        gh run list \
+          --workflow=${{ env.target-workflow.yaml }} --repo=${{ env.target-repo }} \
+          --json url,status \
+          --jq 'map(select(.status != "completed"))[0].url' >>$GITHUB_STEP_SUMMARY

doc/api/report/index.rst

@@ -172,7 +172,6 @@ Utilities
    .. autosummary::
 
       add_replacements
-      as_quantity
       collapse
       collapse_gwp_info
       copy_ts

doc/api/util.rst

@@ -54,7 +54,6 @@ Commonly used:
    private_data_path
    same_node
    same_time
-   series_of_pint_quantity
    show_versions
 
 .. automodule:: message_ix_models.util

doc/conf.py

@@ -75,6 +75,7 @@
 .. role:: underline
 
 .. |c| replace:: :math:`c`
+.. |l| replace:: :math:`l`
 .. |n| replace:: :math:`n`
 .. |t| replace:: :math:`t`
 .. |y| replace:: :math:`y`

doc/transport/input.rst

@@ -204,6 +204,8 @@ node = R12_SAS [2]_
 .. [2] A. Javaid, `message_data#180 (comment) <https://github.com/iiasa/message_data/issues/180#issuecomment-1941860412>`_.
 .. [3] A. Javaid, `message_data#538 (comment) <https://github.com/iiasa/message_data/issues/538#issuecomment-1934663340>`__.
 
+.. _transport-pdt-cap-proj:
+
 :file:`pdt-cap.csv` → ``P activity:scenario-n-t-y:exo``
 -------------------------------------------------------
 
@@ -221,7 +223,6 @@ PDT per capita.
 
   .. todo:: Transcribe the method into this document.
 
-
 :file:`pdt-cap-ref.csv` → ``pdt:n:capita+ref``
 ----------------------------------------------
 

doc/whatsnew.rst

@@ -4,11 +4,12 @@ What's new
 Next release
 ============
 
-- Support for :mod:`ixmp` and :mod:`message_ix` versions 3.4.x and 3.5.x is dropped in accordance with the :ref:`policy-upstream-versions` (:pull:`288`).
-  The minimum version of both packages is 3.6.0.
+- Support for :mod:`ixmp` and :mod:`message_ix` versions 3.4, 3.5, and 3.6 is dropped in accordance with the :ref:`policy-upstream-versions` (:pull:`288`, :pull:`289`).
+  The minimum version of both packages is 3.7.0.
 - Update :class:`.IEA_EWEB` to support :py:`transform="B"` / :func:`.transform_B` (:issue:`230`, :pull:`259`).
 - Add :func:`.prepare_method_B` to :mod:`.ssp.transport` (:pull:`259`).
 - New utility :class:`.sdmx.AnnotationsMixIn` (:pull:`259`).
+- Drop obsolete :py:`series_of_pint_quantity()` (:pull:`289`).
 
 By topic:
 
@@ -19,13 +20,22 @@ By topic:
 Transport
 ---------
 
-Update :doc:`/transport/index` (:pull:`259`).
+Update :doc:`/transport/index` (:pull:`259`, :pull:`289`).
 
 - Adjust constraints on :py:`t="conm_ar"`.
 - Recompute :attr:`.minimum_activity` for transport technologies.
-- Adjust freight activity, freight and passenger mode shares for some regions.
+- Improve freight representation:
+
+  - Adjust freight activity, freight and passenger mode shares for some regions.
+  - Add dynamic constraints on activity of freight technologies.
+  - Fix alignment of freight technology outputs with demand |l|.
+
+- Implement LED override using exogenous passenger activity data from :ref:`transport-pdt-cap-proj`.
 - Drop :file:`base-scenario-url.json`; store base scenario URLs in :ref:`CL_TRANSPORT_SCENARIO`.
+- Generate SDMX-ML structural metadata, including data flow definitions, and SDMX-{CSV,ML} data outputs for certain reported quantities.
+- Expand use of fixed/shared keys from :mod:`.transport.key`.
 - Simplify and consolidate tests.
+- Improve :func:`.simulated_solution` to load ‘simulated’ solution data from file to reduce test durations.
 
 Documentation
 -------------
@@ -49,6 +59,8 @@ Documentation
   :doc:`project/sparccle`, and
   :doc:`project/uptake` (:pull:`282`).
 
+- New utility :class:`.sdmx.AnnotationsMixIn` (:pull:`259`).
+- Bug fix: adjust or guard some Python usage that was not compatible with Python 3.9—the earliest version supported by :mod:`message_ix_models` (:pull:`295`, :issue:`294`).
 
 v2025.1.10
 ==========
@@ -443,12 +455,12 @@ Earlier releases
 2021.7.27
 ---------
 
-- Improve caching using  mod:`genno` v1.8.0 (:pull:`29`).
+- Improve caching using :mod:`genno` v1.8.0 (:pull:`29`).
 
 2021.7.22
 ---------
 
-- Migrate utilities :func:`.cached`, :func:`.check_support`, :func:`.convert_units`, :func:`.maybe_query`, :func:`.series_of_pint_quantity` (:pull:`27`)
+- Migrate utilities :func:`.cached`, :func:`.check_support`, :func:`.convert_units`, :func:`.maybe_query`, :py:`series_of_pint_quantity()` (:pull:`27`)
 - Add :data:`.testing.NIE`.
 - Add the ``--jvmargs`` option to :command:`pytest` (see :func:`.pytest_addoption`).
 - Remove :py:`.Context.get_config_file()`, :py:`.get_path()`, :py:`.load_config()`, and :py:`.units`, all deprecated since 2021-02-28.

message_ix_models/data/sdmx/IIASA_ECE_CL_TRANSPORT_SCENARIO(1.0.0).xml

@@ -3,13 +3,13 @@
   <mes:Header>
     <mes:ID>none</mes:ID>
     <mes:Test>false</mes:Test>
-    <mes:Prepared>2025-01-15T10:37:47.926181</mes:Prepared>
+    <mes:Prepared>2025-02-20T10:44:16.460424</mes:Prepared>
     <mes:Sender id="none"/>
-    <mes:Source xml:lang="en">Generated by message_ix_models 2025.1.11.dev1+gabce19674.d20250113</mes:Source>
+    <mes:Source xml:lang="en">Generated by message_ix_models 2025.1.11.dev97+gdf7a6cee.d20250211</mes:Source>
   </mes:Header>
   <mes:Structures>
     <str:Codelists>
-      <str:Codelist isExternalReference="none" isFinal="none" agencyID="IIASA_ECE" version="1.0.0" id="CL_TRANSPORT_SCENARIO" urn="urn:sdmx:org.sdmx.infomodel.codelist.Codelist=IIASA_ECE:CL_TRANSPORT_SCENARIO(1.0.0)">
+      <str:Codelist isExternalReference="false" isFinal="false" agencyID="IIASA_ECE" version="1.0.0" id="CL_TRANSPORT_SCENARIO" urn="urn:sdmx:org.sdmx.infomodel.codelist.Codelist=IIASA_ECE:CL_TRANSPORT_SCENARIO(1.0.0)">
         <str:Code id="SSP1">
           <com:Annotations>
             <com:Annotation id="SSP-URN">
@@ -22,7 +22,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP1_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP1_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
         </str:Code>
@@ -38,7 +38,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
         </str:Code>
@@ -54,7 +54,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP3_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP3_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
         </str:Code>
@@ -70,7 +70,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP4_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP4_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
         </str:Code>
@@ -86,7 +86,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP5_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP5_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
         </str:Code>
@@ -102,7 +102,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP1_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP1_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
           <com:Name xml:lang="en">Low Energy Demand/High-with-Low scenario with SSP1 demographics</com:Name>
@@ -119,7 +119,7 @@
               <com:AnnotationText xml:lang="en">None</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
           <com:Name xml:lang="en">Low Energy Demand/High-with-Low scenario with SSP2 demographics</com:Name>
@@ -136,7 +136,7 @@
               <com:AnnotationText xml:lang="en">'CA'</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
           <com:Name xml:lang="en">EDITS scenario with ITF PASTA 'CA' activity</com:Name>
@@ -153,7 +153,7 @@
               <com:AnnotationText xml:lang="en">'HA'</com:AnnotationText>
             </com:Annotation>
             <com:Annotation id="base-scenario-URL">
-              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v1.1/baseline_DEFAULT_step_13'</com:AnnotationText>
+              <com:AnnotationText xml:lang="en">'ixmp://ixmp-dev/SSP_SSP2_v2.1/baseline_DEFAULT_step_13'</com:AnnotationText>
             </com:Annotation>
           </com:Annotations>
           <com:Name xml:lang="en">EDITS scenario with ITF PASTA 'HA' activity</com:Name>

message_ix_models/data/transport/R12/pdt-cap.csv

@@ -1,6 +1,6 @@
 # Projected passenger-distance travelled (PDT) per capita
 #
-# Units: km / passenger
+# Units: km / year
 #
 scenario, node,     technology, year, value
 LED,      R12_AFR,  2W,         2020,   360

message_ix_models/data/transport/set.yaml

@@ -168,7 +168,8 @@ commodity:
     "transport vehicle {technology.id}":
       _generate: {technology: LDV}
       name: Activity for {technology.id} LDVs
-      units: Gv km
+      # FIXME This is necessary to avoid imputed dimensionless in <output::LDV+ixmp>
+      units: (Gv km)**2
 
     "transport pax {consumer_group.id}":
       _generate: {consumer_group: null}