Merge pull request #326 from iiasa/project/ssp/static-data

Adjust usage of message-static-data for IEA, SSP inputs

Author: khaeru

.github/workflows/pytest.yaml

@@ -110,6 +110,23 @@ jobs:
       with:
         ref: ${{ needs.check.outputs.ref }}
 
+    - uses: webfactory/[email protected]
+      with:
+        ssh-private-key: |
+          ${{ secrets.MESSAGE_STATIC_DATA_PRIVATE_KEY }}
+
+    - name: Check out message-static-data
+      uses: actions/checkout@v4
+      with:
+        repository: iiasa/message-static-data
+        path: message-static-data
+        ssh-key: ${{ secrets.MESSAGE_STATIC_DATA_PRIVATE_KEY }}
+        lfs: true
+        # Only check out the following directories, in order to limit bandwith usage:
+        sparse-checkout: |
+          iea/eei
+          ssp
+
     - name: Set up uv, Python
       uses: astral-sh/setup-uv@v5
       with:
@@ -167,6 +184,9 @@ jobs:
         mkdir -p message-local-data/cache
         mix-models config set "message local data" "$(realpath message-local-data)"
         mix-models config show
+        # Symlink message-static-data into local data path
+        mkdir -p ${{ github.workspace }}/message-local-data
+        cp -rsv $(realpath message-static-data)/* message-local-data/
 
     - name: Run test suite using pytest
       run: |

.gitignore

@@ -151,9 +151,5 @@ cache/
 # Temporary Excel files
 *~$*
 
-# SSP related files (not ready for public)
-SSP-Review-Phase-1.xlsx
-message_ix_models/data/ssp/*
-
 # Scratch files
-*scratch*
+*scratch*

MANIFEST.in

@@ -6,14 +6,15 @@ prune .github
 prune message_ix_models/data/test/advance
 prune message_ix_models/data/test/gea
 prune message_ix_models/data/test/iea
+prune message_ix_models/data/test/report
 prune message_ix_models/data/test/shape
 prune message_ix_models/data/test/snapshot-*
 prune message_ix_models/data/test/ssp
+prune message_ix_models/data/test/transport
 
 # Larger package data
 # - Not distributed on PyPI.
 # - Should be fetched with Pooch from GitHub.
-exclude message_ix_models/data/ssp/*.gz
 prune message_ix_models/data/water/*
 prune message_ix_models/data/material/*
 exclude message_ix_models/data/water/*.tar.xz

doc/api/util.rst

@@ -82,7 +82,6 @@ Commonly used:
 
 .. automodule:: message_ix_models.util.config
    :members:
-   :exclude-members: Config
 
 :mod:`.util.context`
 ====================

doc/data.rst

@@ -17,6 +17,7 @@ Each HOWTO is contained on its own page:
 
    Run a baseline model <quickstart>
    Run mix-models on UnICC <unicc>
+   path
    migrate
    Release message-ix-models <release>
 

doc/howto/index.rst

@@ -0,0 +1,57 @@
+Work with paths to files and data
+*********************************
+
+This HOWTO contains some code examples that may help with following the requirements on :doc:`data </data>`.
+
+.. contents::
+   :local:
+
+.. _howto-static-to-local:
+
+Connect static data to local data
+=================================
+
+Use the :program:`-rs` options to the :program:`cp` command:
+
+.. code-block:: shell
+
+   git clone [email protected]:iiasa/message-static-data.git
+   cd /path/to/message-local-data
+   cp -rsv /path/to/message-static-data ./
+
+This recursively creates subdirectories in the :ref:`local data <local-data>` directory that mirror those existing in :ref:`message-static-data <static-data>`,
+and creates a symlink to every file in every directory.
+Code that looks within the local data directory will then be able to locate these files.
+
+If needed, delete the directories in message-local-data and repeat the :program:`cp` call to recreate.
+
+Identify the cache path used on the current system
+==================================================
+
+.. code-block:: python
+
+   from message_ix_models.util.config import Config
+
+   cfg = Config()
+
+   print(cfg.cache_path)
+
+Identify the cache path without a Context
+=========================================
+
+Internal code that cannot access a :class:`~.util.config.Config` or :class:`.Context` instance
+**should** instead use :func:`platformdirs.user_cache_path` directly:
+
+.. code-block:: python
+
+   from platformdirs import user_cache_path
+
+   # Always use "message-ix-models" as the `appname` parameter
+   ucp = user_cache_path("message-ix-models")
+
+   # Construct the sub-directory for the current module
+   dir_ = ucp.joinpath("my-project", "subdir")
+   dir_.mkdir(parents=True, exist_ok=True)
+
+   # Construct a file path within this directory
+   p = dir_.joinpath("data-file-name.csv")

doc/howto/path.rst

@@ -86,7 +86,7 @@ Update :doc:`/transport/index` (:pull:`259`, :pull:`289`, :pull:`300`).
 Documentation
 -------------
 
-- New :doc:`/howto/index` documentation sub-tree (:pull:`291`).
+- New :doc:`/howto/index` documentation sub-tree (:pull:`291`, :pull:`326`).
 - New guide on HOWTO :doc:`/howto/unicc` (:pull:`279`) and supporting command :program:`mix-models sbatch` in :mod:`.util.slurm` (:pull:`291`).
 - New summary pages for projects
   :doc:`project/alps`,
@@ -105,6 +105,7 @@ Documentation
   :doc:`project/sparccle`, and
   :doc:`project/uptake` (:pull:`282`, :pull:`312`).
 - Expand the :ref:`costs-usage` section of the :mod:`.tools.costs` documentation to describe the requirement for SSP input data (:issue:`313`, :pull:`322`).
+- Reorganize and improve the :doc:`data` documentation page (:pull:`326`).
 
 v2025.1.10
 ==========

doc/whatsnew.rst

@@ -1,10 +1,8 @@
 import logging
 
-from platformdirs import user_cache_path
-
 from message_ix_models.tools.exo_data import ExoDataSource, register_source
 from message_ix_models.tools.iamc import iamc_like_data_for_query
-from message_ix_models.util import package_data_path, private_data_path
+from message_ix_models.util import path_fallback
 
 __all__ = [
     "SSPOriginal",
@@ -13,6 +11,18 @@
 
 log = logging.getLogger(__name__)
 
+#: :py:`where` argument to :func:`path_fallback`, used by both :class:`.SSPOriginal` and
+#: :class:`.SSPUpdate`. In order:
+#:
+#: 1. Currently data is stored in message-static-data, cloned and linked from within the
+#:    user's 'local' data directory.
+#: 2. Previously some files were stored directly within message_ix_models (available in
+#:    an editable install from a clone of the git repository, 'package') or in
+#:    :mod:`message_data` ('private'). These settings are only provided for backward
+#:    compatibility.
+#: 3. If the above are not available, use the fuzzed/random test data ('test').
+WHERE = "local package private test"
+
 
 @register_source
 class SSPOriginal(ExoDataSource):
@@ -93,25 +103,17 @@ def __init__(self, source, source_kw):
 
         self.raise_on_extra_kw(source_kw)
 
+        # Identify input data path
+        self.path = path_fallback("ssp", self.filename, where=WHERE)
+        if "test" in self.path.parts:
+            log.warning(f"Read random data from {self.path}")
+
         # Assemble a query string
         extra = "d" if ssp_id == "4" and model == "IIASA-WiC POP" else ""
         self.query = (
             f"SCENARIO == 'SSP{ssp_id}{extra}_v9_{date}' and VARIABLE == '{measure}'"
             + (f" and MODEL == '{model}'" if model else "")
         )
-        # log.debug(query)
-
-        # Iterate over possible locations for the data file
-        dirs = [private_data_path("ssp"), package_data_path("test", "ssp")]
-        for path in [d.joinpath(self.filename) for d in dirs]:
-            if not path.exists():
-                log.info(f"Not found: {path}")
-                continue
-            if "test" in path.parts:
-                log.warning(f"Reading random data from {path}")
-            break
-
-        self.path = path
 
     def __call__(self):
         # Use prepared path, query, and replacements
@@ -132,7 +134,7 @@ class SSPUpdate(ExoDataSource):
 
        - `source`: Any value from :data:`.SSP_2024` or equivalent string, for instance
          "ICONICS:SSP(2024).2".
-       - `release`: One of "3.0.1", "3.0", or "preview".
+       - `release`: One of "3.1", "3.0.1", "3.0", or "preview".
 
     Example
     -------
@@ -151,6 +153,7 @@ class SSPUpdate(ExoDataSource):
     filename = {
         "3.0": "1706548837040-ssp_basic_drivers_release_3.0_full.csv.gz",
         "3.0.1": "1710759470883-ssp_basic_drivers_release_3.0.1_full.csv.gz",
+        "3.1": "1721734326790-ssp_basic_drivers_release_3.1_full.csv.gz",
         "preview": "SSP-Review-Phase-1.csv.gz",
     }
 
@@ -183,13 +186,7 @@ def __init__(self, source, source_kw):
         models = []
         scenarios = []
 
-        if release in ("3.0.1", "3.0"):
-            # Directories in which to locate `self.filename`:
-            # - User's local cache (retrieved with "mix-models fetch" or equivalent).
-            # - Stored directly within message_ix_models (editable install from a clone
-            #   of the git repository).
-            dirs = [user_cache_path("message-ix-models"), package_data_path("ssp")]
-
+        if release in ("3.1", "3.0.1", "3.0"):
             scenarios.append(f"SSP{ssp_id}")
 
             if measure == "GDP|PPP":
@@ -202,9 +199,6 @@ def __init__(self, source, source_kw):
                     Scenario={"Historical Reference": scenarios[0]},
                 )
         elif release == "preview":
-            # Look first in message_data, then in message_ix_models test data
-            dirs = [private_data_path("ssp"), package_data_path("test", "ssp")]
-
             models.extend([model] if model is not None else [])
             scenarios.append(f"SSP{ssp_id} - Review Phase 1")
         else:
@@ -214,22 +208,15 @@ def __init__(self, source, source_kw):
             )
             raise ValueError(release)
 
+        # Identify input data path
+        self.path = path_fallback("ssp", self.filename[release], where=WHERE)
+        if "test" in self.path.parts:
+            log.warning(f"Read random data from {self.path}")
+
         # Assemble and store a query string
         self.query = f"Scenario in {scenarios!r} and Variable == '{measure}'" + (
             f"and Model in {models!r}" if models else ""
         )
-        # log.info(f"{self.query = }")
-
-        # Iterate over possible locations for the data file
-        for path in [d.joinpath(self.filename[release]) for d in dirs]:
-            if not path.exists():
-                log.info(f"Not found: {path}")
-                continue
-            if "test" in path.parts:
-                log.warning(f"Reading random data from {path}")
-            break
-
-        self.path = path
 
     def __call__(self):
         # Use prepared path, query, and replacements

message_ix_models/data/ssp/1706548837040-ssp_basic_drivers_release_3.0_full.csv.gz

@@ -142,10 +142,10 @@ def session_context(pytestconfig, tmp_env):
     )
 
     # Store current .util.config.Config.local_data setting from the user's configuration
-    pytestconfig.user_local_data = ctx.core.local_data
+    uld = pytestconfig.user_local_data = ctx.core.local_data
 
     # Other local data in the temporary directory for this session only
-    ctx.core.local_data = session_tmp_dir
+    sld = ctx.core.local_data = session_tmp_dir
 
     # Also set the "message local data" key in the ixmp config
     ixmp_config.set("message local data", session_tmp_dir)
@@ -158,6 +158,17 @@ def session_context(pytestconfig, tmp_env):
         # Create some subdirectories
         util.MESSAGE_DATA_PATH.joinpath("data", "tests").mkdir(parents=True)
 
+    # Symlink some paths from the user's local data into parallel subpaths of the test
+    # local data directory
+    for parts in (
+        ("iea",),
+        ("ssp",),
+    ):
+        target = uld.joinpath(*parts)
+        path = sld.joinpath(*parts)
+        log.info(f"Symlink {path} -> {target}")
+        path.symlink_to(target)
+
     # Add a platform connected to an in-memory database
     platform_name = "message-ix-models"
     ixmp_config.add_platform(

message_ix_models/data/ssp/1710759470883-ssp_basic_drivers_release_3.0.1_full.csv.gz

@@ -11,7 +11,6 @@
     process_file,
 )
 from message_ix_models.testing import MARK
-from message_ix_models.tests.tools.iea.test_web import user_local_data  # noqa: F401
 from message_ix_models.tools.iea import web
 from message_ix_models.util import package_data_path
 
@@ -105,28 +104,25 @@ def _to_long(df):
     )
 
     # Identify the directory from which IEA EWEB data is read
-    iea_data_dir = web.dir_fallback(web.FILES[("IEA", "2024")][0])
-    # True if the full data set is present; False if the fuzzed test data are being used
-    full_iea_eweb_data = not (
-        iea_data_dir.parts[-4:] == ("message_ix_models", "data", "test", "iea")
-    )
+    iea_eweb_dir = web.dir_fallback(web.FILES[("IEA", "2024")][0])
+    # True if the fuzzed test data are being used
+    iea_eweb_test_data = iea_eweb_dir.match("message_ix_models/data/test/iea/web")
 
     # Number of modified values
     N_exp = {
-        (METHOD.A, True): 10280,
         (METHOD.A, False): 10280,
-        (METHOD.B, True): 4660,
-        (METHOD.B, False): 3060,
-        (METHOD.C, True): 3220,
+        (METHOD.A, True): 10280,
+        (METHOD.B, False): 4660,
+        (METHOD.B, True): 3060,
         (METHOD.C, False): 3220,
-    }[(method, full_iea_eweb_data)]
+        (METHOD.C, True): 3220,
+    }[(method, iea_eweb_test_data)]
 
     if N_exp != len(df):
         # df.to_csv("debug-diff.csv")  # DEBUG Dump to file
         # print(df.to_string(max_rows=50))  # DEBUG Show in test output
-        assert N_exp == len(df), (
-            f"Unexpected number of modified values: {len(df)} != {N_exp}"
-        )
+        msg = f"Unexpected number of modified values: {N_exp} != {len(df)}"
+        assert N_exp == len(df)
 
     # All of the expected 'variable' codes have been modified
     assert expected_variables(OUT, method) == set(df["Variable"].unique())
@@ -135,7 +131,7 @@ def _to_long(df):
     if len(cond):
         msg = "Negative emissions totals after processing"
         print(f"\n{msg}:", cond.to_string(), sep="\n")
-        assert not full_iea_eweb_data, msg
+        assert iea_eweb_test_data, msg  # Negative values → fail if NOT using test data
 
 
 def expected_variables(flag: int, method: METHOD) -> set[str]:
@@ -220,7 +216,6 @@ def test_get_scenario_code(expected_id, model_name, scenario_name) -> None:
 
 
 @get_computer.minimum_version
-# @pytest.mark.usefixtures("user_local_data")
 @pytest.mark.parametrize("method", METHOD_PARAM)
 def test_process_df(test_context, input_csv_path, method) -> None:
     df_in = pd.read_csv(input_csv_path)
@@ -233,7 +228,6 @@ def test_process_df(test_context, input_csv_path, method) -> None:
 
 
 @get_computer.minimum_version
-# @pytest.mark.usefixtures("user_local_data")
 @pytest.mark.parametrize("method", METHOD_PARAM)
 def test_process_file(tmp_path, test_context, input_csv_path, method) -> None:
     """Code can be called from Python."""