Merge pull request #341 from iiasa/enh/data-paths

Adjust handling of data paths in tests, CI

Author: khaeru

.github/workflows/pytest.yaml

@@ -77,7 +77,7 @@ jobs:
         path: message-static-data
         ssh-key: ${{ secrets.MESSAGE_STATIC_DATA_PRIVATE_KEY }}
         lfs: true
-        # Only check out the following directories, in order to limit bandwith usage:
+        # Only check out the following directories, in order to limit bandwidth usage:
         sparse-checkout: |
           iea/eei
           ssp
@@ -125,8 +125,10 @@ jobs:
     - name: Cache test data
       uses: actions/cache@v4
       with:
-        path: message-local-data
-        key: ${{ matrix.os }}
+        path: |
+          local-data
+          .pytest_cache/d
+        key: local-data-${{ matrix.os }}-upstream-${{ matrix.version.upstream }}
 
     - name: Check out message-ix-models
       uses: francisbilham11/action-cached-lfs-checkout@v3
@@ -140,9 +142,9 @@ jobs:
         key: static-data-${{ matrix.os }}
 
     - name: Set up uv, Python
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v6
       with:
-        cache-dependency-glob: "**/pyproject.toml"
+        activate-environment: true
         python-version: ${{ matrix.version.python }}
 
     - uses: iiasa/actions/setup-gams@main
@@ -195,12 +197,12 @@ jobs:
 
     - name: Configure local data path
       run: |
-        mkdir -p message-local-data/cache
-        mix-models config set "message local data" "$(realpath message-local-data)"
+        mkdir -p local-data
+        mix-models config set "message local data" "$(realpath local-data)"
         mix-models config show
         # Symlink message-static-data into local data path
-        mkdir -p ${{ github.workspace }}/message-local-data
-        cp -rsv $(realpath message-static-data)/* message-local-data/
+        mkdir -p ${{ github.workspace }}/local-data
+        cp -rsv $(realpath message-static-data)/* local-data/
 
     - name: Run test suite using pytest
       run: |
@@ -224,11 +226,11 @@ jobs:
     - uses: actions/checkout@v4
       with: { ref: "${{ needs.check.outputs.ref }}" }
     - uses: astral-sh/setup-uv@v6
-      with: { enable-cache: true, python-version: "${{ env.python-version }}" }
+      with: { python-version: "${{ env.python-version }}" }
     - name: Clear and re-create the pre-commit environments
       run: uvx pre-commit clean
       if: github.event_name == 'schedule'  # Comment this line to force clear
     - run: |
         uvx --with=pre-commit-uv \
           pre-commit run \
-            --all-files --color=always --show-diff-on-failure --verbose
+          --all-files --color=always --show-diff-on-failure --verbose

doc/api/tests.rst

@@ -0,0 +1,12 @@
+Test suite (:mod:`.tests`)
+**************************
+
+The entire test suite is documented here,
+recursively and automatically.
+
+.. autosummary::
+   :toctree: _autosummary
+   :template: autosummary-module.rst
+   :recursive:
+
+   message_ix_models.tests

doc/api/tools.rst

@@ -58,6 +58,7 @@ Exogenous data (:mod:`.tools.exo_data`)
 
 .. autoclass:: ExoDataSource
    :members:
+   :private-members: _where
    :special-members: __init__, __call__
 
 .. currentmodule:: message_ix_models.tools.advance

doc/conf.py

@@ -223,10 +223,12 @@ def local_inv(name: str, *parts: str) -> Optional[str]:
 
 napoleon_preprocess_types = True
 napoleon_type_aliases = {
+    # Python standard library
     "iterable": ":class:`~collections.abc.Iterable`",
     "sequence": ":class:`~collections.abc.Sequence`",
-    "Path": ":class:`~pathlib.Path`",
     "PathLike": ":class:`os.PathLike`",
+    # genno
+    "AnyQuantity": ":data:`~genno.core.quantity.AnyQuantity`",
 }
 
 # -- Options for sphinx.ext.todo -------------------------------------------------------

doc/global/external_index.rst

@@ -1,5 +1,5 @@
 MESSAGE-GLOBIOM
-====
+***************
 
 MESSAGE-GLOBIOM combines the global energy systems model MESSAGE and the
 land~use model GLOBIOM into a consistent Integrated Assessment Modeling (IAM)

doc/howto/unicc.rst

@@ -904,7 +904,7 @@ However, this will show a lot of information, so it might be better to run a mor
    sacct --format=jobid,MaxRSS,MaxVMSize,start,end,CPUTimeRAW,NodeList
 
 Resources to request for reducing MESSAGEix run time
----------------------------------------------------
+----------------------------------------------------
 
 The following information is based on non-scientific "testing" (goofing around), so take it with a grain of salt.
 I have found that requesting more CPUs per task can help reduce the run time of a MESSAGEix solve.

doc/index.rst

@@ -53,9 +53,10 @@ Commonly used classes may be imported directly from :mod:`message_ix_models`.
 - :doc:`api/report/index`
 - :doc:`api/tools`
 - :doc:`api/data-sources`
+- :doc:`api/workflow`
 - :doc:`api/util`
 - :doc:`api/testing`
-- :doc:`api/workflow`
+- :doc:`api/tests`
 
 .. toctree::
    :maxdepth: 2
@@ -73,9 +74,11 @@ Commonly used classes may be imported directly from :mod:`message_ix_models`.
    api/tools-costs
    api/tools-messagev
    api/data-sources
+   api/workflow
    api/util
    api/testing
-   api/workflow
+   api/tests
+
 
 .. toctree::
    :maxdepth: 2

doc/whatsnew.rst

@@ -16,6 +16,8 @@ Next release
   - :py:`transform="C"` / :func:`.transform_C` (:issue:`229`, :pull:`300`).
   - The :class:`~.web.TRANSFORM` enumeration for specifying and validating multiple transformations.
 
+- Improve :class:`.ExoDataSource` with :attr:`~.ExoDataSource.use_test_data`,
+  :attr:`~.ExoDataSource.where`, and :meth:`~.ExoDataSource._where` (:pull:`341`).
 - New class :class:`.Dataflow` for describing input and/or output data flows (:pull:`300`) that are read from file and attached to a :class:`.Computer`.
   Generalized from former :py:`.transport.files.ExogenousDataFile`.
 - New method :meth:`.Config.regions_from_scenario` (:pull:`300`),
@@ -29,8 +31,19 @@ Next release
   - :func:`.check.verbose_check` (:pull:`300`).
 
 - Display entire result quantity in :func:`.report.report` / :program:`mix-models report` with :py:`verbose=True` (:pull:`300`).
+- New test fixtures (:pull:`341`):
+  :func:`.advance_test_data`,
+  :func:`.gea_test_data`,
+  :func:`.gfei_test_data`,
+  :func:`.iea_eei_user_data`,
+  :func:`.iea_eweb_test_data`,
+  :func:`.iea_eweb_user_data`,
+  :func:`.shape_test_data`,
+  :func:`.ssp_test_data`,
+  :func:`.ssp_user_data`.
 - Bug fix: :program:`mix-models --verbose` command-line option was not stored on :class:`.Context`/:class:`~.util.Config` (:pull:`300`).
-- Bug fix: adjust or guard some Python usage that was not compatible with Python 3.9—the earliest version supported by :mod:`message_ix_models` (:pull:`295`, :issue:`294`).
+- Bug fix: adjust or guard some Python usage that was not compatible with Python 3.9
+  —the earliest version supported by :mod:`message_ix_models` (:pull:`295`, :issue:`294`).
 - Drop obsolete :py:`series_of_pint_quantity()` (:pull:`289`).
 
 By topic:
@@ -108,6 +121,8 @@ Documentation
 - Expand the :ref:`costs-usage` section of the :mod:`.tools.costs` documentation to describe the requirement for SSP input data (:issue:`313`, :pull:`322`).
 - Reorganize and improve the :doc:`data` documentation page (:pull:`326`).
 
+.. _v2025.1.10:
+
 v2025.1.10
 ==========
 

message_ix_models/model/transport/data.py

@@ -150,7 +150,7 @@ class MaybeAdaptR11Source(ExoDataSource):
     _adapter: Optional[Callable] = None
 
     def __init__(self, source, source_kw):
-        from .util import path_fallback
+        from .util import region_path_fallback
 
         # Check that the given measure is supported by the current class
         if not source == self.id:
@@ -175,11 +175,11 @@ def __init__(self, source, source_kw):
 
         filename = self.filename[measure]
         try:
-            self.path = path_fallback(nodes, filename)
+            self.path = region_path_fallback(nodes, filename)
             self._repr = f"Load {self.path}"
         except FileNotFoundError:
             log.info(f"Fall back to R11 data for {self.measure}")
-            self.path = path_fallback("R11", filename)
+            self.path = region_path_fallback("R11", filename)
 
             # Identify an adapter that can convert data from R11 to `nodes`
             self._adapter = {"R12": adapt_R11_R12, "R14": adapt_R11_R14}.get(nodes)

message_ix_models/model/transport/emission.py

@@ -11,7 +11,7 @@
 
 from message_ix_models.util import package_data_path
 
-from .util import path_fallback
+from .util import region_path_fallback
 
 if TYPE_CHECKING:
     from genno.types import AnyQuantity
@@ -26,7 +26,7 @@ def get_emissions_data(context: "Context") -> "ParameterData":
     """Load emissions data from a file."""
 
     fn = f"{context.transport.data_source.emissions}-emission_factor.csv"
-    qty = load_file(path_fallback(context, "emi", fn))
+    qty = load_file(region_path_fallback(context, "emi", fn))
 
     return dict(emission_factor=qty.to_dataframe())
 

message_ix_models/model/transport/operator.py

@@ -268,7 +268,7 @@ def broadcast_t_c_l(
 def broadcast_y_yv_ya(
     y: list[int], y_include: list[int], *, method: str = "product"
 ) -> "AnyQuantity":
-    """Return a quantity for broadcasting y to (yv, ya).
+    r"""Return a quantity for broadcasting y to (yv, ya).
 
     This omits all :math:`y^V \notin y^{include}`.
 

message_ix_models/model/transport/structure.py

@@ -11,7 +11,7 @@
 from message_ix_models.model.structure import generate_set_elements, get_region_codes
 from message_ix_models.util import load_package_data, package_data_path
 
-from .util import path_fallback
+from .util import region_path_fallback
 
 #: Template for disutility technologies.
 TEMPLATE = Code(
@@ -80,7 +80,7 @@ def make_spec(regions: str) -> Spec:
 
         # Load and store the data from the YAML file: either in a subdirectory for
         # context.model.regions, or the top-level data directory
-        path = path_fallback(regions, fn).relative_to(package_data_path())
+        path = region_path_fallback(regions, fn).relative_to(package_data_path())
         tmp[name] = load_package_data(*path.parts)
 
     # Merge contents of technology.yaml into set.yaml

message_ix_models/model/transport/testing.py

@@ -47,6 +47,7 @@
             reason="Graphviz missing on macos-13 GitHub Actions runners",
         ),
         9: pytest.mark.xfail(reason="Missing R14 input data/config"),
+        10: pytest.mark.usefixtures("iea_eweb_test_data", "ssp_user_data"),
         "gh-288": pytest.mark.xfail(
             reason="Temporary, for https://github.com/iiasa/message-ix-models/pull/288",
         ),

message_ix_models/model/transport/util.py

@@ -32,7 +32,7 @@ def has_input_commodity(technology: "Code", commodity: str) -> bool:
         return False
 
 
-def path_fallback(context_or_regions: Union[Context, str], *parts) -> Path:
+def region_path_fallback(context_or_regions: Union[Context, str], *parts) -> Path:
     """Return a :class:`.Path` constructed from `parts`.
 
     If ``context.model.regions`` (or a string value as the first argument) is defined

message_ix_models/project/advance/data.py

@@ -5,12 +5,7 @@
 
 from message_ix_models.tools.exo_data import ExoDataSource, register_source
 from message_ix_models.tools.iamc import iamc_like_data_for_query
-from message_ix_models.util import (
-    HAS_MESSAGE_DATA,
-    iter_keys,
-    package_data_path,
-    private_data_path,
-)
+from message_ix_models.util import iter_keys, path_fallback
 
 __all__ = [
     "ADVANCE",
@@ -89,6 +84,8 @@ class ADVANCE(ExoDataSource):
 
     id = "ADVANCE"
 
+    where = ["private"]
+
     def __init__(self, source, source_kw):
         if not source == self.id:
             raise ValueError(source)
@@ -125,12 +122,7 @@ def __call__(self):
         log.debug(query)
 
         # Expected location of the ADVANCE WP2 data snapshot.
-        parts = LOCATION
-        if HAS_MESSAGE_DATA:
-            path = private_data_path(*parts)
-        else:
-            path = package_data_path("test", *parts)
-            log.warning(f"Reading random data from {path}")
+        path = path_fallback(*LOCATION, where=self._where())
 
         return iamc_like_data_for_query(
             path, query, archive_member=NAME, non_iso_3166="keep"

message_ix_models/project/gea/data.py

@@ -39,6 +39,8 @@ class GEA(ExoDataSource):
     #: By default, do not interpolate.
     interpolate = False
 
+    where = ["private"]
+
     def __init__(self, source, source_kw):
         if source != self.id:
             raise ValueError(source)
@@ -58,10 +60,8 @@ def __init__(self, source, source_kw):
 
         # Identify input data path
         self.path = path_fallback(
-            "gea", "GEADB_ARCHIVE_20171108.zip", where="private test"
+            "gea", "GEADB_ARCHIVE_20171108.zip", where=self._where()
         )
-        if "test" in self.path.parts:
-            log.warning(f"Reading random data from {self.path}")
 
         # Assemble query
         self.query = " and ".join(

message_ix_models/project/shape/data.py

@@ -65,6 +65,8 @@ class SHAPE(ExoDataSource):
 
     id = "SHAPE"
 
+    where = ["private"]
+
     def __init__(self, source, source_kw):
         if source != self.id:
             raise ValueError(source)
@@ -85,13 +87,8 @@ def __init__(self, source, source_kw):
         version = version.replace("latest", info["latest"])
 
         # Construct path to data file
-        self.path = path_fallback(
-            "shape",
-            f"{self.measure}_v{version.replace('.', 'p')}{info['suffix']}",
-            where="private test",
-        )
-        if "test" in self.path.parts:
-            log.warning(f"Reading random data from {self.path}")
+        filename = f"{self.measure}_v{version.replace('.', 'p')}{info['suffix']}"
+        self.path = path_fallback("shape", filename, where=self._where())
 
         variable = info.get("variable", self.measure)
         self.query = " and ".join(