Fix datapoints retrieve_latest with missing instance_id + ignore=True (#2120)

Author: haakonvt

CHANGELOG.md

@@ -22,6 +22,11 @@ Changes are grouped as follows
 - Support for the `/simulators/models` and `/simulators/models/revisions` API endpoints.
 - Support for the `/simulators` and `/simulators/integration` API endpoints.
 
+## [7.73.4] - 2025-02-24
+### Fixed
+- An issue with `DatapointsAPI.retrieve_latest` and usage of `instance_id` when using `ignore_unknown_ids=True`
+  and *any type* of identifier was not found (no longer raises `TypeError`).
+
 ## [7.73.3] - 2025-02-07
 ### Added
 - Listable property types for containers in Data Modeling now accept `max_list_size`.

cognite/client/_api/datapoints.py

@@ -2107,14 +2107,18 @@ def _post_fix_status_codes_and_stringified_floats(self, result: list[dict[str, A
             ids_exists = (
                 {("id", r["id"]) for r in result}
                 .union({("xid", r.get("externalId")) for r in result})
-                .union({("inst_id", r.get("instanceId")) for r in result})
+                .union({("inst_id", NodeId.load_if(r.get("instanceId"))) for r in result})
                 .difference({("xid", None), ("inst_id", None)})
             )  # fmt: skip
             self._all_identifiers = [
                 query
                 for query in self._all_identifiers
                 if ids_exists.intersection(
-                    (("id", query.get("id")), ("xid", query.get("externalId")), ("inst_id", query.get("instanceId")))
+                    (
+                        ("id", query.get("id")),
+                        ("xid", query.get("externalId")),
+                        ("inst_id", NodeId.load_if(query.get("instanceId"))),
+                    )
                 )
             ]
         for query, res in zip(self._all_identifiers, result):

cognite/client/_api/simulators/integrations.py

@@ -76,18 +76,22 @@ def __call__(
 
     def list(
         self,
-        limit: int = DEFAULT_LIMIT_READ,
+        limit: int | None = DEFAULT_LIMIT_READ,
         filter: SimulatorIntegrationFilter | None = None,
     ) -> SimulatorIntegrationList:
         """`Filter simulator integrations <https://developer.cognite.com/api#tag/Simulator-Integrations/operation/filter_simulator_integrations_simulators_integrations_list_post>`_
         Retrieves a list of simulator integrations that match the given criteria
+
         Args:
-            limit (int): The maximum number of simulator integrations to return.
+            limit (int | None): The maximum number of simulator integrations to return, pass None to return all.
             filter (SimulatorIntegrationFilter | None): Filter to apply.
+
         Returns:
             SimulatorIntegrationList: List of simulator integrations
+
         Examples:
-            List simulator integrations:
+
+            List a few simulator integrations:
                 >>> from cognite.client import CogniteClient
                 >>> client = CogniteClient()
                 >>> res = client.simulators.integrations.list()
@@ -97,7 +101,6 @@ def list(
                 >>> res = client.simulators.integrations.list(
                 ...     filter=SimulatorIntegrationFilter(active=True))
         """
-
         self._warning.warn()
         return self._list(
             method="POST",

cognite/client/_version.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
 
-__version__ = "7.73.3"
+__version__ = "7.73.4"
 
 __api_subversion__ = "20230101"

cognite/client/data_classes/simulators/filters.py

@@ -3,23 +3,24 @@
 from collections.abc import Sequence
 
 from cognite.client.data_classes._base import CogniteFilter
-from cognite.client.utils.useful_types import SequenceNotStr
 
 
 class SimulatorIntegrationFilter(CogniteFilter):
     def __init__(
         self,
-        simulator_external_ids: SequenceNotStr[str] | None = None,
+        simulator_external_ids: str | Sequence[str] | None = None,
         active: bool | None = None,
     ) -> None:
-        self.simulator_external_ids = simulator_external_ids
+        self.simulator_external_ids = (
+            [simulator_external_ids] if isinstance(simulator_external_ids, str) else simulator_external_ids
+        )
         self.active = active
 
 
 class SimulatorModelsFilter(CogniteFilter):
     def __init__(
         self,
-        simulator_external_ids: Sequence[str] | None = None,
+        simulator_external_ids: str | Sequence[str] | None = None,
     ) -> None:
         self.simulator_external_ids = (
             [simulator_external_ids] if isinstance(simulator_external_ids, str) else simulator_external_ids
@@ -29,7 +30,7 @@ def __init__(
 class SimulatorModelRevisionsFilter(CogniteFilter):
     def __init__(
         self,
-        model_external_ids: Sequence[str] | None = None,
+        model_external_ids: str | Sequence[str] | None = None,
         all_versions: bool | None = None,
     ) -> None:
         self.model_external_ids = [model_external_ids] if isinstance(model_external_ids, str) else model_external_ids

cognite/client/utils/_identifier.py

@@ -32,6 +32,22 @@ class InstanceId:
     def dump(self, camel_case: bool = True, include_instance_type: bool = True) -> dict[str, str]:
         return {"space": self.space, "externalId" if camel_case else "external_id": self.external_id}
 
+    @overload
+    @classmethod
+    def load_if(cls, data: None) -> None: ...
+
+    @overload
+    @classmethod
+    def load_if(cls, data: dict[str, str] | tuple[str, str] | Self) -> Self: ...
+
+    @classmethod
+    def load_if(cls, data: dict[str, str] | tuple[str, str] | Self | None) -> Self | None:
+        # Note: For experimentation - I'd like to add this as a universal method to all classes to avoid
+        # the endless spam of 'MyClass.load(foo["bar"]) if "bar" in foo else None' in the codebase!
+        if data is None:
+            return None
+        return cls.load(data)
+
     @classmethod
     def load(cls, data: dict[str, str] | tuple[str, str] | Self) -> Self:
         if isinstance(data, cls):

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "cognite-sdk"
 
-version = "7.73.3"
+version = "7.73.4"
 
 description = "Cognite Python SDK"
 readme = "README.md"

tests/tests_integration/test_api/test_datapoints.py

@@ -2826,6 +2826,26 @@ def test_instance_id_usage(self, cognite_client, instance_ts_id, instance_ts_lat
             assert isinstance(dp.value, float)
             assert int(dp.timestamp / 1000) == int(dp.value)
 
+    def test_instance_id_and_missing(self, cognite_client, instance_ts_id):
+        # Before 7.73.4 (and after support for instance_id was added ofc), when a not-found time series was requested
+        # by instance_id and ignore_unknown_ids=True was passed, a TypeError was raised instead of a CogniteNotFoundError
+        # as we tried to put a dict into a set (not hashable)
+        missing = NodeId("I-do-not", "exist")
+        res = cognite_client.time_series.data.retrieve_latest(instance_id=missing, ignore_unknown_ids=True)
+        assert res is None
+        # This could also happen - but on a separate codeline - when the missing was not an instance_id, as we loaded the
+        # dict in before a comparison of the identifiers:
+        res = cognite_client.time_series.data.retrieve_latest(id=1, instance_id=instance_ts_id, ignore_unknown_ids=True)
+        assert type(res) is DatapointsList
+        assert len(res) == 1
+        assert res[0].instance_id == instance_ts_id
+
+        # ...and just to ensure this still works as expected:
+        with pytest.raises(CogniteNotFoundError, match=r"^Not found: \[{'"):
+            cognite_client.time_series.data.retrieve_latest(id=1, instance_id=instance_ts_id, ignore_unknown_ids=False)
+        with pytest.raises(CogniteNotFoundError, match=r"^Not found: \[{'"):
+            cognite_client.time_series.data.retrieve_latest(instance_id=missing, ignore_unknown_ids=False)
+
 
 class TestInsertDatapointsAPI:
     @pytest.mark.usefixtures("post_spy")

tests/tests_integration/test_api/test_simulators/test_integrations.py

@@ -8,45 +8,41 @@
 from tests.tests_integration.test_api.test_simulators.seed.data import simulator_integration
 
 
+@pytest.mark.usefixtures("seed_resource_names", "seed_simulator_integration")
 class TestSimulatorIntegrations:
-    @pytest.mark.usefixtures("seed_resource_names", "seed_simulator_integration")
     def test_list_integrations(self, cognite_client: CogniteClient) -> None:
         integrations = cognite_client.simulators.integrations.list(limit=5)
 
         assert len(integrations) > 0
 
-    def test_filter_integrations(self, cognite_client: CogniteClient, seed_resource_names) -> None:
+    def test_filter_integrations_asdfdsa(self, cognite_client: CogniteClient, seed_resource_names) -> None:
         for integration in cognite_client.simulators.integrations(filter=SimulatorIntegrationFilter(active=True)):
             assert integration.active is True
 
         all_integrations = cognite_client.simulators.integrations.list()
         active_integrations = cognite_client.simulators.integrations.list(
             filter=SimulatorIntegrationFilter(active=True)
         )
-
         filtered_integrations = cognite_client.simulators.integrations.list(
             filter=SimulatorIntegrationFilter(simulator_external_ids=[seed_resource_names["simulator_external_id"]])
         )
-
         assert len(all_integrations) > 0
-        assert filtered_integrations[0].external_id == seed_resource_names["simulator_integration_external_id"]
-        # assert filtered_integrations[0].data_set_id == seed_resource_names["simulator_test_data_set_id"]
-        assert filtered_integrations[0].active is True
-        assert filtered_integrations[0].created_time is not None
-        assert filtered_integrations[0].last_updated_time is not None
-
         assert len(active_integrations) > 0
         assert len(filtered_integrations) > 0
 
+        item = filtered_integrations.get(external_id=seed_resource_names["simulator_integration_external_id"])
+        assert item is not None
+        # assert item.data_set_id == seed_resource_names["simulator_test_data_set_id"]
+        assert item.active is True
+        assert item.created_time is not None
+        assert item.last_updated_time is not None
+
     def test_delete_integrations(self, cognite_client: CogniteClient, seed_resource_names) -> None:
         simulator_integration["heartbeat"] = int(time.time() * 1000)
         simulator_integration["externalId"] = random_string(50)
         simulator_integration["dataSetId"] = seed_resource_names["simulator_test_data_set_id"]
 
-        cognite_client.simulators._post(
-            "/simulators/integrations",
-            json={"items": [simulator_integration]},
-        )
+        cognite_client.simulators._post("/simulators/integrations", json={"items": [simulator_integration]})
 
         all_integrations = cognite_client.simulators.integrations.list(limit=None)
         assert all_integrations.get(external_id=simulator_integration["externalId"]) is not None

tests/tests_unit/test_api_client.py

@@ -163,7 +163,7 @@ def test_headers_correct(self, mock_all_requests_ok, api_client_with_token):
         api_client_with_token._post(URL_PATH, {"any": "OK"}, headers={"additional": "stuff"})
         headers = mock_all_requests_ok.calls[0].request.headers
 
-        assert "gzip, deflate" == headers["accept-encoding"]
+        assert "gzip, deflate, zstd" == headers["accept-encoding"]
         assert "gzip" == headers["content-encoding"]
         assert f"CognitePythonSDK:{utils._auxiliary.get_current_sdk_version()}" == headers["x-cdp-sdk"]
         assert "Bearer abc" == headers["Authorization"]