Make sync() populate objects with same .id with the same set of fields

* Currently sync() will track objects by their Python address, so if we
  have `alice1` with just `name` property, and we have `alice2` with the
  same `id` but only `email` property, after sync() they would only have
  their `.name` and `.email` properties updated respectively.

  We are changing sync() to populate them with the same data, so after
  sync() they would both have `.name` and `.email` attributes and their
  `.__dict__` would be equal.

  We do this because of computed backlinks (although I think we can find
  more motivation for this if we think hard). If `alice1` and `alice2`
  have the same `.id` and `alice1` was added to the `UserGroup.users` multi
  link, we want *both* `alice1.groups` and `alice2.groups` computeds to
  point back to the new user group they are member of. We also want to
  make it irrelevant if it was `alice1` or `alice2` added to
  `UserGroup.users` -- after sync() it shouldn't matter as they will
  both reflect the same data and be completely equivalent (albeit still
  two distinct Python objects, with distinct link set & multi props &
  mutable props collections!)

* LinkSet / LinkWithPropsSet / TrackedList get `__copy__` and
  `__deepcopy__` methods

Author: 1st1

gel/_internal/_qbmodel/_abstract/__init__.py

@@ -40,6 +40,8 @@
     get_proxy_linkprops,
     is_proxy_linked,
     copy_or_ref_lprops,
+    reconcile_link,
+    reconcile_proxy_link,
 )
 
 from ._link_set import (
@@ -184,4 +186,6 @@
     "is_generic_type",
     "is_proxy_linked",
     "maybe_get_protocol_for_py_type",
+    "reconcile_link",
+    "reconcile_proxy_link",
 )

gel/_internal/_qbmodel/_abstract/_descriptors.py

@@ -41,7 +41,10 @@
 
 if TYPE_CHECKING:
     import types
-    from collections.abc import Sequence, Set as AbstractSet
+    import uuid
+
+    from collections.abc import Sequence, Set as AbstractSet, Iterable
+
     from ._link_set import (
         AbstractLinkSet,
         ComputedLinkSet,
@@ -740,6 +743,63 @@ def copy_or_ref_lprops(lp: _LM_co) -> _LM_co:  # type: ignore [misc]
         return lp
 
 
+def reconcile_link(
+    *,
+    existing: _MT_co | None,
+    refetched: _MT_co,  # type: ignore [misc]
+    existing_objects: dict[uuid.UUID, _MT_co | Iterable[_MT_co]],
+    new_objects: dict[uuid.UUID, _MT_co],
+) -> _MT_co:
+    if existing is not None:
+        return existing
+
+    obj_id: uuid.UUID = ll_getattr(refetched, "id")
+
+    if (link_to := existing_objects.get(obj_id)) is not None:
+        if isinstance(link_to, AbstractGelModel):
+            return link_to
+        else:
+            return next(iter(link_to))
+
+    return new_objects[obj_id]
+
+
+def reconcile_proxy_link(
+    *,
+    existing: AbstractGelProxyModel[_MT_co, _LM_co] | None,
+    refetched: AbstractGelProxyModel[_MT_co, _LM_co],
+    existing_objects: dict[uuid.UUID, _MT_co | Iterable[_MT_co]],
+    new_objects: dict[uuid.UUID, _MT_co],
+) -> AbstractGelProxyModel[_MT_co, _LM_co]:
+    if existing is not None:
+        # `refetched` will have newly refetched __linkprops__,
+        # so copy them
+        existing.__gel_replace_linkprops__(
+            copy_or_ref_lprops(refetched.__linkprops__)
+        )
+        return existing
+
+    obj_id: uuid.UUID = ll_getattr(refetched.without_linkprops(), "id")
+
+    if (_link_to := existing_objects.get(obj_id)) is not None:
+        if isinstance(_link_to, AbstractGelModel):
+            link_to = _link_to
+        else:
+            link_to = next(iter(_link_to))
+    else:
+        link_to = new_objects[obj_id]
+
+    # Make sure we create a new proxy model that
+    # would wrap either an object that already exists
+    # or a new one just inserted by sync(); copy linkprops
+    # efficiently.
+    return refetched.__gel_proxy_construct__(
+        link_to,  # pyright: ignore [reportArgumentType]
+        copy_or_ref_lprops(refetched.__linkprops__),
+        linked=True,
+    )
+
+
 def proxy_link(
     *,
     existing: AbstractGelProxyModel[_MT_co, _LM_co] | None,

gel/_internal/_qbmodel/_abstract/_link_set.py

@@ -14,6 +14,7 @@
 
 from gel._internal import _typing_parametric as parametric
 from gel._internal._qbmodel._abstract._base import AbstractGelLinkModel
+from gel._internal._qbmodel import _abstract
 from gel._internal._tracked_list import (
     AbstractCollection,
     DefaultList,
@@ -74,10 +75,32 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         self._index_snapshot = None
         super().__init__(*args, **kwargs)
 
+    def __copy__(self) -> Self:
+        obj = type(self).__new__(type(self))
+
+        obj.__gel_overwrite_data__ = self.__gel_overwrite_data__
+        obj._mode = self._mode
+        obj._items = list(self._items)
+        obj._index_snapshot = (
+            dict(self._index_snapshot) if self._index_snapshot else None
+        )
+        obj._tracking_set = (
+            dict(self._tracking_set) if self._tracking_set else None
+        )
+        obj._tracking_index = (
+            dict(self._tracking_index) if self._tracking_index else None
+        )
+        return obj
+
+    def __deepcopy__(self, memo: dict[int, Any]) -> Self:
+        return self.__copy__()
+
     def __gel_reconcile__(
         self,
         updated: AbstractLinkSet[_MT_co],
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
     ) -> Self:
         raise NotImplementedError
@@ -117,10 +140,10 @@ def _is_tracked(self, item: _MT_co) -> bool:  # type: ignore [misc]
     def _reconcile(
         self,
         updated: AbstractLinkSet[_MT_co],
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
-        *,
-        is_computed: bool,
     ) -> list[_MT_co]:
         raise NotImplementedError
 
@@ -250,6 +273,12 @@ def __repr__(self) -> str:
         else:
             return repr(self._items)
 
+    def __gel_replace_with_empty__(self) -> None:
+        self._items.clear()
+        self._index_snapshot = None
+        self._tracking_set = None
+        self._tracking_index = None
+
 
 class ComputedLinkSet(
     parametric.SingleParametricType[_MT_co],
@@ -266,10 +295,10 @@ def _pyid(item: _MT_co) -> int:  # type: ignore [misc]
     def _reconcile(
         self,
         updated: AbstractLinkSet[_MT_co],
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
-        *,
-        is_computed: bool,
     ) -> list[_MT_co]:
         # This method is called by sync() when it refetches the link.
         #
@@ -319,30 +348,29 @@ def _reconcile(
 
         updated_items: list[_MT_co] = []
         for obj in updated:
-            obj_id: uuid.UUID = obj.id  # type: ignore [attr-defined]
-
-            try:
-                # This works because `GelModel` hashes and compares by `.id`
-                existing = existing_set[obj]
-            except KeyError:
-                if is_computed and obj_id in existing_objects:
-                    updated_items.append(existing_objects[obj_id])  # type: ignore [arg-type]
-                else:
-                    updated_items.append(new_objects[obj_id])  # type: ignore [arg-type]
-            else:
-                updated_items.append(existing)
+            # This works because `GelModel` hashes and compares by `.id`
+            existing: _MT_co | None = existing_set.get(obj)
+
+            updated_items.append(
+                _abstract.reconcile_link(  # type: ignore [misc]
+                    existing=existing,
+                    refetched=obj,
+                    existing_objects=existing_objects,
+                    new_objects=new_objects,
+                )
+            )
 
         return updated_items
 
     def __gel_reconcile__(  # pyright: ignore [reportIncompatibleMethodOverride]
         self,
         updated: AbstractLinkSet[_MT_co],
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
     ) -> Self:
-        new_items = self._reconcile(
-            updated, existing_objects, new_objects, is_computed=True
-        )
+        new_items = self._reconcile(updated, existing_objects, new_objects)
         return type(self)(
             new_items,
             __mode__=self._mode,
@@ -415,10 +443,10 @@ def __gel_basetype_iter__(self) -> Iterator[_BMT_co]:  # type: ignore [override]
     def _reconcile(  # pyright: ignore [reportIncompatibleMethodOverride]
         self,
         updated: LinkWithPropsSet[_PT_co, _BMT_co],  # type: ignore [override]
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
-        *,
-        is_computed: bool,
     ) -> list[_PT_co]:
         # See comments in `LinkSet._reconcile()` for most implementation
         # details.
@@ -432,36 +460,29 @@ def _reconcile(  # pyright: ignore [reportIncompatibleMethodOverride]
 
         updated_items: list[_PT_co] = []
         for obj in updated:
-            obj_id: uuid.UUID = obj.id  # type: ignore [attr-defined]
-
-            try:
-                existing = existing_set[obj]
-            except KeyError:
-                if is_computed and obj_id in existing_objects:
-                    new = existing_objects[obj_id]
-                else:
-                    new = new_objects[obj_id]
-                # `obj` will have updated __linkprops__ but the wrapped
-                # model will be coming from new_objects
-                obj.__gel_replace_wrapped_model__(new)
-                updated_items.append(obj)
-            else:
-                # updated will have newly refetched __linkprops__, so
-                # copy them
-                existing.__gel_replace_linkprops__(obj.__linkprops__)
-                updated_items.append(existing)
+            # This works because `GelModel` hashes and compares by `.id`
+            existing = existing_set.get(obj)
+
+            updated_items.append(
+                _abstract.reconcile_proxy_link(
+                    existing=existing,
+                    refetched=obj,
+                    existing_objects=existing_objects,
+                    new_objects=new_objects,
+                )  # type: ignore [arg-type]
+            )
 
         return updated_items
 
     def __gel_reconcile__(  # pyright: ignore [reportIncompatibleMethodOverride]
         self,
         updated: LinkWithPropsSet[_PT_co, _BMT_co],  # type: ignore [override]
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
     ) -> Self:
-        new_items = self._reconcile(
-            updated, existing_objects, new_objects, is_computed=True
-        )
+        new_items = self._reconcile(updated, existing_objects, new_objects)
         return type(self)(
             new_items,
             __mode__=self._mode,
@@ -562,12 +583,12 @@ class AbstractMutableLinkSet(
     def __gel_reconcile__(  # pyright: ignore [reportIncompatibleMethodOverride]
         self,
         updated: AbstractLinkSet[_MT_co],
-        existing_objects: dict[uuid.UUID, AbstractGelModel],
+        existing_objects: dict[
+            uuid.UUID, AbstractGelModel | Iterable[AbstractGelModel]
+        ],
         new_objects: dict[uuid.UUID, AbstractGelModel],
     ) -> Self:
-        self._items = self._reconcile(
-            updated, existing_objects, new_objects, is_computed=False
-        )
+        self._items = self._reconcile(updated, existing_objects, new_objects)
         # Reset tracking: it will likely be not needed. Typically there
         # should be just one `sync()` call with no modifications of anything
         # after it.
@@ -595,12 +616,6 @@ def _ensure_snapshot(self) -> None:
             assert self._tracking_index is not None
             self._index_snapshot = dict(self._tracking_index)
 
-    def __gel_replace_with_empty__(self) -> None:
-        self._items.clear()
-        self._index_snapshot = None
-        self._tracking_set = None
-        self._tracking_index = None
-
     def __gel_get_added__(self) -> list[_MT_co]:
         match bool(self._index_snapshot), bool(self._tracking_index):
             case True, False: