feat: basic support for custom fields (#75)

basic support for custom fields

Author: ltucker

netbox_diode_plugin/api/applier.py

@@ -18,7 +18,7 @@
 logger = logging.getLogger(__name__)
 
 
-def apply_changeset(change_set: ChangeSet) -> ChangeSetResult:
+def apply_changeset(change_set: ChangeSet, request) -> ChangeSetResult:
     """Apply a change set."""
     _validate_change_set(change_set)
 
@@ -33,7 +33,7 @@ def apply_changeset(change_set: ChangeSet) -> ChangeSetResult:
         try:
             model_class = get_object_type_model(object_type)
             data = _pre_apply(model_class, change, created)
-            _apply_change(data, model_class, change, created)
+            _apply_change(data, model_class, change, created, request)
         except ValidationError as e:
             raise _err_from_validation_error(e, f"changes[{i}]")
         except ObjectDoesNotExist:
@@ -45,42 +45,58 @@ def apply_changeset(change_set: ChangeSet) -> ChangeSetResult:
         id=change_set.id,
     )
 
-def _apply_change(data: dict, model_class: models.Model, change: Change, created: dict):
+def _apply_change(data: dict, model_class: models.Model, change: Change, created: dict, request):
     serializer_class = get_serializer_for_model(model_class)
     change_type = change.change_type
     if change_type == ChangeType.CREATE.value:
-        serializer = serializer_class(data=data)
+        serializer = serializer_class(data=data, context={"request": request})
         serializer.is_valid(raise_exception=True)
         instance = serializer.save()
         created[change.ref_id] = instance
 
     elif change_type == ChangeType.UPDATE.value:
         if object_id := change.object_id:
             instance = model_class.objects.get(id=object_id)
-            serializer = serializer_class(instance, data=data, partial=True)
+            serializer = serializer_class(instance, data=data, partial=True, context={"request": request})
             serializer.is_valid(raise_exception=True)
             serializer.save()
         # create and update in a same change set
         elif change.ref_id and (instance := created[change.ref_id]):
-            serializer = serializer_class(instance, data=data, partial=True)
+            serializer = serializer_class(instance, data=data, partial=True, context={"request": request})
             serializer.is_valid(raise_exception=True)
             serializer.save()
 
+def _set_path(data, path, value):
+    path = path.split(".")
+    key = path.pop(0)
+    while len(path) > 0:
+        data = data[key]
+        key = path.pop(0)
+    data[key] = value
+
+def _get_path(data, path):
+    path = path.split(".")
+    v = data
+    for p in path:
+        v = v[p]
+    return v
+
 def _pre_apply(model_class: models.Model, change: Change, created: dict):
     data = change.data.copy()
 
     # resolve foreign key references to new objects
     for ref_field in change.new_refs:
-        if isinstance(data[ref_field], (list, tuple)):
+        v = _get_path(data, ref_field)
+        if isinstance(v, (list, tuple)):
             ref_list = []
-            for ref in data[ref_field]:
+            for ref in v:
                 if isinstance(ref, str):
                     ref_list.append(created[ref].pk)
                 elif isinstance(ref, int):
                     ref_list.append(ref)
-            data[ref_field] = ref_list
+            _set_path(data, ref_field, ref_list)
         else:
-            data[ref_field] = created[data[ref_field]].pk
+            _set_path(data, ref_field, created[v].pk)
 
     # ignore? fields that are not in the data model (error?)
     allowed_fields = legal_fields(model_class)

netbox_diode_plugin/api/common.py

@@ -13,6 +13,7 @@
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ValidationError
 from django.db import models
+from extras.models import CustomField
 from rest_framework import status
 
 logger = logging.getLogger("netbox.diode_data")
@@ -114,13 +115,41 @@ def validate(self) -> dict[str, list[str]]:
                 errors[change.object_type] = rel_errors
 
             try:
+                custom_fields = change_data.pop('custom_fields', None)
+                if custom_fields:
+                    self._validate_custom_fields(custom_fields, model)
+
                 instance = model(**change_data)
                 instance.clean_fields(exclude=excluded_relation_fields)
             except ValidationError as e:
-                errors[change.object_type].update(e.error_dict)
+                errors[change.object_type].update(_error_dict(e))
 
         return errors or None
 
+    def _validate_custom_fields(self, data: dict, model: models.Model) -> None:
+        custom_fields = {
+            cf.name: cf for cf in CustomField.objects.get_for_model(model)
+        }
+
+        unknown_errors = []
+        for field_name, value in data.items():
+            if field_name not in custom_fields:
+                unknown_errors.append(f"Unknown field name '{field_name}' in custom field data.")
+                continue
+        if unknown_errors:
+            raise ValidationError({
+                "custom_fields": unknown_errors
+            })
+
+        req_errors = []
+        for field_name, cf in custom_fields.items():
+            if cf.required and field_name not in data:
+                req_errors.append(f"Custom field '{field_name}' is required.")
+        if req_errors:
+            raise ValidationError({
+                "custom_fields": req_errors
+            })
+
     def _validate_relations(self, change_data: dict, model: models.Model) -> tuple[list[str], dict]:
         # check that there is some value for every required
         # reference field, but don't validate the actual reference.
@@ -191,3 +220,18 @@ def __str__(self):
         if self.errors:
             return f"{self.message}: {self.errors}"
         return self.message
+
+def _error_dict(e: ValidationError) -> dict:
+    """Convert a ValidationError to a dictionary."""
+    if hasattr(e, "error_dict"):
+        return e.error_dict
+    return {
+        "__all__": e.error_list
+    }
+
+@dataclass
+class AutoSlug:
+    """A class that marks an auto-generated slug."""
+
+    field_name: str
+    value: str

netbox_diode_plugin/api/differ.py

@@ -3,6 +3,7 @@
 """Diode NetBox Plugin - API - Differ."""
 
 import copy
+import datetime
 import logging
 
 from django.contrib.contenttypes.models import ContentType
@@ -12,7 +13,7 @@
 from .common import Change, ChangeSet, ChangeSetException, ChangeSetResult, ChangeType
 from .plugin_utils import get_primary_value, legal_fields
 from .supported_models import extract_supported_models
-from .transformer import cleanup_unresolved_references, transform_proto_json
+from .transformer import cleanup_unresolved_references, set_custom_field_defaults, transform_proto_json
 
 logger = logging.getLogger(__name__)
 
@@ -68,9 +69,31 @@ def prechange_data_from_instance(instance) -> dict: # noqa: C901
         else:
             prechange_data[field_name] = value
 
+    if hasattr(instance, "get_custom_fields"):
+        custom_field_values = instance.get_custom_fields()
+        cfmap = {}
+        for cf, value in custom_field_values.items():
+            if isinstance(value, (datetime.datetime, datetime.date)):
+                cfmap[cf.name] = value
+            else:
+                cfmap[cf.name] = cf.serialize(value)
+        prechange_data["custom_fields"] = cfmap
+
     return prechange_data
 
 
+def _harmonize_formats(prechange_data: dict, postchange_data: dict):
+    for k, v in prechange_data.items():
+        if isinstance(v, datetime.datetime):
+            prechange_data[k] = v.strftime("%Y-%m-%dT%H:%M:%SZ")
+        elif isinstance(v, datetime.date):
+            prechange_data[k] = v.strftime("%Y-%m-%d")
+        elif isinstance(v, int) and k in postchange_data:
+            postchange_data[k] = int(postchange_data[k])
+        elif isinstance(v, dict):
+            _harmonize_formats(v, postchange_data.get(k, {}))
+
+
 def clean_diff_data(data: dict, exclude_empty_values: bool = True) -> dict:
     """Clean diff data by removing null values."""
     result = {}
@@ -80,8 +103,10 @@ def clean_diff_data(data: dict, exclude_empty_values: bool = True) -> dict:
                 continue
             if isinstance(v, list) and len(v) == 0:
                 continue
-            if isinstance(v, dict) and len(v) == 0:
-                continue
+            if isinstance(v, dict):
+                if len(v) == 0:
+                    continue
+                v = clean_diff_data(v, exclude_empty_values)
             if isinstance(v, str) and v == "":
                 continue
         result[k] = v
@@ -100,7 +125,7 @@ def diff_to_change(
     if change_type == ChangeType.UPDATE and not len(changed_attrs) > 0:
         change_type = ChangeType.NOOP
 
-    primary_value = get_primary_value(prechange_data | postchange_data, object_type)
+    primary_value = str(get_primary_value(prechange_data | postchange_data, object_type))
     if primary_value is None:
         primary_value = "(unnamed)"
 
@@ -111,6 +136,8 @@ def diff_to_change(
 
     change = Change(
         change_type=change_type,
+        before=_tidy(prechange_data),
+        data={},
         object_type=object_type,
         object_id=prior_id if isinstance(prior_id, int) else None,
         ref_id=ref_id,
@@ -119,17 +146,13 @@ def diff_to_change(
     )
 
     if change_type != ChangeType.NOOP:
-        postchange_data_clean = clean_diff_data(postchange_data)
-        change.data = sort_dict_recursively(postchange_data_clean)
-    else:
-        change.data = {}
-
-    if change_type == ChangeType.UPDATE or change_type == ChangeType.NOOP:
-        prechange_data_clean = clean_diff_data(prechange_data)
-        change.before = sort_dict_recursively(prechange_data_clean)
+        change.data = _tidy(postchange_data)
 
     return change
 
+def _tidy(data: dict) -> dict:
+    return sort_dict_recursively(clean_diff_data(data))
+
 def sort_dict_recursively(d):
     """Recursively sorts a dictionary by keys."""
     if isinstance(d, dict):
@@ -161,7 +184,11 @@ def generate_changeset(entity: dict, object_type: str) -> ChangeSetResult:
             # prior state is a model instance
             else:
                 prechange_data = prechange_data_from_instance(instance)
-
+                # merge the prior state that we don't want to overwrite with the new state
+                # this is also important for custom fields because they do not appear to
+                # respsect paritial update serialization.
+                entity = _partially_merge(prechange_data, entity, instance)
+                _harmonize_formats(prechange_data, entity)
             changed_data = shallow_compare_dict(
                 prechange_data, entity,
             )
@@ -187,7 +214,25 @@ def generate_changeset(entity: dict, object_type: str) -> ChangeSetResult:
     if errors := change_set.validate():
         raise ChangeSetException("Invalid change set", errors)
 
-    return ChangeSetResult(
+
+    cs = ChangeSetResult(
         id=change_set.id,
         change_set=change_set,
     )
+    return cs
+
+def _partially_merge(prechange_data: dict, postchange_data: dict, instance) -> dict:
+    """Merge lists and custom_fields rather than replacing the full value..."""
+    result = {}
+    for key, value in postchange_data.items():
+        # TODO: partially merge lists like tags? all lists?
+        result[key] = value
+
+    # these are fully merged in from the prechange state because
+    # they don't respect partial update serialization.
+    if "custom_fields" in postchange_data:
+        for key, value in prechange_data.get("custom_fields", {}).items():
+            if value is not None and key not in postchange_data["custom_fields"]:
+                result["custom_fields"][key] = value
+        set_custom_field_defaults(result, instance)
+    return result