Store empty list instead of unset

docs/changelog.rst

@@ -8,6 +8,7 @@ Development
 ===========
 - (Fill this out as you fix issues and develop your features).
 - EnumField improvements: now `choices` limits the values of an enum to allow
+- renamed Doc._get_changed_fields into Doc._get_updated_fields
 
 Changes in 0.23.1
 ===========

mongoengine/base/__init__.py

@@ -13,6 +13,7 @@
 __all__ = (
     # common
     "UPDATE_OPERATORS",
+    "UNSET_SENTINEL",
     "_document_registry",
     "get_document",
     # datastructures

mongoengine/base/common.py

@@ -1,6 +1,6 @@
 from mongoengine.errors import NotRegistered
 
-__all__ = ("UPDATE_OPERATORS", "get_document", "_document_registry")
+__all__ = ("UPDATE_OPERATORS", "get_document", "_document_registry", "UNSET_SENTINEL")
 
 
 UPDATE_OPERATORS = {
@@ -21,6 +21,7 @@
     "rename",
 }
 
+UNSET_SENTINEL = object()
 
 _document_registry = {}
 

mongoengine/base/datastructures.py

@@ -38,21 +38,36 @@ def wrapper(self, key, *args, **kwargs):
     return wrapper
 
 
+def mark_key_as_unset_wrapper(parent_method):
+    """Decorator that ensures _mark_as_unset method gets called with the key argument"""
+
+    def wrapper(self, key, *args, **kwargs):
+        # Can't use super() in the decorator.
+        result = parent_method(self, key, *args, **kwargs)
+        self._mark_as_unset(key)
+        return result
+
+    return wrapper
+
+
 class BaseDict(dict):
     """A special dict so we can watch any changes."""
 
     _dereferenced = False
-    _instance = None
-    _name = None
 
     def __init__(self, dict_items, instance, name):
         BaseDocument = _import_class("BaseDocument")
 
         if isinstance(instance, BaseDocument):
             self._instance = weakref.proxy(instance)
+        else:
+            self._instance = instance
         self._name = name
         super().__init__(dict_items)
 
+    def _get_resolved_key(self, key=None):
+        return f"{self._name}.{key}" if key else self._name
+
     def get(self, key, default=None):
         # get does not use __getitem__ by default so we must override it as well
         try:
@@ -63,17 +78,18 @@ def get(self, key, default=None):
     def __getitem__(self, key):
         value = super().__getitem__(key)
 
+        resolved_key = self._get_resolved_key(key)
+
         EmbeddedDocument = _import_class("EmbeddedDocument")
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
-            value = BaseDict(value, None, f"{self._name}.{key}")
+            value = BaseDict(value, None, resolved_key)
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
-            value = BaseList(value, None, f"{self._name}.{key}")
+            value = BaseList(value, self._instance, resolved_key)
             super().__setitem__(key, value)
-            value._instance = self._instance
         return value
 
     def __getstate__(self):
@@ -86,37 +102,47 @@ def __setstate__(self, state):
         return self
 
     __setitem__ = mark_key_as_changed_wrapper(dict.__setitem__)
-    __delattr__ = mark_key_as_changed_wrapper(dict.__delattr__)
-    __delitem__ = mark_key_as_changed_wrapper(dict.__delitem__)
+    __delattr__ = mark_key_as_unset_wrapper(dict.__delattr__)
+    __delitem__ = mark_key_as_unset_wrapper(dict.__delitem__)
     pop = mark_as_changed_wrapper(dict.pop)
     clear = mark_as_changed_wrapper(dict.clear)
     update = mark_as_changed_wrapper(dict.update)
     popitem = mark_as_changed_wrapper(dict.popitem)
     setdefault = mark_as_changed_wrapper(dict.setdefault)
 
+    def _mark_as_unset(self, key):
+        resolved_key = self._get_resolved_key(key)
+        if hasattr(self._instance, "_mark_as_unset"):
+            self._instance._mark_as_unset(resolved_key)
+
     def _mark_as_changed(self, key=None):
+        resolved_key = self._get_resolved_key(key)
         if hasattr(self._instance, "_mark_as_changed"):
-            if key:
-                self._instance._mark_as_changed(f"{self._name}.{key}")
-            else:
-                self._instance._mark_as_changed(self._name)
+            self._instance._mark_as_changed(resolved_key)
 
 
 class BaseList(list):
     """A special list so we can watch any changes."""
 
     _dereferenced = False
-    _instance = None
-    _name = None
 
     def __init__(self, list_items, instance, name):
         BaseDocument = _import_class("BaseDocument")
 
         if isinstance(instance, BaseDocument):
             self._instance = weakref.proxy(instance)
+        else:
+            self._instance = instance
+
         self._name = name
         super().__init__(list_items)
 
+    def _get_resolved_key(self, key=None):
+        if key is not None:
+            return f"{self._name}.{key % len(self)}"
+        else:
+            return self._name
+
     def __getitem__(self, key):
         # change index to positive value because MongoDB does not support negative one
         if isinstance(key, int) and key < 0:
@@ -128,19 +154,20 @@ def __getitem__(self, key):
             # to parent's instance. This is buggy for now but would require more work to be handled properly
             return value
 
+        resolved_key = self._get_resolved_key(key)
+
         EmbeddedDocument = _import_class("EmbeddedDocument")
         if isinstance(value, EmbeddedDocument) and value._instance is None:
             value._instance = self._instance
         elif isinstance(value, dict) and not isinstance(value, BaseDict):
             # Replace dict by BaseDict
-            value = BaseDict(value, None, f"{self._name}.{key}")
+            value = BaseDict(value, None, resolved_key)
             super().__setitem__(key, value)
             value._instance = self._instance
         elif isinstance(value, list) and not isinstance(value, BaseList):
             # Replace list by BaseList
-            value = BaseList(value, None, f"{self._name}.{key}")
+            value = BaseList(value, self._instance, resolved_key)
             super().__setitem__(key, value)
-            value._instance = self._instance
         return value
 
     def __iter__(self):
@@ -178,11 +205,9 @@ def __setitem__(self, key, value):
     __imul__ = mark_as_changed_wrapper(list.__imul__)
 
     def _mark_as_changed(self, key=None):
+        resolved_key = self._get_resolved_key(key)
         if hasattr(self._instance, "_mark_as_changed"):
-            if key is not None:
-                self._instance._mark_as_changed(f"{self._name}.{key % len(self)}")
-            else:
-                self._instance._mark_as_changed(self._name)
+            self._instance._mark_as_changed(resolved_key)
 
 
 class EmbeddedDocumentList(BaseList):