Merge branch 'develop' into engine-refactor

Author: latteier

docs/source/conf.py

@@ -50,7 +50,7 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'Nefertari'
+project = u'nefertari-mongodb'
 copyright = u'2015, Brandicted'
 author = u'Brandicted'
 

docs/source/index.rst

@@ -1,8 +1,8 @@
 MongoDB Engine
 ==============
 
-Index
------
+Table of Contents
+=================
 
 .. toctree::
    :maxdepth: 2

nefertari_mongodb/documents.py

@@ -7,8 +7,7 @@
 from nefertari.json_httpexceptions import (
     JHTTPBadRequest, JHTTPNotFound, JHTTPConflict)
 from nefertari.utils import (
-    process_fields, process_limit, _split, dictset, DataProxy,
-    drop_reserved_params)
+    process_fields, process_limit, _split, dictset, drop_reserved_params)
 from .metaclasses import ESMetaclass, DocumentMetaclass
 from .signals import on_bulk_update
 from .fields import (
@@ -64,7 +63,7 @@ def process_bools(_dict):
     return _dict
 
 
-TYPES_MAP = {
+types_map = {
     StringField: {'type': 'string'},
     TextField: {'type': 'string'},
     UnicodeField: {'type': 'string'},
@@ -104,35 +103,45 @@ class BaseMixin(object):
             included documents. If reference/relationship field is not
             present in this list, this field's value in JSON will be an
             object's ID or list of IDs.
+        _nesting_depth: Depth of relationship field nesting in JSON.
+            Defaults to 1(one) which makes only one level of relationship
+            nested.
     """
     _public_fields = None
     _auth_fields = None
     _nested_relationships = ()
     _backref_hooks = ()
+    _nesting_depth = 1
 
     _type = property(lambda self: self.__class__.__name__)
     Q = mongo.Q
 
     @classmethod
-    def get_es_mapping(cls):
+    def get_es_mapping(cls, _depth=None):
         """ Generate ES mapping from model schema. """
         from nefertari.elasticsearch import ES
+        if _depth is None:
+            _depth = cls._nesting_depth
+        depth_reached = _depth <= 0
+
         properties = {}
         mapping = {
             ES.src2type(cls.__name__): {
                 'properties': properties
             }
         }
         fields = cls._fields.copy()
-
         for name, field in fields.items():
             if isinstance(field, RelationshipField):
                 field = field.field
             if isinstance(field, (ReferenceField, RelationshipField)):
-                if name in cls._nested_relationships:
+                if name in cls._nested_relationships and not depth_reached:
                     field_mapping = {'type': 'object'}
+                    submapping = field.document_type.get_es_mapping(
+                        _depth=_depth-1)
+                    field_mapping.update(list(submapping.values())[0])
                 else:
-                    field_mapping = TYPES_MAP[
+                    field_mapping = types_map[
                         field.document_type.pk_field_type()]
                 properties[name] = field_mapping
                 continue
@@ -142,9 +151,9 @@ def get_es_mapping(cls):
             field_type = type(field)
             if field_type is ListField:
                 field_type = field.item_type
-            if field_type not in TYPES_MAP:
+            if field_type not in types_map:
                 continue
-            properties[name] = TYPES_MAP[field_type]
+            properties[name] = types_map[field_type]
 
         properties['_pk'] = {'type': 'string'}
         return mapping
@@ -441,8 +450,11 @@ def get_by_ids(cls, ids, **params):
     @classmethod
     def get_null_values(cls):
         """ Get null values of :cls: fields. """
+        skip_fields = {'_version', '_acl'}
         null_values = {}
         for name in cls._fields.keys():
+            if name in skip_fields:
+                continue
             field = getattr(cls, name)
             if isinstance(field, RelationshipField):
                 value = []
@@ -453,31 +465,36 @@ def get_null_values(cls):
         return null_values
 
     def to_dict(self, **kwargs):
-        __depth = kwargs.get('__depth')
-        depth_reached = __depth is not None and __depth <= 0
-
-        def _process(key, val):
-            is_doc = isinstance(val, mongo.Document)
-            include = key in self._nested_relationships
-            if is_doc and (not include or depth_reached):
-                val = getattr(val, val.pk_field(), None)
-            return val
-
-        _data = {}
-        for attr, field_type in self._fields.items():
+        _depth = kwargs.get('_depth')
+        if _depth is None:
+            _depth = self._nesting_depth
+        depth_reached = _depth is not None and _depth <= 0
+
+        _data = dictset()
+        for field, field_type in self._fields.items():
             # Ignore ForeignKeyField fields
             if isinstance(field_type, ForeignKeyField):
                 continue
-            value = getattr(self, attr, None)
-            if isinstance(value, list):
-                value = [_process(attr, v) for v in value]
-            else:
-                value = _process(attr, value)
-            _data[attr] = value
-        _dict = DataProxy(_data).to_dict(**kwargs)
-        _dict['_type'] = self._type
-        _dict['_pk'] = str(getattr(self, self.pk_field()))
-        return _dict
+            value = getattr(self, field, None)
+
+            if value is not None:
+                include = field in self._nested_relationships
+                if not include or depth_reached:
+                    encoder = lambda v: getattr(v, v.pk_field(), None)
+                else:
+                    encoder = lambda v: v.to_dict(_depth=_depth-1)
+
+                if isinstance(field_type, ReferenceField):
+                    value = encoder(value)
+                elif isinstance(field_type, RelationshipField):
+                    value = [encoder(val) for val in value]
+                elif hasattr(value, 'to_dict'):
+                    value = value.to_dict(_depth=_depth-1)
+
+            _data[field] = value
+        _data['_type'] = self._type
+        _data['_pk'] = str(getattr(self, self.pk_field()))
+        return _data
 
     def get_related_documents(self, nested_only=False):
         """ Return pairs of (Model, istances) of relationship fields.
@@ -621,6 +638,9 @@ def _is_modified(self):
         modified = bool(self._get_changed_fields())
         return modified
 
+    def _is_created(self):
+        return self._created
+
 
 class BaseDocument(six.with_metaclass(DocumentMetaclass,
                                       BaseMixin, mongo.Document)):

nefertari_mongodb/fields.py

@@ -392,7 +392,6 @@ class ReferenceField(BaseFieldMixin, fields.ReferenceField):
     """
     _valid_kwargs = (
         'document_type', 'dbref', 'reverse_delete_rule',
-        'before_validation', 'after_validation',
     )
     _kwargs_prefix = 'ref_'
     _backref_prefix = 'backref_'
@@ -577,8 +576,8 @@ class RelationshipField(BaseFieldMixin, fields.ListField):
         'db_field', 'required', 'default', 'unique',
         'unique_with', 'primary_key', 'validation', 'choices',
         'verbose_name', 'help_text', 'sparse',
-        'before_validation', 'after_validation',
     )
+
     _backref_prefix = 'backref_'
     reverse_rel_field = None
 
@@ -690,14 +689,5 @@ def Relationship(**kwargs):
     switching between the sqla and mongo engines.
     """
     uselist = kwargs.pop('uselist', True)
-    # many-to-one
-    if uselist:
-        return RelationshipField(**kwargs)
-    # one-to-one
-    else:
-        fields = ReferenceField._valid_kwargs + (
-            'document', 'ondelete')
-        fields += tuple(ReferenceField._backref_prefix + f for f in fields)
-        fields += (ReferenceField._backref_prefix + 'name',)
-        ref_kwargs = {f: kwargs.get(f) for f in fields if f in kwargs}
-        return ReferenceField(**ref_kwargs)
+    field_cls = RelationshipField if uselist else ReferenceField
+    return field_cls(**kwargs)

nefertari_mongodb/tests/test_documents.py

@@ -1,5 +1,5 @@
 import pytest
-from mock import patch, Mock, call
+from mock import patch, Mock
 
 import mongoengine as mongo
 from mongoengine.errors import FieldDoesNotExist
@@ -67,20 +67,23 @@ class TestBaseMixin(object):
     @patch('nefertari.elasticsearch.engine')
     def test_get_es_mapping(self, mock_conv):
         class MyModel(docs.BaseDocument):
+            _nested_relationships = ['parent']
+            _nesting_depth = 0
             my_id = fields.IdField()
             name = fields.StringField(primary_key=True)
             status = fields.ChoiceField(choices=['active'])
             groups = fields.ListField(item_type=fields.IntegerField)
 
         class MyModel2(docs.BaseDocument):
             _nested_relationships = ['child']
-            __tablename__ = 'mymodel2'
+            _nesting_depth = 1
             name = fields.StringField(primary_key=True)
             child = fields.Relationship(
                 document='MyModel', backref_name='parent',
                 uselist=False, backref_uselist=False)
 
-        assert MyModel.get_es_mapping() == {
+        mymodel_mapping = MyModel.get_es_mapping()
+        assert mymodel_mapping == {
             'MyModel': {
                 'properties': {
                     '_pk': {'type': 'string'},
@@ -94,13 +97,18 @@ class MyModel2(docs.BaseDocument):
             }
         }
 
-        assert MyModel2.get_es_mapping() == {
+        mymodel2_mapping = MyModel2.get_es_mapping()
+        child_props = mymodel_mapping['MyModel']['properties']
+        assert mymodel2_mapping == {
             'MyModel2': {
                 'properties': {
                     '_pk': {'type': 'string'},
                     '_version': {'type': 'long'},
                     'name': {'type': 'string'},
-                    'child': {'type': 'object'},
+                    'child': {
+                        'type': 'object',
+                        'properties': child_props
+                    },
                 }
             }
         }
@@ -188,6 +196,13 @@ def test_is_modified(self):
         obj.id = 1
         assert obj._is_modified()
 
+    def test_is_created(self):
+        obj = docs.BaseMixin()
+        obj._created = True
+        assert obj._is_created()
+        obj._created = False
+        assert not obj._is_created()
+
 
 class TestBaseDocument(object):
 
@@ -217,13 +232,11 @@ class MyModel2(docs.BaseDocument):
                 document='MyModel1', backref_name='model2')
 
         assert MyModel1.get_null_values() == {
-            '_version': None,
             'name': None,
             'model2': None,
         }
 
         assert MyModel2.get_null_values() == {
-            '_version': None,
             'models1': [],
             'name': None,
         }