add hydratation feature

Author: jbdemonte

README.md

@@ -15,6 +15,7 @@ mongoose-elasticsearch-xp is a [mongoose](http://mongoosejs.com/) plugin that ca
   - [Indexing on demand](#indexing-on-demand)
 - [Mapping](#mapping)
   - [Creating mappings on-demand](#creating-mappings-on-demand)
+- [Hydration](#hydration)
 
 ## Why this plugin?
 
@@ -44,6 +45,7 @@ Options are:
 * `port` - the port Elasticsearch is running on
 * `auth` - the authentication needed to reach Elasticsearch server. In the standard format of 'username:password'
 * `protocol` - the protocol the Elasticsearch server uses. Defaults to http
+* `hydrate` - whether or not to replace ES source by mongo document
 
 
 To have a model indexed into Elasticsearch simply add the plugin.
@@ -295,6 +297,43 @@ User
 You'll have to manage whether or not you need to create the mapping, mongoose-elasticsearch-xp will make no assumptions and simply attempt to create the mapping. 
 If the mapping already exists, an Exception detailing such will be populated in the `err` argument. 
 
+## Hydration
+By default objects returned from performing a search will be the objects as is in Elasticsearch. 
+This is useful in cases where only what was indexed needs to be displayed (think a list of results) while the actual mongoose object contains the full data when viewing one of the results.
+
+However, if you want the results to be actual mongoose objects you can provide {hydrate:true} as the second argument to a search call.
+
+```javascript
+User
+  .esSearch({query_string: {query: "john"}}, {hydrate:true})
+  .then(function (results) {
+  // results here
+  });
+```
+
+To modify default hydratation, provide an object to `hydrate` instead of "true".
+`hydrate` accept {select: string, options: object, docsOnly: boolean}
+
+```javascript
+User
+  .esSearch({query_string: {query: "john"}}, {hydrate: {select: 'name age', options: {lean: true}}})
+  .then(function (results) {
+    // results here
+  });
+```
+
+When using hydration, `hits._source` is replaced by `hits.doc`.
+
+If you only want the models, instead of the complete ES results, use the option "docsOnly".
+
+```javascript
+User
+  .esSearch({query_string: {query: "john"}}, {hydrate: {select: 'name age', docsOnly; true}})
+  .then(function (users) {
+    // users is an array of User
+  });
+```
+
 
 [npm-url]: https://npmjs.org/package/mongoose-elasticsearch-xp
 [npm-image]: https://badge.fury.io/js/mongoose-elasticsearch-xp.svg

index.js

@@ -2,6 +2,7 @@ var generateMapping = require('./lib/mapping').generate;
 var client = require('./lib/client');
 var utils = require('./lib/utils');
 var Bulker = require('./lib/bulker');
+var mongoose = require('mongoose');
 
 
 module.exports = function (schema, options) {
@@ -146,7 +147,8 @@ function search(query, options, callback) {
   query = query || {};
   options = options || {};
 
-  var esOptions = this.esOptions();
+  var self = this;
+  var esOptions = self.esOptions();
   var params = {
     index: esOptions.index,
     type: esOptions.type
@@ -158,7 +160,51 @@ function search(query, options, callback) {
   } else {
     params.body = query.query ? query : {query: query};
   }
-  esOptions.client.search(params, defer.callback);
+  if (options.hydrate) {
+    params._source = false;
+  }
+  esOptions.client.search(params, function (err, result) {
+    if (err) {
+      return defer.reject(err);
+    }
+    if (!options.hydrate) {
+      return defer.resolve(result);
+    }
+    if (!result.hits.total) {
+      return defer.resolve(result);
+    }
+
+    var ids = result.hits.hits.map(function (hit) {
+      return mongoose.Types.ObjectId(hit._id);
+    });
+
+    var hydrate = options.hydrate || {};
+    var select = hydrate.select || null;
+    var opts = hydrate.options || null;
+    var docsOnly = hydrate.docsOnly || false;
+
+
+    self.find({_id: {$in: ids}}, select, opts, function (err, users) {
+      if (err) {
+        return defer.reject(err);
+      }
+      var userByIds = {};
+      users.forEach(function (user) {
+        userByIds[user._id] = user;
+      });
+      if (docsOnly) {
+        result = ids.map(function (id) {
+          return userByIds[id];
+        });
+      } else {
+        result.hits.hits.forEach(function (hit) {
+          hit.doc = userByIds[hit._id];
+        });
+      }
+      return defer.resolve(result);
+    });
+
+  });
 
   return defer.promise;
 }
@@ -187,7 +233,7 @@ function synchronize(conditions, projection, options, callback) {
     options = null;
   }
 
-  var schema = this;
+  var model = this;
   var defer = utils.defer(callback);
   var esOptions = this.esOptions();
   var batch = esOptions.bulk && esOptions.bulk.batch ? esOptions.bulk.batch : 50;
@@ -202,7 +248,7 @@ function synchronize(conditions, projection, options, callback) {
   }
 
   function onError(err) {
-    schema.emit('es-bulk-error', err);
+    model.emit('es-bulk-error', err);
     if (streamClosed) {
       finalize();
     } else {
@@ -211,7 +257,7 @@ function synchronize(conditions, projection, options, callback) {
   }
 
   function onSent(len) {
-    schema.emit('es-bulk-sent', len);
+    model.emit('es-bulk-sent', len);
     if (streamClosed) {
       finalize();
     } else {
@@ -228,7 +274,7 @@ function synchronize(conditions, projection, options, callback) {
       {index: {_index: esOptions.index, _type: esOptions.type, _id: doc._id.toString()}},
       utils.serialize(doc, esOptions.mapping)
     );
-    schema.emit('es-bulk-data', doc);
+    model.emit('es-bulk-data', doc);
     if (!sending) {
       stream.resume();
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "mongoose-elasticsearch-xp",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "A mongoose plugin that indexes models into elastic search (an alternative to mongoosastic)",
   "tags": [
     "mongodb",

test/hydratation.js

@@ -0,0 +1,189 @@
+var utils = require('./utils');
+var mongoose = require('mongoose');
+var plugin = require('../');
+
+describe("hydratation", function () {
+
+  utils.setup();
+
+  beforeEach(function (done) {
+
+    var UserSchema = new mongoose.Schema({
+      name: String,
+      age: Number
+    });
+
+    UserSchema.plugin(plugin);
+
+    var UserModel = mongoose.model('User', UserSchema);
+
+    var john = new UserModel({name: 'John', age: 35});
+    var jane = new UserModel({name: 'Jane', age: 34});
+    var bob = new UserModel({name: 'Bob', age: 36});
+
+    this.model = UserModel;
+    this.users = {
+      john: john,
+      jane: jane,
+      bob: bob
+    };
+
+    utils.deleteModelIndexes(UserModel)
+      .then(function () {
+        return UserModel.esCreateMapping();
+      })
+      .then(function () {
+        return utils.Promise.all([john, jane, bob].map(function (user) {
+          return new utils.Promise(function (resolve) {
+            user.on('es-indexed', resolve);
+            user.save();
+          });
+        }));
+      })
+      .then(function () {
+        return UserModel.esRefresh();
+      })
+      .then(function () {
+        done();
+      });
+  });
+
+  it('should hydrate results', function (done) {
+
+    var UserModel = this.model;
+    var john = this.users.john;
+    var bob = this.users.bob;
+
+    UserModel
+      .esSearch(
+        {
+          query: {match_all: {}},
+          sort: [
+            {age: {order: "desc"}}
+          ],
+          filter: {range: {age: {gte: 35}}}
+        },
+        {hydrate: true}
+      )
+      .then(function (result) {
+        var hit;
+        expect(result.hits.total).to.eql(2);
+
+        hit = result.hits.hits[0];
+        expect(hit._source).to.be.undefined;
+        expect(hit.doc).to.be.an.instanceof(UserModel);
+        expect(hit.doc._id.toString()).to.eql(bob._id.toString());
+        expect(hit.doc.name).to.eql(bob.name);
+        expect(hit.doc.age).to.eql(bob.age);
+
+        hit = result.hits.hits[1];
+        expect(hit._source).to.be.undefined;
+        expect(hit.doc).to.be.an.instanceof(UserModel);
+        expect(hit.doc._id.toString()).to.eql(john._id.toString());
+        expect(hit.doc.name).to.eql(john.name);
+        expect(hit.doc.age).to.eql(john.age);
+
+        done();
+      })
+      .catch(function (err) {
+        done(err);
+      });
+  });
+
+  it('should hydrate returning only models', function (done) {
+
+    var UserModel = this.model;
+    var john = this.users.john;
+    var bob = this.users.bob;
+
+    UserModel
+      .esSearch(
+        {
+          query: {match_all: {}},
+          sort: [
+            {age: {order: "desc"}}
+          ],
+          filter: {range: {age: {gte: 35}}}
+        },
+        {hydrate: {docsOnly: true}}
+      )
+      .then(function (users) {
+        var user;
+        expect(users.length).to.eql(2);
+
+        user = users[0];
+        expect(user._id.toString()).to.eql(bob._id.toString());
+        expect(user.name).to.eql(bob.name);
+        expect(user.age).to.eql(bob.age);
+
+        user = users[1];
+        expect(user._id.toString()).to.eql(john._id.toString());
+        expect(user.name).to.eql(john.name);
+        expect(user.age).to.eql(john.age);
+
+        done();
+      })
+      .catch(function (err) {
+        done(err);
+      });
+  });
+
+  it('should hydrate using projection', function (done) {
+
+    var UserModel = this.model;
+    var jane = this.users.jane;
+
+    return UserModel
+
+    .esSearch(
+      'name:jane',
+      {hydrate: {select: 'name'}}
+    )
+    .then(function (result) {
+      var hit;
+      expect(result.hits.total).to.eql(1);
+
+      hit = result.hits.hits[0];
+      expect(hit._source).to.be.undefined;
+      expect(hit.doc).to.be.an.instanceof(UserModel);
+      expect(hit.doc._id.toString()).to.eql(jane._id.toString());
+      expect(hit.doc.name).to.eql(jane.name);
+      expect(hit.doc.age).to.be.undefined;
+
+      done();
+    })
+    .catch(function (err) {
+      done(err);
+    });
+  });
+
+  it('should hydrate using options', function (done) {
+
+    var UserModel = this.model;
+    var jane = this.users.jane;
+
+    return UserModel
+
+    .esSearch(
+      'name:jane',
+      {hydrate: {options: {lean: true}}}
+    )
+    .then(function (result) {
+      var hit;
+      expect(result.hits.total).to.eql(1);
+
+      hit = result.hits.hits[0];
+      expect(hit._source).to.be.undefined;
+      expect(hit.doc).not.to.be.an.instanceof(UserModel);
+      expect(hit.doc._id.toString()).to.eql(jane._id.toString());
+      expect(hit.doc.name).to.eql(jane.name);
+      expect(hit.doc.age).to.eql(jane.age);
+
+      done();
+    })
+    .catch(function (err) {
+      done(err);
+    });
+  });
+
+});