migrate fts/haystack + add/del logic SERVER-8791 SERVER-9164 SERVER-9165

Author: Hari Khalsa

src/mongo/SConscript

@@ -398,8 +398,10 @@ serverOnlyFiles = [ "db/curop.cpp",
                     "db/index/btree_index_cursor.cpp",
                     "db/index/btree_interface.cpp",
                     "db/index/btree_key_generator.cpp",
+                    "db/index/fts_access_method.cpp",
                     "db/index/hash_access_method.cpp",
                     "db/index/hash_index_cursor.cpp",
+                    "db/index/haystack_access_method.cpp",
                     "db/index/s2_access_method.cpp",
                     "db/index/s2_index_cursor.cpp",
                     "db/index/s2_near_cursor.cpp",

src/mongo/db/btreecursor.cpp

@@ -66,7 +66,8 @@ namespace mongo {
         _order = indexDetails.keyPattern();
     }
 
-    void BtreeCursor::init( const BSONObj& sk, const BSONObj& ek, bool endKeyInclusive, int direction ) {
+    void BtreeCursor::init(const BSONObj& sk, const BSONObj& ek, bool endKeyInclusive,
+                           int direction ) {
         _finishConstructorInit();
         startKey = sk;
         endKey = ek;

src/mongo/db/geo/haystack.cpp

@@ -1,3 +1,4 @@
+// XXX THIS FILE IS DEPRECATED.  PLEASE DON'T MODIFY.
 // db/geo/haystack.cpp
 
 /**
@@ -23,6 +24,10 @@
 #include "mongo/db/auth/action_set.h"
 #include "mongo/db/auth/action_type.h"
 #include "mongo/db/auth/privilege.h"
+#include "mongo/db/index/catalog_hack.h"
+#include "mongo/db/index/haystack_access_method.h"
+#include "mongo/db/index/index_descriptor.h"
+#include "mongo/db/index/index_access_method.h"
 #include "mongo/db/namespace-inl.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/index.h"
@@ -343,13 +348,6 @@ namespace mongo {
                 return false;
             }
 
-            int idxNum = idxs[0];
-
-            IndexDetails& id = nsd->idx(idxNum);
-            GeoHaystackSearchIndex *si =
-                static_cast<GeoHaystackSearchIndex*>(id.getSpec().getType());
-            verify(&id == si->getDetails());
-
             BSONElement nearElt = cmdObj["near"];
             BSONElement maxDistance = cmdObj["maxDistance"];
             BSONElement search = cmdObj["search"];
@@ -362,8 +360,20 @@ namespace mongo {
             if (cmdObj["limit"].isNumber())
                 limit = static_cast<unsigned>(cmdObj["limit"].numberInt());
 
-            si->searchCommand(nsd, nearElt.Obj(), maxDistance.numberDouble(), search.Obj(),
-                              result, limit);
+            int idxNum = idxs[0];
+            IndexDetails& id = nsd->idx(idxNum);
+            if (CatalogHack::testIndexMigration()) {
+                auto_ptr<IndexDescriptor> desc(CatalogHack::getDescriptor(nsd, idxNum));
+                auto_ptr<HaystackAccessMethod> ham(new HaystackAccessMethod(desc.get()));
+                ham->searchCommand(nearElt.Obj(), maxDistance.numberDouble(), search.Obj(),
+                                   &result, limit);
+            } else {
+                GeoHaystackSearchIndex *si =
+                    static_cast<GeoHaystackSearchIndex*>(id.getSpec().getType());
+                verify(&id == si->getDetails());
+                si->searchCommand(nsd, nearElt.Obj(), maxDistance.numberDouble(), search.Obj(),
+                                  result, limit);
+            }
             return 1;
         }
     } nameSearchCommand;

src/mongo/db/index/btree_access_method.cpp

@@ -52,8 +52,9 @@ namespace mongo {
                                       options.dupsAllowed, _descriptor->getOnDisk(), true);
                 ++*numInserted;
             } catch (AssertionException& e) {
-                if (10287 == e.getCode() && options.dupsAllowed) {
-                    // Duplicate key, but our options say to ignore it.
+                if (10287 == e.getCode() && _descriptor->isBackgroundIndex()) {
+                    // This is the duplicate key exception.  We ignore it for some reason in BG
+                    // indexing.
                     DEV log() << "info: key already in index during bg indexing (ok)\n";
                 } else if (!options.dupsAllowed) {
                     // Assuming it's a duplicate key exception.  Clean up any inserted keys.
@@ -71,6 +72,10 @@ namespace mongo {
             }
         }
 
+        if (*numInserted > 1) {
+            _descriptor->setMultikey();
+        }
+
         return ret;
     }
 
@@ -161,8 +166,6 @@ namespace mongo {
         data->loc = record;
         data->dupsAllowed = options.dupsAllowed;
 
-        status->_isMultiKey = data->newKeys.size() > 1;
-
         setDifference(data->oldKeys, data->newKeys, &data->removed);
         setDifference(data->newKeys, data->oldKeys, &data->added);
 
@@ -193,6 +196,10 @@ namespace mongo {
         BtreeBasedPrivateUpdateData* data =
             static_cast<BtreeBasedPrivateUpdateData*>(ticket._indexSpecificUpdateData.get());
 
+        if (data->oldKeys.size() + data->added.size() - data->removed.size() > 1) {
+            _descriptor->setMultikey();
+        }
+
         for (size_t i = 0; i < data->added.size(); ++i) {
             _interface->bt_insert(_descriptor->getHead(), data->loc, *data->added[i], _ordering,
                                   data->dupsAllowed, _descriptor->getOnDisk(), true);

src/mongo/db/index/catalog_hack.h

@@ -15,7 +15,9 @@
 */
 
 #include "mongo/db/index/btree_access_method.h"
+#include "mongo/db/index/fts_access_method.h"
 #include "mongo/db/index/hash_access_method.h"
+#include "mongo/db/index/haystack_access_method.h"
 #include "mongo/db/index/index_access_method.h"
 #include "mongo/db/index/index_descriptor.h"
 #include "mongo/db/index/s2_access_method.h"
@@ -35,15 +37,28 @@ namespace mongo {
 
         static bool isIndexMigrated(const BSONObj& keyPattern) {
             string type = KeyPattern::findPluginName(keyPattern);
-            return "hashed" == type || "2dsphere" == type;
+            return "" == type || "fts" == type || "hashed" == type || "2dsphere" == type
+                   || "geoHaystack" == type;
         }
 
-        static IndexAccessMethod* getSpecialIndex(IndexDescriptor* desc) {
+        // If true, use EmulatedCursor + IndexCursor when answering "special" queries.
+        static bool testCursorMigration() { return true; }
+
+        // If true, use IndexAccessMethod for insert/delete when possible.
+        static bool testIndexMigration() { return true; }
+
+        static IndexAccessMethod* getIndex(IndexDescriptor* desc) {
             string type = KeyPattern::findPluginName(desc->keyPattern());
             if ("hashed" == type) {
                 return new HashAccessMethod(desc);
             } else if ("2dsphere" == type) {
                 return new S2AccessMethod(desc);
+            } else if ("fts" == type) {
+                return new FTSAccessMethod(desc);
+            } else if ("geoHaystack" == type) {
+                return new HaystackAccessMethod(desc);
+            } else if ("" == type) {
+                return new BtreeAccessMethod(desc);
             } else {
                 verify(0);
                 return NULL;

src/mongo/db/index/emulated_cursor.h

@@ -50,12 +50,12 @@ namespace mongo {
                                     int numWanted,
                                     const BSONObj& keyPattern) {
 
-            EmulatedCursor* ret = new EmulatedCursor(descriptor, indexAccessMethod,
-                                                     order, numWanted, keyPattern);
+            auto_ptr<EmulatedCursor> ret(new EmulatedCursor(descriptor, indexAccessMethod,
+                                                            order, numWanted, keyPattern));
             // Why do we have to do this?  No reading from disk is allowed in constructors,
             // and seeking involves reading from disk.
             ret->seek(query);
-            return ret;
+            return ret.release();
         }
 
         // We defer everything we can to the underlying cursor.

src/mongo/db/index/fts_access_method.cpp

@@ -0,0 +1,33 @@
+/**
+*    Copyright (C) 2013 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include "mongo/db/index/fts_access_method.h"
+#include "mongo/db/fts/fts_index_format.h"
+
+namespace mongo {
+
+    FTSAccessMethod::FTSAccessMethod(IndexDescriptor* descriptor)
+        : BtreeBasedAccessMethod(descriptor), _ftsSpec(descriptor->infoObj()) { }
+
+    void FTSAccessMethod::getKeys(const BSONObj& obj, BSONObjSet* keys) {
+        fts::FTSIndexFormat::getKeys(_ftsSpec, obj, keys);
+    }
+
+    Status FTSAccessMethod::newCursor(IndexCursor** out) {
+        return Status::OK();
+    }
+
+}  // namespace mongo

src/mongo/db/index/fts_access_method.h

@@ -0,0 +1,42 @@
+/**
+*    Copyright (C) 2013 10gen Inc.
+*
+*    This program is free software: you can redistribute it and/or  modify
+*    it under the terms of the GNU Affero General Public License, version 3,
+*    as published by the Free Software Foundation.
+*
+*    This program is distributed in the hope that it will be useful,
+*    but WITHOUT ANY WARRANTY; without even the implied warranty of
+*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*    GNU Affero General Public License for more details.
+*
+*    You should have received a copy of the GNU Affero General Public License
+*    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+
+#include "mongo/base/status.h"
+#include "mongo/db/fts/fts_spec.h"
+#include "mongo/db/index/index_descriptor.h"
+#include "mongo/db/index/btree_access_method_internal.h"
+#include "mongo/db/jsobj.h"
+
+namespace mongo {
+
+    class FTSAccessMethod : public BtreeBasedAccessMethod {
+    public:
+        FTSAccessMethod(IndexDescriptor* descriptor);
+        virtual ~FTSAccessMethod() { }
+
+        // Not implemented:
+        virtual Status newCursor(IndexCursor** out);
+
+    private:
+        // Implemented:
+        virtual void getKeys(const BSONObj& obj, BSONObjSet* keys);
+
+        fts::FTSSpec _ftsSpec;
+    };
+
+} //namespace mongo

src/mongo/db/index/hash_access_method.h

@@ -35,6 +35,7 @@ namespace mongo {
         using BtreeBasedAccessMethod::_ordering;
 
         HashAccessMethod(IndexDescriptor* descriptor);
+        virtual ~HashAccessMethod() { }
 
         virtual Status newCursor(IndexCursor** out);