SERVER-6718 new parameter to control replication index prefetching behavior

New parameter replIndexPrefetch.
Settable via the command line or use setParameter() command on a running server.
Only valid on a server started with --replSet parameter.
This allows a user to disable the prefetching of all index pages prior to the application
of a replicated op on a secondary.
For most use cases, prefetching all indexes for a given op will provide good performance.
For use cases that do many in-place updates to documents in a collection with many indexes
that are unaffected by such updates, using the '_id_only' setting may provide better performance
than the 'all' setting.  The '_id_only' setting only prefetches index pages for the _id index,
which will always be used to find the document needed to be updated.
The 'none' setting will be used for debugging only; there should be no production use cases for
this setting.
getParameter() supports getting the current value of this parameter.

Author: milkie

src/mongo/db/cmdline.h

@@ -68,6 +68,8 @@ namespace mongo {
         }
         bool usingReplSets() const { return !_replSet.empty(); }
 
+        string rsIndexPrefetch;// --indexPrefetch
+
         // for master/slave replication
         string source;         // --source
         string only;           // --only

src/mongo/db/db.cpp

@@ -701,6 +701,7 @@ static int mongoDbMain(int argc, char* argv[]) {
 
     rs_options.add_options()
     ("replSet", po::value<string>(), "arg is <setname>[/<optionalseedhostlist>]")
+    ("replIndexPrefetch", po::value<string>(), "specify index prefetching behavior (if secondary) [none|_id_only|all]")
     ;
 
     sharding_options.add_options()
@@ -959,6 +960,9 @@ static int mongoDbMain(int argc, char* argv[]) {
             /* seed list of hosts for the repl set */
             cmdLine._replSet = params["replSet"].as<string>().c_str();
         }
+        if (params.count("replIndexPrefetch")) {
+            cmdLine.rsIndexPrefetch = params["replIndexPrefetch"].as<std::string>();
+        }
         if (params.count("only")) {
             cmdLine.only = params["only"].as<string>().c_str();
         }

src/mongo/db/dbcommands.cpp

@@ -66,9 +66,46 @@ namespace mongo {
             dur::setAgeOutJournalFiles(r);
             return true;
         }
+        if( cmdObj.hasElement( "replIndexPrefetch" ) ) {
+            if (!theReplSet) {
+                errmsg = "replication is not enabled";
+                return false;
+            }
+            std::string prefetch = cmdObj["replIndexPrefetch"].valuestrsafe();
+            log() << "changing replication index prefetch behavior to " << prefetch << endl;
+            // default:
+            ReplSetImpl::IndexPrefetchConfig prefetchConfig = ReplSetImpl::PREFETCH_ALL;
+            if (prefetch == "none")
+                prefetchConfig = ReplSetImpl::PREFETCH_NONE;
+            else if (prefetch == "_id_only")
+                prefetchConfig = ReplSetImpl::PREFETCH_ID_ONLY;
+            else if (prefetch == "all")
+                prefetchConfig = ReplSetImpl::PREFETCH_ALL;
+            else {
+                warning() << "unrecognized indexPrefetch setting: " << prefetch << endl;
+            }
+            theReplSet->setIndexPrefetchConfig(prefetchConfig);
+            return true;
+        }
+
         return false;
     }
 
+    const char* fetchReplIndexPrefetchParam() {
+        if (!theReplSet) return "uninitialized";
+        ReplSetImpl::IndexPrefetchConfig ip = theReplSet->getIndexPrefetchConfig();
+        switch (ip) {
+        case ReplSetImpl::PREFETCH_NONE:
+            return "none";
+        case ReplSetImpl::PREFETCH_ID_ONLY:
+            return "_id_only";
+        case ReplSetImpl::PREFETCH_ALL:
+            return "all";
+        default:
+            return "invalid";
+        }
+    }
+
     /* reset any errors so that getlasterror comes back clean.
 
        useful before performing a long series of operations where we want to

src/mongo/db/dbcommands_generic.cpp

@@ -124,6 +124,8 @@ namespace mongo {
         */
     unsigned replApplyBatchSize = 1;
 
+    const char* fetchReplIndexPrefetchParam();
+
     class CmdGet : public Command {
     public:
         CmdGet() : Command( "getParameter" ) { }
@@ -160,7 +162,9 @@ namespace mongo {
             if( all || cmdObj.hasElement("replApplyBatchSize") ) {
                 result.append("replApplyBatchSize", replApplyBatchSize);
             }
-
+            if (all || cmdObj.hasElement("replIndexPrefetch")) {
+                result.append("replIndexPrefetch", fetchReplIndexPrefetchParam());
+            }
             if ( before == result.len() ) {
                 errmsg = "no option found to get";
                 return false;

src/mongo/db/prefetch.cpp

@@ -24,6 +24,7 @@
 #include "mongo/db/index_update.h"
 #include "mongo/db/jsobj.h"
 #include "mongo/db/namespace_details.h"
+#include "mongo/db/repl/rs.h"
 
 namespace mongo {
 
@@ -67,23 +68,58 @@ namespace mongo {
     void prefetchIndexPages(NamespaceDetails *nsd, const BSONObj& obj) {
         DiskLoc unusedDl; // unused
         IndexInterface::IndexInserter inserter;
-
-        // includes all indexes, including ones
-        // in the process of being built
-        int indexCount = nsd->nIndexesBeingBuilt(); 
         BSONObjSet unusedKeys;
-        for ( int indexNo = 0; indexNo < indexCount; indexNo++ ) {
-            // This will page in all index pages for the given object.
+        ReplSetImpl::IndexPrefetchConfig prefetchConfig = theReplSet->getIndexPrefetchConfig();
+        switch (prefetchConfig) {
+        case ReplSetImpl::PREFETCH_NONE:
+            return;
+        case ReplSetImpl::PREFETCH_ID_ONLY:
+        {
+            int indexNo = nsd->findIdIndex();
+            if (indexNo == -1) return;
             try {
-                fetchIndexInserters(/*out*/unusedKeys, inserter, nsd, indexNo, obj, unusedDl, /*allowDups*/true);
+                fetchIndexInserters(/*out*/unusedKeys, 
+                                    inserter, 
+                                    nsd, 
+                                    indexNo, 
+                                    obj, 
+                                    unusedDl, 
+                                    /*allowDups*/true);
             }
             catch (const DBException& e) {
                 LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl;
             }
-            unusedKeys.clear();
+            break;
+        }
+        case ReplSetImpl::PREFETCH_ALL:
+        {
+            // indexCount includes all indexes, including ones
+            // in the process of being built
+            int indexCount = nsd->nIndexesBeingBuilt(); 
+            for ( int indexNo = 0; indexNo < indexCount; indexNo++ ) {
+                // This will page in all index pages for the given object.
+                try {
+                    fetchIndexInserters(/*out*/unusedKeys, 
+                                        inserter, 
+                                        nsd, 
+                                        indexNo, 
+                                        obj, 
+                                        unusedDl, 
+                                        /*allowDups*/true);
+                }
+                catch (const DBException& e) {
+                    LOG(2) << "ignoring exception in prefetchIndexPages(): " << e.what() << endl;
+                }
+                unusedKeys.clear();
+            }
+            break;
+        }
+        default:
+            fassertFailed(16427);
         }
     }
 
+
     void prefetchRecordPages(const char* ns, const BSONObj& obj) {
         BSONElement _id;
         if( obj.getObjectID(_id) ) {

src/mongo/db/repl/rs.cpp

@@ -397,6 +397,21 @@ namespace mongo {
                 log() << "replSet warning command line seed " << i->toString() << " is not present in the current repl set config" << rsLog;
             }
         }
+
+        // Figure out indexPrefetch setting
+        std::string& prefetch = cmdLine.rsIndexPrefetch;
+        if (!prefetch.empty()) {
+            IndexPrefetchConfig prefetchConfig = PREFETCH_ALL;
+            if (prefetch == "none")
+                prefetchConfig = PREFETCH_NONE;
+            else if (prefetch == "_id_only")
+                prefetchConfig = PREFETCH_ID_ONLY;
+            else if (prefetch == "all")
+                prefetchConfig = PREFETCH_ALL;
+            else
+                warning() << "unrecognized indexPrefetch setting: " << prefetch << endl;
+            setIndexPrefetchConfig(prefetchConfig);
+        }
     }
 
     ReplSetImpl::ReplSetImpl() :

src/mongo/db/repl/rs.h

@@ -515,16 +515,32 @@ namespace mongo {
         threadpool::ThreadPool _prefetcherPool;
 
     public:
+        // Allow index prefetching to be turned on/off
+        enum IndexPrefetchConfig {
+            PREFETCH_NONE=0, PREFETCH_ID_ONLY=1, PREFETCH_ALL=2
+        };
+
+        void setIndexPrefetchConfig(const IndexPrefetchConfig cfg) {
+            _indexPrefetchConfig = cfg;
+        }
+        IndexPrefetchConfig getIndexPrefetchConfig() {
+            return _indexPrefetchConfig;
+        }
+            
         static const int replWriterThreadCount;
         static const int replPrefetcherThreadCount;
         threadpool::ThreadPool& getPrefetchPool() { return _prefetcherPool; }
         threadpool::ThreadPool& getWriterPool() { return _writerPool; }
 
+
         const ReplSetConfig::MemberCfg& myConfig() const { return _config; }
         bool tryToGoLiveAsASecondary(OpTime&); // readlocks
         void syncRollback(OplogReader& r);
         void syncThread();
         const OpTime lastOtherOpTime() const;
+
+    private:
+        IndexPrefetchConfig _indexPrefetchConfig;
     };
 
     class ReplSet : public ReplSetImpl {

src/mongo/s/commands_public.cpp

@@ -45,6 +45,10 @@ namespace mongo {
         return true;
     }
 
+    const char* fetchReplIndexPrefetchParam() { 
+        return "unsupported"; 
+    }
+
     namespace dbgrid_pub_cmds {
 
         class PublicGridCommand : public Command {