SERVER-9306 Ability to temporarily forbid query optimizer from using index ("Hidden Index") SERVER-47275 Take over and complete Hidden Indexes PR

Co-authored-by: Ruoxin Xu <[email protected]>

Author: 2 people

jstests/core/hidden_index.js

@@ -0,0 +1,126 @@
+/**
+ * Test expected behavior for hidden indexes. A hidden index is invisible to the query planner so
+ * it will not be used in planning. It is handled in the same way as other indexes by the index
+ * catalog and for TTL purposes.
+ * @tags: [
+ *  multiversion_incompatible,
+ *  requires_fcv_44,
+ *  requires_non_retryable_commands,    // CollMod is not retryable.
+ * ]
+ */
+
+(function() {
+'use strict';
+load("jstests/libs/analyze_plan.js");              // For getPlanStages.
+load("jstests/libs/collection_drop_recreate.js");  // For assert[Drop|Create]Collection.
+load("jstests/libs/fixture_helpers.js");           // For FixtureHelpers.
+load("jstests/libs/get_index_helpers.js");         // For GetIndexHelpers.findByName.
+
+const collName = "hidden_index";
+let coll = assertDropAndRecreateCollection(db, collName);
+
+function numOfUsedIXSCAN(query) {
+    const explain = assert.commandWorked(coll.find(query).explain());
+    const ixScans = getPlanStages(explain.queryPlanner.winningPlan, "IXSCAN");
+    return ixScans.length;
+}
+
+function validateHiddenIndexBehaviour(query, index_type, wildcard) {
+    let index_name;
+    if (wildcard)
+        index_name = 'a.$**_' + index_type;
+    else
+        index_name = 'a_' + index_type;
+
+    if (wildcard)
+        assert.commandWorked(coll.createIndex({"a.$**": index_type}));
+    else
+        assert.commandWorked(coll.createIndex({"a": index_type}));
+
+    let idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name);
+    assert.eq(idxSpec.hidden, undefined);
+    assert.gt(numOfUsedIXSCAN(query), 0);
+
+    assert.commandWorked(coll.hideIndex(index_name));
+    idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name);
+    assert(idxSpec.hidden);
+    if (index_type === "text") {
+        assert.commandFailedWithCode(coll.runCommand("find", {filter: query}, {hint: {a: 1}}), 291);
+        assert.commandWorked(coll.dropIndexes());
+        return;
+    }
+    assert.eq(numOfUsedIXSCAN(query), 0);
+
+    assert.commandWorked(coll.unhideIndex(index_name));
+    idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name);
+    assert.eq(idxSpec.hidden, undefined);
+    assert.gt(numOfUsedIXSCAN(query), 0);
+
+    assert.commandWorked(coll.dropIndex(index_name));
+
+    if (wildcard)
+        assert.commandWorked(coll.createIndex({"a.$**": index_type}, {hidden: true}));
+    else
+        assert.commandWorked(coll.createIndex({"a": index_type}, {hidden: true}));
+
+    idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), index_name);
+    assert(idxSpec.hidden);
+    assert.eq(numOfUsedIXSCAN(query), 0);
+    assert.commandWorked(coll.dropIndexes());
+}
+
+// Normal index testing.
+validateHiddenIndexBehaviour({a: 1}, 1);
+
+// GEO index testing.
+validateHiddenIndexBehaviour({
+    a: {$geoWithin: {$geometry: {type: "Polygon", coordinates: [[[0, 0], [3, 6], [6, 1], [0, 0]]]}}}
+},
+                             "2dsphere");
+
+// Fts index.
+validateHiddenIndexBehaviour({$text: {$search: "java"}}, "text");
+
+// Wildcard index.
+validateHiddenIndexBehaviour({"a.f": 1}, 1, true);
+
+// Hidden index on capped collection.
+if (!FixtureHelpers.isMongos(db)) {
+    coll = assertDropAndRecreateCollection(db, collName, {capped: true, size: 100});
+    validateHiddenIndexBehaviour({a: 1}, 1);
+    coll = assertDropAndRecreateCollection(db, collName);
+}
+// Test that index 'hidden' status can be found in listIndexes command.
+assert.commandWorked(coll.createIndex({lsIdx: 1}, {hidden: true}));
+let res = assert.commandWorked(db.runCommand({"listIndexes": collName}));
+let idxSpec = GetIndexHelpers.findByName(res.cursor.firstBatch, "lsIdx_1");
+assert.eq(idxSpec.hidden, true);
+
+// Can't hide any index in a system collection.
+const systemColl = db.getSiblingDB('admin').system.version;
+assert.commandWorked(systemColl.createIndex({a: 1}));
+assert.commandFailedWithCode(systemColl.hideIndex("a_1"), 2);
+assert.commandFailedWithCode(systemColl.createIndex({a: 1}, {hidden: true}), 2);
+
+// Can't hide the '_id' index.
+assert.commandFailed(coll.hideIndex("_id_"));
+
+// Can't 'hint' a hidden index.
+assert.commandWorked(coll.createIndex({"a": 1}, {"hidden": true}));
+assert.commandFailedWithCode(coll.runCommand("find", {hint: {a: 1}}), 2);
+
+// We can change ttl index and hide info at the same time.
+assert.commandWorked(coll.dropIndexes());
+assert.commandWorked(coll.createIndex({"tm": 1}, {expireAfterSeconds: 10}));
+idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), "tm_1");
+assert.eq(idxSpec.hidden, undefined);
+assert.eq(idxSpec.expireAfterSeconds, 10);
+
+db.runCommand({
+    "collMod": coll.getName(),
+    "index": {"name": "tm_1", "expireAfterSeconds": 1, "hidden": true}
+});
+idxSpec = GetIndexHelpers.findByName(coll.getIndexes(), "tm_1");
+assert(idxSpec.hidden);
+assert.eq(idxSpec.expireAfterSeconds, 1);
+})();

jstests/noPassthrough/ttl_hidden_index.js

@@ -0,0 +1,29 @@
+// Make sure the TTL index still work after we hide it
+(function() {
+"use strict";
+let runner = MongoRunner.runMongod({setParameter: "ttlMonitorSleepSecs=1"});
+let coll = runner.getDB("test").ttl_hiddenl_index;
+coll.drop();
+
+// Create TTL index.
+assert.commandWorked(coll.ensureIndex({x: 1}, {expireAfterSeconds: 0}));
+let now = new Date();
+
+assert.commandWorked(coll.hideIndex("x_1"));
+
+// Insert docs after having set hidden index in order to prevent inserted docs being expired out
+// before the hidden index is set.
+assert.commandWorked(coll.insert({x: now}));
+assert.commandWorked(coll.insert({x: now}));
+
+// Wait for the TTL monitor to run at least twice (in case we weren't finished setting up our
+// collection when it ran the first time).
+var ttlPass = coll.getDB().serverStatus().metrics.ttl.passes;
+assert.soon(function() {
+    return coll.getDB().serverStatus().metrics.ttl.passes >= ttlPass + 2;
+}, "TTL monitor didn't run before timing out.");
+
+assert.eq(coll.count(), 0, "We should get 0 documents after TTL monitor run");
+
+MongoRunner.stopMongod(runner);
+})();

src/mongo/db/auth/auth_op_observer.cpp

@@ -116,11 +116,11 @@ void AuthOpObserver::onCollMod(OperationContext* opCtx,
                                OptionalCollectionUUID uuid,
                                const BSONObj& collModCmd,
                                const CollectionOptions& oldCollOptions,
-                               boost::optional<TTLCollModInfo> ttlInfo) {
+                               boost::optional<IndexCollModInfo> indexInfo) {
     const auto cmdNss = nss.getCommandNS();
 
     // Create the 'o' field object.
-    const auto cmdObj = makeCollModCmdObj(collModCmd, oldCollOptions, ttlInfo);
+    const auto cmdObj = makeCollModCmdObj(collModCmd, oldCollOptions, indexInfo);
 
     AuthorizationManager::get(opCtx->getServiceContext())
         ->logOp(opCtx, "c", cmdNss, cmdObj, nullptr);

src/mongo/db/auth/auth_op_observer.h

@@ -113,7 +113,7 @@ class AuthOpObserver final : public OpObserver {
                    OptionalCollectionUUID uuid,
                    const BSONObj& collModCmd,
                    const CollectionOptions& oldCollOptions,
-                   boost::optional<TTLCollModInfo> ttlInfo) final;
+                   boost::optional<IndexCollModInfo> indexInfo) final;
 
     void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final;
 

src/mongo/db/catalog/coll_mod.cpp

@@ -69,6 +69,7 @@ MONGO_FAIL_POINT_DEFINE(assertAfterIndexUpdate);
 struct CollModRequest {
     const IndexDescriptor* idx = nullptr;
     BSONElement indexExpireAfterSeconds = {};
+    BSONElement indexHidden = {};
     BSONElement viewPipeLine = {};
     std::string viewOn = {};
     BSONElement collValidator = {};
@@ -125,14 +126,18 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx,
             }
 
             cmr.indexExpireAfterSeconds = indexObj["expireAfterSeconds"];
-            if (cmr.indexExpireAfterSeconds.eoo()) {
-                return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field");
+            cmr.indexHidden = indexObj["hidden"];
+
+            if (cmr.indexExpireAfterSeconds.eoo() && cmr.indexHidden.eoo()) {
+                return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds or hidden field");
             }
-            if (!cmr.indexExpireAfterSeconds.isNumber()) {
+            if (!cmr.indexExpireAfterSeconds.eoo() && !cmr.indexExpireAfterSeconds.isNumber()) {
                 return Status(ErrorCodes::InvalidOptions,
                               "expireAfterSeconds field must be a number");
             }
-
+            if (!cmr.indexHidden.eoo() && !cmr.indexHidden.isBoolean()) {
+                return Status(ErrorCodes::InvalidOptions, "hidden field must be a boolean");
+            }
             if (!indexName.empty()) {
                 cmr.idx = coll->getIndexCatalog()->findIndexByName(opCtx, indexName);
                 if (!cmr.idx) {
@@ -161,15 +166,17 @@ StatusWith<CollModRequest> parseCollModRequest(OperationContext* opCtx,
                 cmr.idx = indexes[0];
             }
 
-            BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds");
-            if (oldExpireSecs.eoo()) {
-                return Status(ErrorCodes::InvalidOptions, "no expireAfterSeconds field to update");
-            }
-            if (!oldExpireSecs.isNumber()) {
-                return Status(ErrorCodes::InvalidOptions,
-                              "existing expireAfterSeconds field is not a number");
+            if (!cmr.indexExpireAfterSeconds.eoo()) {
+                BSONElement oldExpireSecs = cmr.idx->infoObj().getField("expireAfterSeconds");
+                if (oldExpireSecs.eoo()) {
+                    return Status(ErrorCodes::InvalidOptions,
+                                  "no expireAfterSeconds field to update");
+                }
+                if (!oldExpireSecs.isNumber()) {
+                    return Status(ErrorCodes::InvalidOptions,
+                                  "existing expireAfterSeconds field is not a number");
+                }
             }
-
         } else if (fieldName == "validator" && !isView) {
             // Save this to a variable to avoid reading the atomic variable multiple times.
             const auto currentFCV = serverGlobalParams.featureCompatibility.getVersion();
@@ -250,20 +257,35 @@ class CollModResultChange : public RecoveryUnit::Change {
 public:
     CollModResultChange(const BSONElement& oldExpireSecs,
                         const BSONElement& newExpireSecs,
+                        const BSONElement& oldHidden,
+                        const BSONElement& newHidden,
                         BSONObjBuilder* result)
-        : _oldExpireSecs(oldExpireSecs), _newExpireSecs(newExpireSecs), _result(result) {}
+        : _oldExpireSecs(oldExpireSecs),
+          _newExpireSecs(newExpireSecs),
+          _oldHidden(oldHidden),
+          _newHidden(newHidden),
+          _result(result) {}
 
     void commit(boost::optional<Timestamp>) override {
         // add the fields to BSONObjBuilder result
-        _result->appendAs(_oldExpireSecs, "expireAfterSeconds_old");
-        _result->appendAs(_newExpireSecs, "expireAfterSeconds_new");
+        if (!_oldExpireSecs.eoo()) {
+            _result->appendAs(_oldExpireSecs, "expireAfterSeconds_old");
+            _result->appendAs(_newExpireSecs, "expireAfterSeconds_new");
+        }
+        if (!_newHidden.eoo()) {
+            bool oldValue = _oldHidden.eoo() ? false : _oldHidden.booleanSafe();
+            _result->append("hidden_old", oldValue);
+            _result->appendAs(_newHidden, "hidden_new");
+        }
     }
 
     void rollback() override {}
 
 private:
     const BSONElement _oldExpireSecs;
     const BSONElement _newExpireSecs;
+    const BSONElement _oldHidden;
+    const BSONElement _newHidden;
     BSONObjBuilder* _result;
 };
 
@@ -326,6 +348,19 @@ Status _collModInternal(OperationContext* opCtx,
     const CollModRequest cmrOld = statusW.getValue();
     CollModRequest cmrNew = statusW.getValue();
 
+    if (!cmrOld.indexHidden.eoo()) {
+
+        if (serverGlobalParams.featureCompatibility.getVersion() <
+                ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo46 &&
+            cmrOld.indexHidden.booleanSafe()) {
+            return Status(ErrorCodes::BadValue, "Hidden indexes can only be created with FCV 4.6");
+        }
+        if (coll->ns().isSystem())
+            return Status(ErrorCodes::BadValue, "Can't hide index on system collection");
+        if (cmrOld.idx->isIdIndex())
+            return Status(ErrorCodes::BadValue, "can't hide _id index");
+    }
+
     return writeConflictRetry(opCtx, "collMod", nss.ns(), [&] {
         WriteUnitOfWork wunit(opCtx);
 
@@ -362,40 +397,64 @@ Status _collModInternal(OperationContext* opCtx,
         CollectionOptions oldCollOptions =
             DurableCatalog::get(opCtx)->getCollectionOptions(opCtx, coll->getCatalogId());
 
-        boost::optional<TTLCollModInfo> ttlInfo;
+        boost::optional<IndexCollModInfo> indexCollModInfo;
 
         // Handle collMod operation type appropriately.
 
-        // TTLIndex
-        if (!cmrOld.indexExpireAfterSeconds.eoo()) {
-            BSONElement newExpireSecs = cmrOld.indexExpireAfterSeconds;
-            BSONElement oldExpireSecs = cmrOld.idx->infoObj().getField("expireAfterSeconds");
-
-            if (SimpleBSONElementComparator::kInstance.evaluate(oldExpireSecs != newExpireSecs)) {
-                // Change the value of "expireAfterSeconds" on disk.
-                DurableCatalog::get(opCtx)->updateTTLSetting(opCtx,
-                                                             coll->getCatalogId(),
-                                                             cmrOld.idx->indexName(),
-                                                             newExpireSecs.safeNumberLong());
-
-                // Notify the index catalog that the definition of this index changed. This will
-                // invalidate the idx pointer in cmrOld. On rollback of this WUOW, the idx pointer
-                // in cmrNew will be invalidated and the idx pointer in cmrOld will be valid again.
-                cmrNew.idx = coll->getIndexCatalog()->refreshEntry(opCtx, cmrOld.idx);
-                opCtx->recoveryUnit()->registerChange(
-                    std::make_unique<CollModResultChange>(oldExpireSecs, newExpireSecs, result));
-
-                if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) {
-                    LOGV2(20307, "collMod - assertAfterIndexUpdate fail point enabled.");
-                    uasserted(50970, "trigger rollback after the index update");
+        if (!cmrOld.indexExpireAfterSeconds.eoo() || !cmrOld.indexHidden.eoo()) {
+            BSONElement newExpireSecs = {};
+            BSONElement oldExpireSecs = {};
+            BSONElement newHidden = {};
+            BSONElement oldHidden = {};
+            // TTL Index
+            if (!cmrOld.indexExpireAfterSeconds.eoo()) {
+                newExpireSecs = cmrOld.indexExpireAfterSeconds;
+                oldExpireSecs = cmrOld.idx->infoObj().getField("expireAfterSeconds");
+                if (SimpleBSONElementComparator::kInstance.evaluate(oldExpireSecs !=
+                                                                    newExpireSecs)) {
+                    // Change the value of "expireAfterSeconds" on disk.
+                    DurableCatalog::get(opCtx)->updateTTLSetting(opCtx,
+                                                                 coll->getCatalogId(),
+                                                                 cmrOld.idx->indexName(),
+                                                                 newExpireSecs.safeNumberLong());
+                }
+            }
+
+            // User wants to hide or unhide index.
+            if (!cmrOld.indexHidden.eoo()) {
+                newHidden = cmrOld.indexHidden;
+                oldHidden = cmrOld.idx->infoObj().getField("hidden");
+                // Make sure when we set 'hidden' to false, we can remove the hidden field from
+                // catalog.
+                if (SimpleBSONElementComparator::kInstance.evaluate(oldHidden != newHidden)) {
+                    DurableCatalog::get(opCtx)->updateHiddenSetting(opCtx,
+                                                                    coll->getCatalogId(),
+                                                                    cmrOld.idx->indexName(),
+                                                                    newHidden.booleanSafe());
                 }
             }
 
 
-            // Save previous TTL index expiration.
-            ttlInfo = TTLCollModInfo{Seconds(newExpireSecs.safeNumberLong()),
-                                     Seconds(oldExpireSecs.safeNumberLong()),
-                                     cmrNew.idx->indexName()};
+            indexCollModInfo = IndexCollModInfo{
+                cmrOld.indexExpireAfterSeconds.eoo() ? boost::optional<Seconds>()
+                                                     : Seconds(newExpireSecs.safeNumberLong()),
+                cmrOld.indexExpireAfterSeconds.eoo() ? boost::optional<Seconds>()
+                                                     : Seconds(oldExpireSecs.safeNumberLong()),
+                cmrOld.indexHidden.eoo() ? boost::optional<bool>() : newHidden.booleanSafe(),
+                cmrOld.indexHidden.eoo() ? boost::optional<bool>() : oldHidden.booleanSafe(),
+                cmrNew.idx->indexName()};
+
+            // Notify the index catalog that the definition of this index changed. This will
+            // invalidate the idx pointer in cmrOld. On rollback of this WUOW, the idx pointer
+            // in cmrNew will be invalidated and the idx pointer in cmrOld will be valid again.
+            cmrNew.idx = coll->getIndexCatalog()->refreshEntry(opCtx, cmrOld.idx);
+            opCtx->recoveryUnit()->registerChange(std::make_unique<CollModResultChange>(
+                oldExpireSecs, newExpireSecs, oldHidden, newHidden, result));
+
+            if (MONGO_unlikely(assertAfterIndexUpdate.shouldFail())) {
+                LOGV2(20307, "collMod - assertAfterIndexUpdate fail point enabled.");
+                uasserted(50970, "trigger rollback after the index update");
+            }
         }
 
         // The Validator, ValidationAction and ValidationLevel are already parsed and must be OK.
@@ -412,8 +471,10 @@ Status _collModInternal(OperationContext* opCtx,
 
         // Only observe non-view collMods, as view operations are observed as operations on the
         // system.views collection.
+
         auto* const opObserver = opCtx->getServiceContext()->getOpObserver();
-        opObserver->onCollMod(opCtx, nss, coll->uuid(), oplogEntryObj, oldCollOptions, ttlInfo);
+        opObserver->onCollMod(
+            opCtx, nss, coll->uuid(), oplogEntryObj, oldCollOptions, indexCollModInfo);
 
         wunit.commit();
         return Status::OK();