SERVER-10026 stages can die natural deaths

Author: Hari Khalsa

src/mongo/db/exec/collection_scan.cpp

@@ -47,13 +47,13 @@ namespace mongo {
 
     PlanStage::StageState CollectionScan::work(WorkingSetID* out) {
         ++_commonStats.works;
-        if (_nsDropped) { return PlanStage::IS_EOF; }
+        if (_nsDropped) { return PlanStage::DEAD; }
 
         if (NULL == _iter) {
             Collection* collection = cc().database()->getCollection( _params.ns );
             if ( collection == NULL ) {
                 _nsDropped = true;
-                return PlanStage::IS_EOF;
+                return PlanStage::DEAD;
             }
 
             _iter.reset( collection->getIterator( _params.start,
@@ -128,6 +128,7 @@ namespace mongo {
         ++_commonStats.unyields;
         if (NULL != _iter) {
             if (!_iter->recoverFromYield()) {
+                warning() << "collection dropped during yield of collscan or state deleted";
                 _nsDropped = true;
             }
         }

src/mongo/db/exec/fetch.cpp

@@ -73,7 +73,6 @@ namespace mongo {
 
         // If we asked our parent for a page-in last time work(...) was called, finish the fetch.
         if (WorkingSet::INVALID_ID != _idBeingPagedIn) {
-            cout << "fetch completed, id being paged on " << _idBeingPagedIn << endl;
             return fetchCompleted(out);
         }
 

src/mongo/db/exec/plan_stage.h

@@ -119,6 +119,10 @@ namespace mongo {
             // nothing output in the out parameter.
             NEED_TIME,
 
+            // Something went wrong but it's not an internal error.  Perhaps our collection was
+            // dropped or state deleted.
+            DEAD,
+
             // Something has gone unrecoverably wrong.  Stop running this query.  There is nothing
             // output in the out parameter.
             FAILURE,

src/mongo/db/query/multi_plan_runner.cpp

@@ -309,8 +309,8 @@ namespace mongo {
                 planHitEOF = true;
             }
             else {
-                // FAILURE.  Do we want to just tank that plan and try the rest?  We probably want
-                // to fail globally as this shouldn't happen anyway.
+                // FAILURE or DEAD.  Do we want to just tank that plan and try the rest?  We
+                // probably want to fail globally as this shouldn't happen anyway.
 
                 candidate.failed = true;
                 ++_failureCount;

src/mongo/db/query/new_find.cpp

@@ -311,7 +311,7 @@ namespace mongo {
         // This is a read lock.  TODO: There is a cursor flag for not needing this.  Do we care?
         Client::ReadContext ctx(ns);
 
-        //cout << "running getMore in new system, cursorid " << cursorid << endl;
+        QLOG() << "running getMore in new system, cursorid " << cursorid << endl;
 
         // This checks to make sure the operation is allowed on a replicated node.  Since we are not
         // passing in a query object (necessary to check SlaveOK query option), the only state where
@@ -413,6 +413,15 @@ namespace mongo {
             if (Runner::RUNNER_DEAD == state || Runner::RUNNER_ERROR == state) {
                 // If we're dead there's no way to get more results.
                 saveClientCursor = false;
+                // In the old system tailable capped cursors would be killed off at the
+                // cursorid level.  If a tailable capped cursor is nuked the cursorid
+                // would vanish.
+                // 
+                // In the new system they die and are cleaned up later (or time out).
+                // So this is where we get to remove the cursorid.
+                if (0 == numResults) {
+                    resultFlags = ResultFlag_CursorNotFound;
+                }
             }
             else if (Runner::RUNNER_EOF == state) {
                 // EOF is also end of the line unless it's tailable.
@@ -428,11 +437,17 @@ namespace mongo {
                 // cc is now invalid, as is the runner
                 cursorid = 0;
                 cc = NULL;
+                QLOG() << "getMore NOT saving client cursor, ended w/state "
+                       << Runner::statestr(state)
+                       << endl;
             }
             else {
                 // Continue caching the ClientCursor.
                 cc->incPos(numResults);
                 runner->saveState();
+                QLOG() << "getMore saving client cursor ended w/state "
+                       << Runner::statestr(state)
+                       << endl;
 
                 // Possibly note slave's position in the oplog.
                 if ((queryOptions & QueryOption_OplogReplay) && !slaveReadTill.isNull()) {
@@ -455,7 +470,7 @@ namespace mongo {
         qr->startingFrom = startingResult;
         qr->nReturned = numResults;
         bb.decouple();
-        //cout << "getMore returned " << numResults << " results\n";
+        QLOG() << "getMore returned " << numResults << " results\n";
         return qr;
     }
 
@@ -749,8 +764,8 @@ namespace mongo {
                                                 cq->getParsed().getFilter());
             ccId = cc->cursorid();
 
-            //cout << "caching runner with cursorid " << ccId
-                 //<< " after returning " << numResults << " results" << endl;
+            QLOG() << "caching runner with cursorid " << ccId
+                   << " after returning " << numResults << " results" << endl;
 
             // ClientCursor takes ownership of runner.  Release to make sure it's not deleted.
             runner.release();
@@ -774,7 +789,7 @@ namespace mongo {
             cc->setLeftoverMaxTimeMicros(curop.getRemainingMaxTimeMicros());
         }
         else {
-            //cout << "not caching runner but returning " << numResults << " results\n";
+            QLOG() << "not caching runner but returning " << numResults << " results\n";
         }
 
         // Add the results from the query into the output buffer.

src/mongo/db/query/plan_executor.cpp

@@ -154,6 +154,9 @@ namespace mongo {
             else if (PlanStage::IS_EOF == code) {
                 return Runner::RUNNER_EOF;
             }
+            else if (PlanStage::DEAD == code) {
+                return Runner::RUNNER_DEAD;
+            }
             else {
                 verify(PlanStage::FAILURE == code);
                 return Runner::RUNNER_ERROR;

src/mongo/db/query/runner.h

@@ -58,6 +58,22 @@ namespace mongo {
             RUNNER_ERROR,
         };
 
+        static string statestr(RunnerState s) {
+            if (RUNNER_ADVANCED == s) {
+                return "RUNNER_ADVANCED";
+            }
+            else if (RUNNER_EOF == s) {
+                return "RUNNER_EOF";
+            }
+            else if (RUNNER_DEAD == s) {
+                return "RUNNER_DEAD";
+            }
+            else {
+                verify(RUNNER_ERROR == s);
+                return "RUNNER_ERROR";
+            }
+        }
+
         /**
          * The yielding policy of the runner.  By default, a runner does not yield itself
          * (YIELD_MANUAL).