SERVER-705 check for negative effects of large clock skews, seed last optime using oplog

Author: astaple

db/db.cpp

@@ -943,13 +943,13 @@ namespace mongo {
 
 #undef out
 
-    void exitCleanly() {
+    void exitCleanly( ExitCode code ) {
         goingAway = true;
         killCurrentOp = 1;
         {
             dblock lk;
             log() << "now exiting" << endl;
-            dbexit( EXIT_KILL );        
+            dbexit( code );        
         }
     }
 
@@ -995,7 +995,7 @@ namespace mongo {
         int x;
         sigwait( &asyncSignals, &x );
         log() << "got kill or ctrl c signal " << x << " (" << strsignal( x ) << "), will terminate after current cmd ends" << endl;
-        exitCleanly();
+        exitCleanly( EXIT_KILL );
     }
 
     void setupSignals() {
@@ -1018,7 +1018,7 @@ namespace mongo {
 #else
 void ctrlCTerminate() {
     log() << "got kill or ctrl c signal, will terminate after current cmd ends" << endl;
-    exitCleanly();
+    exitCleanly( EXIT_KILL );
 }
 BOOL CtrlHandler( DWORD fdwCtrlType )
 {

db/db.h

@@ -129,6 +129,8 @@ namespace mongo {
         }
     };
 
+    void exitCleanly( ExitCode code );
+
 } // namespace mongo
 
 #include "dbinfo.h"

db/instance.cpp

@@ -262,6 +262,9 @@ namespace mongo {
                     ss << " exception " + e.toString();
                     log = true;
                 }
+                catch ( ClockSkewException &e ) {
+                    exitCleanly( EXIT_CLOCK_SKEW );
+                }
             }
             else if ( op == dbKillCursors ) {
                 OPREAD;

db/repl.cpp

@@ -1644,8 +1644,14 @@ namespace mongo {
         const char * ns = "local.oplog.$main";
         setClient(ns);
 
-        if ( nsdetails( ns ) )
+        if ( nsdetails( ns ) ) {
+            DBDirectClient c;
+            BSONObj lastOp = c.findOne( ns, Query().sort( BSON( "$natural" << -1 ) ) );
+            if ( !lastOp.isEmpty() ) {
+                OpTime::setLast( lastOp[ "ts" ].date() );
+            }
             return;
+        }
 
         /* create an oplog collection, if it doesn't yet exist. */
         BSONObjBuilder b;

stdafx.h

@@ -53,6 +53,7 @@ namespace mongo {
         EXIT_OOM_MALLOC = 42 , 
         EXIT_OOM_REALLOC = 43 , 
         EXIT_FS = 45 ,
+        EXIT_CLOCK_SKEW = 47 ,
         EXIT_POSSIBLE_CORRUPTION = 60 , // this means we detected a possible corruption situation, like a buf overflow
         EXIT_UNCAUGHT = 100 , // top level exception that wasn't caught
         EXIT_TEST = 101 ,

util/optime.h

@@ -20,15 +20,24 @@
 #include "../db/concurrency.h"
 
 namespace mongo {
+    void exitCleanly( int code );
 
     /* Operation sequence #.  A combination of current second plus an ordinal value.
      */
+    struct ClockSkewException : public DBException {
+        virtual const char* what() const throw() { return "clock skew exception"; }
+        virtual int getCode(){ return 20001; }
+    };
+    
 #pragma pack(4)
     class OpTime {
         unsigned i;
         unsigned secs;
         static OpTime last;
     public:
+        static void setLast(const Date_t &date) {
+            last = OpTime(date);
+        }
         unsigned getSecs() const {
             return secs;
         }
@@ -47,7 +56,17 @@ namespace mongo {
             unsigned t = (unsigned) time(0);
 //            DEV assertInWriteLock();
             if ( t < last.secs ){
-                log() << "clock skew detected  prev: " << last.secs << " now: " << t << " trying to handle..." << endl;
+                bool toLog = false;
+                ONCE toLog = true;
+                RARELY toLog = true;
+                if ( last.i & 0x80000000 )
+                    toLog = true;
+                if ( toLog )
+                    log() << "clock skew detected  prev: " << last.secs << " now: " << t << " trying to handle..." << endl;
+                if ( last.i & 0x80000000 ) {
+                    log() << "ERROR Large clock skew detected, shutting down" << endl;
+                    throw ClockSkewException();
+                }
                 t = last.secs;
             }
             if ( last.secs == t ) {