Consistency check should compare last record replayed, not last record read.

EndRecPtr is the last record that we've read, but not necessarily yet
replayed. CheckRecoveryConsistency should compare minRecoveryPoint with the
last replayed record instead. This caused recovery to think it's reached
consistency too early.

Now that we do the check in CheckRecoveryConsistency correctly, we have to
move the call of that function to after redoing a record. The current place,
after reading a record but before replaying it, is wrong. In particular, if
there are no more records after the one ending at minRecoveryPoint, we don't
enter hot standby until one extra record is generated and read by the
standby, and CheckRecoveryConsistency is called. These two bugs conspired
to make the code appear to work correctly, except for the small window
between reading the last record that reaches minRecoveryPoint, and
replaying it.

In the passing, rename recoveryLastRecPtr, which is the last record
replayed, to lastReplayedEndRecPtr. This makes it slightly less confusing
with replayEndRecPtr, which is the last record read that we're about to
replay.

Original report from Kyotaro HORIGUCHI, further diagnosis by Fujii Masao.
Backpatch to 9.0, where Hot Standby subtly changed the test from
"minRecoveryPoint < EndRecPtr" to "minRecoveryPoint <= EndRecPtr". The
former works because where the test is performed, we have always read one
more record than we've replayed.

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 7adbc3e82ec0595d48aa6d6ce5b272eb727092f1..702487be3bdc71159856e8a6cce8c6b391ecfd36 100644 (file)
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -448,10 +448,14 @@ typedef struct XLogCtlData
    XLogRecPtr  lastCheckPointRecPtr;
    CheckPoint  lastCheckPoint;
 
-   /* end+1 of the last record replayed (or being replayed) */
+   /*
+    * lastReplayedEndRecPtr points to end+1 of the last record successfully
+    * replayed. When we're currently replaying a record, ie. in a redo
+    * function, replayEndRecPtr points to the end+1 of the record being
+    * replayed, otherwise it's equal to lastReplayedEndRecPtr.
+    */
+   XLogRecPtr  lastReplayedEndRecPtr;
    XLogRecPtr  replayEndRecPtr;
-   /* end+1 of the last record replayed */
-   XLogRecPtr  recoveryLastRecPtr;
    /* timestamp of last COMMIT/ABORT record replayed (or being replayed) */
    TimestampTz recoveryLastXTime;
    /* current effective recovery target timeline */
@@ -6589,7 +6593,7 @@ StartupXLOG(void)
        }
 
        /*
-        * Initialize shared replayEndRecPtr, recoveryLastRecPtr, and
+        * Initialize shared replayEndRecPtr, lastReplayedEndRecPtr, and
         * recoveryLastXTime.
         *
         * This is slightly confusing if we're starting from an online
@@ -6602,7 +6606,7 @@ StartupXLOG(void)
         */
        SpinLockAcquire(&xlogctl->info_lck);
        xlogctl->replayEndRecPtr = ReadRecPtr;
-       xlogctl->recoveryLastRecPtr = EndRecPtr;
+       xlogctl->lastReplayedEndRecPtr = EndRecPtr;
        xlogctl->recoveryLastXTime = 0;
        xlogctl->currentChunkStartTime = 0;
        xlogctl->recoveryPause = false;
@@ -6694,9 +6698,6 @@ StartupXLOG(void)
                /* Handle interrupt signals of startup process */
                HandleStartupProcInterrupts();
 
-               /* Allow read-only connections if we're consistent now */
-               CheckRecoveryConsistency();
-
                /*
                 * Pause WAL replay, if requested by a hot-standby session via
                 * SetRecoveryPause().
@@ -6797,16 +6798,19 @@ StartupXLOG(void)
                }
 
                /*
-                * Update shared recoveryLastRecPtr after this record has been
-                * replayed.
+                * Update lastReplayedEndRecPtr after this record has been
+                * successfully replayed.
                 */
                SpinLockAcquire(&xlogctl->info_lck);
-               xlogctl->recoveryLastRecPtr = EndRecPtr;
+               xlogctl->lastReplayedEndRecPtr = EndRecPtr;
                SpinLockRelease(&xlogctl->info_lck);
 
                /* Remember this record as the last-applied one */
                LastRec = ReadRecPtr;
 
+               /* Allow read-only connections if we're consistent now */
+               CheckRecoveryConsistency();
+
                /* Exit loop if we reached inclusive recovery target */
                if (!recoveryContinue)
                    break;
@@ -7176,10 +7180,11 @@ CheckRecoveryConsistency(void)
     * Have we passed our safe starting point? Note that minRecoveryPoint
     * is known to be incorrectly set if ControlFile->backupEndRequired,
     * until the XLOG_BACKUP_RECORD arrives to advise us of the correct
-    * minRecoveryPoint. All we prior to that is its not consistent yet.
+    * minRecoveryPoint. All we know prior to that is that we're not
+    * consistent yet.
     */
    if (!reachedConsistency && !ControlFile->backupEndRequired &&
-       XLByteLE(minRecoveryPoint, EndRecPtr) &&
+       XLByteLE(minRecoveryPoint, XLogCtl->lastReplayedEndRecPtr) &&
        XLogRecPtrIsInvalid(ControlFile->backupStartPoint))
    {
        /*
@@ -7191,7 +7196,8 @@ CheckRecoveryConsistency(void)
        reachedConsistency = true;
        ereport(LOG,
                (errmsg("consistent recovery state reached at %X/%X",
-                       EndRecPtr.xlogid, EndRecPtr.xrecoff)));
+                       XLogCtl->lastReplayedEndRecPtr.xlogid,
+                       XLogCtl->lastReplayedEndRecPtr.xrecoff)));
    }
 
    /*
@@ -9927,7 +9933,7 @@ GetXLogReplayRecPtr(TimeLineID *targetTLI)
    XLogRecPtr  recptr;
 
    SpinLockAcquire(&xlogctl->info_lck);
-   recptr = xlogctl->recoveryLastRecPtr;
+   recptr = xlogctl->lastReplayedEndRecPtr;
    if (targetTLI)
        *targetTLI = xlogctl->RecoveryTargetTLI;
    SpinLockRelease(&xlogctl->info_lck);