To support external compression of archived WAL data, add a flag bit to

WAL records that shows whether it is safe to remove full-page images
(ie, whether or not an on-line backup was in progress when the WAL entry
was made).  Also make provision for an XLOG_NOOP record type that can be
used to fill in the extra space when decompressing the data for restore.

This is the portion of Koichi Suzuki's "full page writes" patch that
has to go into the core database.  The remainder of that work is two
external compression and decompression programs, which for the time being
will undergo separate development on pgfoundry.  Per discussion.

Also, twiddle the handling of BTREE_SPLIT records to ensure it'll be
possible to compress them (the previous coding caused essential info
to be omitted).  The other commonly-used record types seem OK already,
with the possible exception of GIN and GIST WAL records, which I don't
understand well enough to opine on.

diff --git a/src/backend/access/nbtree/nbtinsert.c b/src/backend/access/nbtree/nbtinsert.c
index 775eaca2427e48591feebf311ce43b46e4814790..5f8be398ca11d755b43887a417e854295b0182b7 100644 (file)
--- a/src/backend/access/nbtree/nbtinsert.c
+++ b/src/backend/access/nbtree/nbtinsert.c
@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *   $PostgreSQL: pgsql/src/backend/access/nbtree/nbtinsert.c,v 1.156 2007/04/11 20:47:37 tgl Exp $
+ *   $PostgreSQL: pgsql/src/backend/access/nbtree/nbtinsert.c,v 1.157 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1034,21 +1034,23 @@ _bt_split(Relation rel, Buffer buf, OffsetNumber firstright,
         * Log the new item and its offset, if it was inserted on the left
         * page. (If it was put on the right page, we don't need to explicitly
         * WAL log it because it's included with all the other items on the
-        * right page.) Show these as belonging to the left page buffer,
-        * so that they are not stored if XLogInsert decides it needs a
-        * full-page image of the left page.
+        * right page.) Show the new item as belonging to the left page buffer,
+        * so that it is not stored if XLogInsert decides it needs a full-page
+        * image of the left page.  We store the offset anyway, though, to
+        * support archive compression of these records.
         */
        if (newitemonleft)
        {
            lastrdata->next = lastrdata + 1;
            lastrdata++;
+
            lastrdata->data = (char *) &newitemoff;
            lastrdata->len = sizeof(OffsetNumber);
-           lastrdata->buffer = buf;        /* backup block 1 */
-           lastrdata->buffer_std = true;
+           lastrdata->buffer = InvalidBuffer;
 
            lastrdata->next = lastrdata + 1;
            lastrdata++;
+
            lastrdata->data = (char *) newitem;
            lastrdata->len = MAXALIGN(newitemsz);
            lastrdata->buffer = buf;        /* backup block 1 */
@@ -1064,6 +1066,7 @@ _bt_split(Relation rel, Buffer buf, OffsetNumber firstright,
             */
            lastrdata->next = lastrdata + 1;
            lastrdata++;
+
            lastrdata->data = NULL;
            lastrdata->len = 0;
            lastrdata->buffer = buf;        /* backup block 1 */

diff --git a/src/backend/access/nbtree/nbtxlog.c b/src/backend/access/nbtree/nbtxlog.c
index ff41be37679523a343aeeca392e8e89e0a047055..189695853a52b899dec157238c41f4583c1611df 100644 (file)
--- a/src/backend/access/nbtree/nbtxlog.c
+++ b/src/backend/access/nbtree/nbtxlog.c
@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *   $PostgreSQL: pgsql/src/backend/access/nbtree/nbtxlog.c,v 1.43 2007/04/11 20:47:38 tgl Exp $
+ *   $PostgreSQL: pgsql/src/backend/access/nbtree/nbtxlog.c,v 1.44 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -292,14 +292,17 @@ btree_xlog_split(bool onleft, bool isroot,
    }
 
    /* Extract newitem and newitemoff, if present */
-   if (onleft && !(record->xl_info & XLR_BKP_BLOCK_1))
+   if (onleft)
    {
-       IndexTupleData itupdata;
-
        /* Extract the offset (still assuming 16-bit alignment) */
        memcpy(&newitemoff, datapos, sizeof(OffsetNumber));
        datapos += sizeof(OffsetNumber);
        datalen -= sizeof(OffsetNumber);
+   }
+
+   if (onleft && !(record->xl_info & XLR_BKP_BLOCK_1))
+   {
+       IndexTupleData itupdata;
 
        /*
         * We need to copy the tuple header to apply IndexTupleDSize, because

diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c
index 655563a3d841ca58584d6daad664593af52074eb..3dc00499bfb441c8bd79391910dc511963a08a09 100644 (file)
--- a/src/backend/access/transam/xlog.c
+++ b/src/backend/access/transam/xlog.c
@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.268 2007/04/30 21:01:52 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.269 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -783,6 +783,19 @@ begin:;
        }
    }
 
+   /*
+    * If we backed up any full blocks and online backup is not in progress,
+    * mark the backup blocks as removable.  This allows the WAL archiver to
+    * know whether it is safe to compress archived WAL data by transforming
+    * full-block records into the non-full-block format.
+    *
+    * Note: we could just set the flag whenever !forcePageWrites, but
+    * defining it like this leaves the info bit free for some potential
+    * other use in records without any backup blocks.
+    */
+   if ((info & XLR_BKP_BLOCK_MASK) && !Insert->forcePageWrites)
+       info |= XLR_BKP_REMOVABLE;
+
    /*
     * If there isn't enough space on the current XLOG page for a record
     * header, advance to the next page (leaving the unused space as zeroes).
@@ -5868,6 +5881,10 @@ xlog_redo(XLogRecPtr lsn, XLogRecord *record)
 
        RecoveryRestartPoint(&checkPoint);
    }
+   else if (info == XLOG_NOOP)
+   {
+       /* nothing to do here */
+   }
    else if (info == XLOG_SWITCH)
    {
        /* nothing to do here */
@@ -5894,6 +5911,10 @@ xlog_desc(StringInfo buf, uint8 xl_info, char *rec)
                         checkpoint->nextMultiOffset,
                 (info == XLOG_CHECKPOINT_SHUTDOWN) ? "shutdown" : "online");
    }
+   else if (info == XLOG_NOOP)
+   {
+       appendStringInfo(buf, "xlog no-op");
+   }
    else if (info == XLOG_NEXTOID)
    {
        Oid         nextOid;

diff --git a/src/include/access/xlog.h b/src/include/access/xlog.h
index bb53a04b4981647e19b05b12f5f0fbdd6dd80118..87ff6aba5018794a1e185ececa7f9de93ce23ef4 100644 (file)
--- a/src/include/access/xlog.h
+++ b/src/include/access/xlog.h
@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.76 2007/01/05 22:19:51 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.77 2007/05/20 21:08:19 tgl Exp $
  */
 #ifndef XLOG_H
 #define XLOG_H
@@ -66,8 +66,7 @@ typedef struct XLogRecord
 /*
  * If we backed up any disk blocks with the XLOG record, we use flag bits in
  * xl_info to signal it.  We support backup of up to 3 disk blocks per XLOG
- * record. (Could support 4 if we cared to dedicate all the xl_info bits for
- * this purpose; currently bit 0 of xl_info is unused and available.)
+ * record.
  */
 #define XLR_BKP_BLOCK_MASK     0x0E    /* all info bits used for bkp blocks */
 #define XLR_MAX_BKP_BLOCKS     3
@@ -76,6 +75,15 @@ typedef struct XLogRecord
 #define XLR_BKP_BLOCK_2            XLR_SET_BKP_BLOCK(1)    /* 0x04 */
 #define XLR_BKP_BLOCK_3            XLR_SET_BKP_BLOCK(2)    /* 0x02 */
 
+/*
+ * Bit 0 of xl_info is set if the backed-up blocks could safely be removed
+ * from a compressed version of XLOG (that is, they are backed up only to
+ * prevent partial-page-write problems, and not to ensure consistency of PITR
+ * recovery).  The compression algorithm would need to extract data from the
+ * blocks to create an equivalent non-full-page XLOG record.
+ */
+#define XLR_BKP_REMOVABLE      0x01
+
 /*
  * Sometimes we log records which are out of transaction control.
  * Rmgr may "or" XLOG_NO_TRAN into info passed to XLogInsert to indicate this.

diff --git a/src/include/access/xlog_internal.h b/src/include/access/xlog_internal.h
index cc9c6070bc7795111cac60ea54fdcfbbcbedd370..c5645b138999f5e8a89a4252627d7360fa1d2d40 100644 (file)
--- a/src/include/access/xlog_internal.h
+++ b/src/include/access/xlog_internal.h
@@ -11,7 +11,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlog_internal.h,v 1.20 2007/04/30 21:01:53 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlog_internal.h,v 1.21 2007/05/20 21:08:19 tgl Exp $
  */
 #ifndef XLOG_INTERNAL_H
 #define XLOG_INTERNAL_H
@@ -71,7 +71,7 @@ typedef struct XLogContRecord
 /*
  * Each page of XLOG file has a header like this:
  */
-#define XLOG_PAGE_MAGIC 0xD061 /* can be used as WAL version indicator */
+#define XLOG_PAGE_MAGIC 0xD062 /* can be used as WAL version indicator */
 
 typedef struct XLogPageHeaderData
 {

diff --git a/src/include/catalog/pg_control.h b/src/include/catalog/pg_control.h
index 23c2681a8fbac4013f6c7bbe44ca518435c93ed4..8203cc3f81df31290d4131482e718eb115623a2e 100644 (file)
--- a/src/include/catalog/pg_control.h
+++ b/src/include/catalog/pg_control.h
@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.37 2007/04/03 04:14:26 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_control.h,v 1.38 2007/05/20 21:08:19 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -44,6 +44,7 @@ typedef struct CheckPoint
 /* XLOG info values for XLOG rmgr */
 #define XLOG_CHECKPOINT_SHUTDOWN       0x00
 #define XLOG_CHECKPOINT_ONLINE         0x10
+#define XLOG_NOOP                      0x20
 #define XLOG_NEXTOID                   0x30
 #define XLOG_SWITCH                        0x40