Add end-to-end testing of pg_basebackup's tar-format output.

The existing test script does run pg_basebackup with the -Ft option,
but it makes no real attempt to verify the sanity of the results.
We wouldn't know if the output is incompatible with standard "tar"
programs, nor if the server fails to start from the restored output.
Notably, this means that xlog.c's read_tablespace_map() is not being
meaningfully tested, since that code is used only in the tar-format
case.  (We do have reasonable coverage of restoring from plain-format
output, though it's over in src/test/recovery not here.)

Hence, attempt to untar the output and start a server from it,
rather just hoping it's OK.

This test assumes that the local "tar" has the "-C directory"
switch.  Although that's not promised by POSIX, my research
suggests that all non-extinct tar implementations have it.
Should the buildfarm's opinion differ, we can complicate the
test a bit to avoid requiring that.

Possibly this should be back-patched, but I'm unsure about
whether it could work on Windows before d66b23b03.

diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile
index 1d861087ad1e7cbb1006b59ca41cedc7ca18e72e..66e0070f1a4f3387a77ee5a40fa7672677b98290 100644 (file)
--- a/src/bin/pg_basebackup/Makefile
+++ b/src/bin/pg_basebackup/Makefile
@@ -18,6 +18,9 @@ subdir = src/bin/pg_basebackup
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+# make this available to TAP test scripts
+export TAR
+
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 

diff --git a/src/bin/pg_basebackup/t/010_pg_basebackup.pl b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
index 9eba7d8d7d260b2e8862046dad9ec4fde3ca7b6a..019c15c62fa47b11da4a4355b3e3f3f3176f72fc 100644 (file)
--- a/src/bin/pg_basebackup/t/010_pg_basebackup.pl
+++ b/src/bin/pg_basebackup/t/010_pg_basebackup.pl
@@ -6,7 +6,7 @@ use File::Basename qw(basename dirname);
 use File::Path qw(rmtree);
 use PostgresNode;
 use TestLib;
-use Test::More tests => 109;
+use Test::More tests => 110;
 
 program_help_ok('pg_basebackup');
 program_version_ok('pg_basebackup');
@@ -229,6 +229,7 @@ dir_symlink("$tempdir/pg_replslot", "$pgdata/pg_replslot")
 
 $node->start;
 
+# Test backup of a tablespace using tar format.
 # Create a temporary directory in the system location and symlink it
 # to our physical temp location.  That way we can use shorter names
 # for the tablespace directories, which hopefully won't run afoul of
@@ -242,14 +243,52 @@ my $real_tempdir = TestLib::perl2host($tempdir);
 $node->safe_psql('postgres',
    "CREATE TABLESPACE tblspc1 LOCATION '$realTsDir';");
 $node->safe_psql('postgres',
-   "CREATE TABLE test1 (a int) TABLESPACE tblspc1;");
-$node->command_ok(
-   [ 'pg_basebackup', '-D', "$real_tempdir/tarbackup2", '-Ft' ],
-   'tar format with tablespaces');
-ok(-f "$tempdir/tarbackup2/base.tar", 'backup tar was created');
-my @tblspc_tars = glob "$tempdir/tarbackup2/[0-9]*.tar";
+       "CREATE TABLE test1 (a int) TABLESPACE tblspc1;"
+     . "INSERT INTO test1 VALUES (1234);");
+$node->backup('tarbackup2', backup_options => ['-Ft']);
+# empty test1, just so that it's different from the to-be-restored data
+$node->safe_psql('postgres', "TRUNCATE TABLE test1;");
+
+# basic checks on the output
+my $backupdir = $node->backup_dir . '/tarbackup2';
+ok(-f "$backupdir/base.tar",   'backup tar was created');
+ok(-f "$backupdir/pg_wal.tar", 'WAL tar was created');
+my @tblspc_tars = glob "$backupdir/[0-9]*.tar";
 is(scalar(@tblspc_tars), 1, 'one tablespace tar was created');
-rmtree("$tempdir/tarbackup2");
+
+# Try to verify the tar-format backup by restoring it.
+# For this, we use the tar program identified by configure.
+SKIP:
+{
+   my $tar = $ENV{TAR};
+   skip "no tar program available", 1
+     if (!defined $tar || $tar eq '');
+
+   my $node2 = get_new_node('replica');
+
+   # Recover main data directory
+   $node2->init_from_backup($node, 'tarbackup2', tar_program => $tar);
+
+   # Recover tablespace into a new directory (not where it was!)
+   mkdir "$tempdir/tblspc1replica";
+   my $realRepTsDir = TestLib::perl2host("$shorter_tempdir/tblspc1replica");
+   TestLib::system_or_bail($tar, 'xf', $tblspc_tars[0], '-C', $realRepTsDir);
+
+   # Update tablespace map to point to new directory.
+   # XXX Ideally pg_basebackup would handle this.
+   $tblspc_tars[0] =~ m|/([0-9]*)\.tar$|;
+   my $tblspcoid       = $1;
+   my $escapedRepTsDir = $realRepTsDir;
+   $escapedRepTsDir =~ s/\\/\\\\/g;
+   open my $mapfile, '>', $node2->data_dir . '/tablespace_map';
+   print $mapfile "$tblspcoid $escapedRepTsDir\n";
+   close $mapfile;
+
+   $node2->start;
+   my $result = $node2->safe_psql('postgres', 'SELECT * FROM test1');
+   is($result, '1234', "tablespace data restored from tar-format backup");
+   $node2->stop;
+}
 
 # Create an unlogged table to test that forks other than init are not copied.
 $node->safe_psql('postgres',

diff --git a/src/test/perl/PostgresNode.pm b/src/test/perl/PostgresNode.pm
index 9667f7667eccdf436a66533441d00cb9b48de7ed..97e05993bec3cb43a618396789b572062d802387 100644 (file)
--- a/src/test/perl/PostgresNode.pm
+++ b/src/test/perl/PostgresNode.pm
@@ -538,8 +538,11 @@ sub append_conf
 =item $node->backup(backup_name)
 
 Create a hot backup with B<pg_basebackup> in subdirectory B<backup_name> of
-B<< $node->backup_dir >>, including the WAL. WAL files
-fetched at the end of the backup, not streamed.
+B<< $node->backup_dir >>, including the WAL.
+
+By default, WAL files are fetched at the end of the backup, not streamed.
+You can adjust that and other things by passing an array of additional
+B<pg_basebackup> command line options in the keyword parameter backup_options.
 
 You'll have to configure a suitable B<max_wal_senders> on the
 target server since it isn't done by default.
@@ -548,7 +551,7 @@ target server since it isn't done by default.
 
 sub backup
 {
-   my ($self, $backup_name) = @_;
+   my ($self, $backup_name, %params) = @_;
    my $backup_path = $self->backup_dir . '/' . $backup_name;
    my $name        = $self->name;
 
@@ -556,7 +559,8 @@ sub backup
    TestLib::system_or_bail(
        'pg_basebackup', '-D', $backup_path, '-h',
        $self->host,     '-p', $self->port,  '--checkpoint',
-       'fast',          '--no-sync');
+       'fast',          '--no-sync',
+       @{ $params{backup_options} });
    print "# Backup finished\n";
    return;
 }
@@ -650,6 +654,11 @@ of a backup previously created on that node with $node->backup.
 
 Does not start the node after initializing it.
 
+By default, the backup is assumed to be plain format.  To restore from
+a tar-format backup, pass the name of the tar program to use in the
+keyword parameter tar_program.  Note that tablespace tar files aren't
+handled here.
+
 Streaming replication can be enabled on this node by passing the keyword
 parameter has_streaming => 1. This is disabled by default.
 
@@ -687,8 +696,21 @@ sub init_from_backup
    mkdir $self->archive_dir;
 
    my $data_path = $self->data_dir;
-   rmdir($data_path);
-   RecursiveCopy::copypath($backup_path, $data_path);
+   if (defined $params{tar_program})
+   {
+       mkdir($data_path);
+       TestLib::system_or_bail($params{tar_program}, 'xf',
+           $backup_path . '/base.tar',
+           '-C', $data_path);
+       TestLib::system_or_bail($params{tar_program}, 'xf',
+           $backup_path . '/pg_wal.tar',
+           '-C', $data_path . '/pg_wal');
+   }
+   else
+   {
+       rmdir($data_path);
+       RecursiveCopy::copypath($backup_path, $data_path);
+   }
    chmod(0700, $data_path);
 
    # Base configuration for this node