Extend TAP tests of pg_dump to test for compression with gzip

The test logic is extended with two new concepts:
- Addition of a compression command called compress_cmd, executed
between restore_cmd and dump_cmd to control the contents of the dumps.
In the case of this commit, this is used to compress or decompress
elements of a dump to test new code paths.
- Addition of a new flag called compile_option, to check if a set of
tests can be executed depending on the ./configure options used in a
given build.

The tests introduced here are for gzip, but they are designed so as they
can easily be extended for new compression methods.

Author: Georgios Kokolatos, Rachel Heaton
Discussion: https://postgr.es/m/faUNEOpts9vunEaLnmxmG-DldLSg_ql137OC3JYDmgrOMHm1RvvWY2IdBkv_CRxm5spCCb_OmKNk2T03TMm0fBEWveFF9wA1WizPuAgB7Ss=@protonmail.com

diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile
index 302f7e02d6806a338a2d45e2d917c1602e502a1d..2f524b09bf53a55037013d78148f8cbca4fa7eee 100644 (file)
--- a/src/bin/pg_dump/Makefile
+++ b/src/bin/pg_dump/Makefile
@@ -16,6 +16,8 @@ subdir = src/bin/pg_dump
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
+export GZIP_PROGRAM=$(GZIP)
+
 override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
 LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport)
 

diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index af5d6fa5a334db19a0c91d2132a239f420fecaf2..75b754a4202d3c18c20b52fd29bb8a0183c85328 100644 (file)
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -20,12 +20,22 @@ my $tempdir       = PostgreSQL::Test::Utils::tempdir;
 # test_key indicates that a given run should simply use the same
 # set of like/unlike tests as another run, and which run that is.
 #
+# compile_option indicates if the commands run depend on a compilation
+# option, if any.  This can be used to control if tests should be
+# skipped when a build dependency is not satisfied.
+#
 # dump_cmd is the pg_dump command to run, which is an array of
 # the full command and arguments to run.  Note that this is run
 # using $node->command_ok(), so the port does not need to be
 # specified and is pulled from $PGPORT, which is set by the
 # PostgreSQL::Test::Cluster system.
 #
+# compress_cmd is the utility command for (de)compression, if any.
+# Note that this should generally be used on pg_dump's output
+# either to generate a text file to run the through the tests, or
+# to test pg_restore's ability to parse manually compressed files
+# that otherwise pg_dump does not compress on its own (e.g. *.toc).
+#
 # restore_cmd is the pg_restore command to run, if any.  Note
 # that this should generally be used when the pg_dump goes to
 # a non-text file and that the restore can then be used to
@@ -54,6 +64,58 @@ my %pgdump_runs = (
                        "$tempdir/binary_upgrade.dump",
                ],
        },
+
+       # Do not use --no-sync to give test coverage for data sync.
+       compression_gzip_custom => {
+               test_key       => 'compression',
+               compile_option => 'gzip',
+               dump_cmd       => [
+                       'pg_dump',      '--format=custom',
+                       '--compress=1', "--file=$tempdir/compression_gzip_custom.dump",
+                       'postgres',
+               ],
+               restore_cmd => [
+                       'pg_restore',
+                       "--file=$tempdir/compression_gzip_custom.sql",
+                       "$tempdir/compression_gzip_custom.dump",
+               ],
+       },
+
+       # Do not use --no-sync to give test coverage for data sync.
+       compression_gzip_dir => {
+               test_key       => 'compression',
+               compile_option => 'gzip',
+               dump_cmd       => [
+                       'pg_dump',                              '--jobs=2',
+                       '--format=directory',                   '--compress=1',
+                       "--file=$tempdir/compression_gzip_dir", 'postgres',
+               ],
+               # Give coverage for manually compressed blob.toc files during
+               # restore.
+               compress_cmd => {
+                       program => $ENV{'GZIP_PROGRAM'},
+                       args    => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
+               },
+               restore_cmd => [
+                       'pg_restore', '--jobs=2',
+                       "--file=$tempdir/compression_gzip_dir.sql",
+                       "$tempdir/compression_gzip_dir",
+               ],
+       },
+
+       compression_gzip_plain => {
+               test_key       => 'compression',
+               compile_option => 'gzip',
+               dump_cmd       => [
+                       'pg_dump', '--format=plain', '-Z1',
+                       "--file=$tempdir/compression_gzip_plain.sql.gz", 'postgres',
+               ],
+               # Decompress the generated file to run through the tests.
+               compress_cmd => {
+                       program => $ENV{'GZIP_PROGRAM'},
+                       args    => [ '-d', "$tempdir/compression_gzip_plain.sql.gz", ],
+               },
+       },
        clean => {
                dump_cmd => [
                        'pg_dump',
@@ -424,6 +486,7 @@ my %full_runs = (
        binary_upgrade           => 1,
        clean                    => 1,
        clean_if_exists          => 1,
+       compression              => 1,
        createdb                 => 1,
        defaults                 => 1,
        exclude_dump_test_schema => 1,
@@ -3098,6 +3161,7 @@ my %tests = (
                        binary_upgrade          => 1,
                        clean                   => 1,
                        clean_if_exists         => 1,
+                       compression             => 1,
                        createdb                => 1,
                        defaults                => 1,
                        exclude_test_table      => 1,
@@ -3171,6 +3235,7 @@ my %tests = (
                        binary_upgrade           => 1,
                        clean                    => 1,
                        clean_if_exists          => 1,
+                       compression              => 1,
                        createdb                 => 1,
                        defaults                 => 1,
                        exclude_dump_test_schema => 1,
@@ -3833,8 +3898,9 @@ if ($collation_check_stderr !~ /ERROR: /)
        $collation_support = 1;
 }
 
-# Determine whether build supports LZ4.
-my $supports_lz4 = check_pg_config("#define USE_LZ4 1");
+# Determine whether build supports LZ4 and gzip.
+my $supports_lz4  = check_pg_config("#define USE_LZ4 1");
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
 
 # Create additional databases for mutations of schema public
 $node->psql('postgres', 'create database regress_pg_dump_test;');
@@ -3947,9 +4013,32 @@ foreach my $run (sort keys %pgdump_runs)
        my $test_key = $run;
        my $run_db   = 'postgres';
 
+       # Skip command-level tests for gzip if there is no support for it.
+       if (   defined($pgdump_runs{$run}->{compile_option})
+               && $pgdump_runs{$run}->{compile_option} eq 'gzip'
+               && !$supports_gzip)
+       {
+               note "$run: skipped due to no gzip support";
+               next;
+       }
+
        $node->command_ok(\@{ $pgdump_runs{$run}->{dump_cmd} },
                "$run: pg_dump runs");
 
+       if ($pgdump_runs{$run}->{compress_cmd})
+       {
+               my ($compress_cmd) = $pgdump_runs{$run}->{compress_cmd};
+               my $compress_program = $compress_cmd->{program};
+
+               # Skip the rest of the test if the compression program is
+               # not defined.
+               next if (!defined($compress_program) || $compress_program eq '');
+
+               my @full_compress_cmd =
+                 ($compress_cmd->{program}, @{ $compress_cmd->{args} });
+               command_ok(\@full_compress_cmd, "$run: compression commands");
+       }
+
        if ($pgdump_runs{$run}->{restore_cmd})
        {
                $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },