Provide test coverage in pg_dump for default behaviors with compression

By default, the contents generated by the custom and directory dump
formats are compressed.  However, with the existing test facility, the
restore program will succeed regardless of whether the dumped output was
compressed or not without checking if anything has been compressed.

This commit implements a portable way to check the contents of the
custom and directory dump formats:
- glob_patterns, that can be defined for each test as an array of
glob()-compilable strings, tracking the contents that should or should
not be compressed.  While this is useful to make sure that the table
data is compressed, this also checks that blobs.toc and toc.dat are
never compressed.
- command_like, to execute a command on a dump and check its generated
output.  This is used here in correlation with pg_restore -l to check if
the dumps have been compressed or not, depending on if the build
supports gzip, or not.

This hole in the tests has come up when working on 5e73a60, where
compression has to be applied by default, if available, for both dump
formats.

The idea of glob_patterns comes from me, and Georgios has come up with
the design for command_like.

Author: Georgios Kokolatos, Michael Paquier
Discussion: https://postgr.es/m/DQn4czCWR1rcbGPLL7p3LfEr5-kGmlySm-H05VgroINdikvhtS5r9EdI6b8D8sjnbKdJ09k-cxs2AqijBeHAWk9Q8gvEAxPRHuLRhwONcGc=@pm.me

diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index c2da1df39d33dd6de176c1857bd9189b5d0b50ca..6656222363e914e3adc9b9d83fd864e471d3c81d 100644 (file)
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -36,6 +36,13 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
 # to test pg_restore's ability to parse manually compressed files
 # that otherwise pg_dump does not compress on its own (e.g. *.toc).
 #
+# glob_patterns is an optional array consisting of strings compilable
+# with glob() to check the files generated after a dump.
+#
+# command_like is an optional utility that can be used to compare
+# the output generated by a command on the contents of an existing
+# dump, like the TOC description of a pg_restore command.
+#
 # restore_cmd is the pg_restore command to run, if any.  Note
 # that this should generally be used when the pg_dump goes to
 # a non-text file and that the restore can then be used to
@@ -46,6 +53,10 @@ my $tempdir = PostgreSQL::Test::Utils::tempdir;
 # database and then pg_dump *that* database (or something along
 # those lines) to validate that part of the process.
 
+my $supports_icu  = ($ENV{with_icu} eq 'yes');
+my $supports_lz4  = check_pg_config("#define USE_LZ4 1");
+my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
+
 my %pgdump_runs = (
        binary_upgrade => {
                dump_cmd => [
@@ -79,6 +90,13 @@ my %pgdump_runs = (
                        "--file=$tempdir/compression_gzip_custom.sql",
                        "$tempdir/compression_gzip_custom.dump",
                ],
+               command_like => {
+                       command => [
+                               'pg_restore', '-l', "$tempdir/compression_gzip_custom.dump",
+                       ],
+                       expected => qr/Compression: 1/,
+                       name     => 'data content is gzip-compressed'
+               },
        },
 
        # Do not use --no-sync to give test coverage for data sync.
@@ -96,6 +114,11 @@ my %pgdump_runs = (
                        program => $ENV{'GZIP_PROGRAM'},
                        args    => [ '-f', "$tempdir/compression_gzip_dir/blobs.toc", ],
                },
+               # Verify that only data files where compressed
+               glob_patterns => [
+                       "$tempdir/compression_gzip_dir/toc.dat",
+                       "$tempdir/compression_gzip_dir/*.dat.gz",
+               ],
                restore_cmd => [
                        'pg_restore', '--jobs=2',
                        "--file=$tempdir/compression_gzip_dir.sql",
@@ -198,11 +221,13 @@ my %pgdump_runs = (
        },
 
        # Do not use --no-sync to give test coverage for data sync.
+       # By default, the custom format compresses its data file
+       # when the code is compiled with gzip support, and lets them
+       # uncompressed when not compiled with it.
        defaults_custom_format => {
                test_key => 'defaults',
-               compile_option => 'gzip',
                dump_cmd => [
-                       'pg_dump', '-Fc', '-Z6',
+                       'pg_dump', '-Fc',
                        "--file=$tempdir/defaults_custom_format.dump", 'postgres',
                ],
                restore_cmd => [
@@ -210,9 +235,20 @@ my %pgdump_runs = (
                        "--file=$tempdir/defaults_custom_format.sql",
                        "$tempdir/defaults_custom_format.dump",
                ],
+               command_like => {
+                       command =>
+                         [ 'pg_restore', '-l', "$tempdir/defaults_custom_format.dump", ],
+                       expected => $supports_gzip ?
+                       qr/Compression: -1/ :
+                       qr/Compression: 0/,
+                       name => 'data content is gzip-compressed by default if available',
+               },
        },
 
        # Do not use --no-sync to give test coverage for data sync.
+       # By default, the directory format compresses its data files
+       # when the code is compiled with gzip support, and lets them
+       # uncompressed when not compiled with it.
        defaults_dir_format => {
                test_key => 'defaults',
                dump_cmd => [
@@ -224,6 +260,21 @@ my %pgdump_runs = (
                        "--file=$tempdir/defaults_dir_format.sql",
                        "$tempdir/defaults_dir_format",
                ],
+               command_like => {
+                       command =>
+                         [ 'pg_restore', '-l', "$tempdir/defaults_dir_format", ],
+                       expected => $supports_gzip ?
+                       qr/Compression: -1/ :
+                       qr/Compression: 0/,
+                       name => 'data content is gzip-compressed by default',
+               },
+               glob_patterns => [
+                       "$tempdir/defaults_dir_format/toc.dat",
+                       "$tempdir/defaults_dir_format/blobs.toc",
+                       $supports_gzip ?
+                       "$tempdir/defaults_dir_format/*.dat.gz" :
+                       "$tempdir/defaults_dir_format/*.dat",
+               ],
        },
 
        # Do not use --no-sync to give test coverage for data sync.
@@ -3920,10 +3971,6 @@ if ($collation_check_stderr !~ /ERROR: /)
        $collation_support = 1;
 }
 
-my $supports_icu  = ($ENV{with_icu} eq 'yes');
-my $supports_lz4  = check_pg_config("#define USE_LZ4 1");
-my $supports_gzip = check_pg_config("#define HAVE_LIBZ 1");
-
 # ICU doesn't work with some encodings
 my $encoding = $node->safe_psql('postgres', 'show server_encoding');
 $supports_icu = 0 if $encoding eq 'SQL_ASCII';
@@ -4153,6 +4200,24 @@ foreach my $run (sort keys %pgdump_runs)
                command_ok(\@full_compress_cmd, "$run: compression commands");
        }
 
+       if ($pgdump_runs{$run}->{glob_patterns})
+       {
+               my $glob_patterns = $pgdump_runs{$run}->{glob_patterns};
+               foreach my $glob_pattern (@{$glob_patterns})
+               {
+                       my @glob_output = glob($glob_pattern);
+                       is(scalar(@glob_output) > 0, 1, "$run: glob check for $glob_pattern");
+               }
+       }
+
+       if ($pgdump_runs{$run}->{command_like})
+       {
+               my $cmd_like = $pgdump_runs{$run}->{command_like};
+               $node->command_like(\@{ $cmd_like->{command} },
+                               $cmd_like->{expected},
+                               "$run: " . $cmd_like->{name})
+       }
+
        if ($pgdump_runs{$run}->{restore_cmd})
        {
                $node->command_ok(\@{ $pgdump_runs{$run}->{restore_cmd} },