Advance input pointer when LZ4 compressing data

LZ4File_write() did not advance the input pointer on subsequent invocations of
LZ4F_compressUpdate(). As a result the generated compressed output would be a
compressed version of the same input chunk.

Tests failed to catch this error because the data would comfortably fit
within the default buffer size, as a single chunk. Tests have been added
to provide adequate coverage of multi-chunk compression.

WriteDataToArchiveLZ4() which is also using LZ4F_compressUpdate() did
not suffer from this omission.

Author: Georgios Kokolatos <[email protected]>
Reported-by: Michael Paquier <[email protected]>
Discussion: https://postgr.es/m/ZFhCyn4Gm2eu60rB%40paquier.xyz

diff --git a/src/bin/pg_dump/compress_io.h b/src/bin/pg_dump/compress_io.h
index fd8752db0de391d9986a77411736b9cf854cc385..621e03a5c8583f38e505862f54a9267f8361c66f 100644 (file)
--- a/src/bin/pg_dump/compress_io.h
+++ b/src/bin/pg_dump/compress_io.h
@@ -17,7 +17,13 @@
 
 #include "pg_backup_archiver.h"
 
-/* Default size used for IO buffers */
+/*
+ * Default size used for IO buffers
+ *
+ * When changing this value, it's necessary to check the relevant test cases
+ * still exercise all the branches. This applies especially if the value is
+ * increased, in which case the overflow buffer may not be needed.
+ */
 #define DEFAULT_IO_BUFFER_SIZE 4096
 
 extern char *supports_compression(const pg_compress_specification compression_spec);

diff --git a/src/bin/pg_dump/compress_lz4.c b/src/bin/pg_dump/compress_lz4.c
index f97b7550d1c7edf18a80b637f43e4cc8e498ae46..b869780c0b4aa4133da6a03f9023430169d41af7 100644 (file)
--- a/src/bin/pg_dump/compress_lz4.c
+++ b/src/bin/pg_dump/compress_lz4.c
@@ -588,6 +588,8 @@ LZ4Stream_write(const void *ptr, size_t size, CompressFileHandle *CFH)
                        errno = (errno) ? errno : ENOSPC;
                        return false;
                }
+
+               ptr = ((const char *) ptr) + chunk;
        }
 
        return true;

diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl
index 93e24d5145727c9027285c146703586e0bbc0dc6..d66f3b42ea42dcd793b29514f4a1fd1bafdd8921 100644 (file)
--- a/src/bin/pg_dump/t/002_pg_dump.pl
+++ b/src/bin/pg_dump/t/002_pg_dump.pl
@@ -3108,6 +3108,52 @@ my %tests = (
                },
        },
 
+       'CREATE TABLE test_compression_method' => {
+               create_order => 110,
+               create_sql   => 'CREATE TABLE dump_test.test_compression_method (
+                                                  col1 text
+                                          );',
+               regexp => qr/^
+                       \QCREATE TABLE dump_test.test_compression_method (\E\n
+                       \s+\Qcol1 text\E\n
+                       \Q);\E
+                       /xm,
+               like => {
+                       %full_runs,
+                       %dump_test_schema_runs,
+                       section_pre_data     => 1,
+               },
+               unlike => {
+                       exclude_dump_test_schema => 1,
+                       only_dump_measurement    => 1,
+               },
+       },
+
+       # Insert enough data to surpass DEFAULT_IO_BUFFER_SIZE during
+       # (de)compression operations
+       'COPY test_compression_method' => {
+               create_order => 111,
+               create_sql   => 'INSERT INTO dump_test.test_compression_method (col1) '
+                 . 'SELECT string_agg(a::text, \'\') FROM generate_series(1,4096) a;',
+               regexp => qr/^
+                       \QCOPY dump_test.test_compression_method (col1) FROM stdin;\E
+                       \n(?:\d{15277}\n){1}\\\.\n
+                       /xm,
+               like => {
+                       %full_runs,
+                       data_only                => 1,
+                       section_data             => 1,
+                       only_dump_test_schema    => 1,
+                       test_schema_plus_large_objects    => 1,
+               },
+               unlike => {
+                       binary_upgrade           => 1,
+                       exclude_dump_test_schema => 1,
+                       schema_only              => 1,
+                       only_dump_measurement    => 1,
+               },
+       },
+
        'CREATE TABLE fk_reference_test_table' => {
                create_order => 21,
                create_sql   => 'CREATE TABLE dump_test.fk_reference_test_table (