From dfd10f909d03e1fd16a32ffc0fead28fe69c0e6f Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Fri, 9 Jan 2015 14:26:20 +0200 Subject: [PATCH] Add logging functions. Set minRecoveryPoint more carefully. Stop reading WAL at the shutdown record, to avoid printing an error message that's actually normal. --- contrib/pg_rewind/Makefile | 2 +- contrib/pg_rewind/copy_fetch.c | 178 +++++++++-------------------- contrib/pg_rewind/fetch.h | 1 + contrib/pg_rewind/filemap.c | 43 +++---- contrib/pg_rewind/libpq_fetch.c | 191 +++++++++++++++++--------------- contrib/pg_rewind/parsexlog.c | 84 +++++++------- contrib/pg_rewind/pg_rewind.c | 129 ++++++++++----------- contrib/pg_rewind/pg_rewind.h | 11 +- 8 files changed, 285 insertions(+), 354 deletions(-) diff --git a/contrib/pg_rewind/Makefile b/contrib/pg_rewind/Makefile index 0c8b218015..a6550e2d78 100644 --- a/contrib/pg_rewind/Makefile +++ b/contrib/pg_rewind/Makefile @@ -8,7 +8,7 @@ PGAPPICON = win32 PROGRAM = pg_rewind OBJS = pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \ - fetch.o copy_fetch.o libpq_fetch.o filemap.o $(WIN32RES) + fetch.o copy_fetch.o libpq_fetch.o filemap.o logging.o $(WIN32RES) PG_CPPFLAGS = -I$(libpq_srcdir) PG_LIBS = $(libpq_pgport) diff --git a/contrib/pg_rewind/copy_fetch.c b/contrib/pg_rewind/copy_fetch.c index 5de6416284..392aef973e 100644 --- a/contrib/pg_rewind/copy_fetch.c +++ b/contrib/pg_rewind/copy_fetch.c @@ -17,9 +17,10 @@ #include #include "pg_rewind.h" +#include "datapagemap.h" #include "fetch.h" #include "filemap.h" -#include "datapagemap.h" +#include "logging.h" #include "catalog/catalog.h" @@ -69,11 +70,8 @@ recurse_dir(const char *datadir, const char *parentpath, xldir = opendir(fullparentpath); if (xldir == NULL) - { - fprintf(stderr, "could not open directory \"%s\": %s\n", - fullparentpath, strerror(errno)); - exit(1); - } + pg_fatal("could not open directory \"%s\": %s\n", + fullparentpath, strerror(errno)); while (errno = 0, (xlde = readdir(xldir)) != NULL) { @@ -89,7 +87,7 @@ recurse_dir(const char *datadir, const char *parentpath, if (lstat(fullpath, &fst) < 0) { - fprintf(stderr, "warning: could not stat file \"%s\": %s", + pg_log(PG_WARNING, "could not stat file \"%s\": %s", fullpath, strerror(errno)); /* * This is ok, if the new master is running and the file was @@ -126,17 +124,14 @@ recurse_dir(const char *datadir, const char *parentpath, len = readlink(fullpath, link_target, sizeof(link_target) - 1); if (len == -1) - { - fprintf(stderr, "readlink() failed on \"%s\": %s\n", - fullpath, strerror(errno)); - exit(1); - } + pg_fatal("readlink() failed on \"%s\": %s\n", + fullpath, strerror(errno)); + if (len == sizeof(link_target) - 1) { /* path was truncated */ - fprintf(stderr, "symbolic link \"%s\" target path too long\n", - fullpath); - exit(1); + pg_fatal("symbolic link \"%s\" target path too long\n", + fullpath); } callback(path, FILE_TYPE_SYMLINK, 0, link_target); @@ -148,24 +143,19 @@ recurse_dir(const char *datadir, const char *parentpath, if (strcmp(parentpath, "pg_tblspc") == 0) recurse_dir(datadir, path, callback); #else - fprintf(stderr, "\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n", fullpath); - exit(1); + pg_fatal("\"%s\" is a symbolic link, but symbolic links are not supported on this platform\n", + fullpath); #endif /* HAVE_READLINK */ } } if (errno) - { - fprintf(stderr, "could not read directory \"%s\": %s\n", - fullparentpath, strerror(errno)); - exit(1); - } + pg_fatal("could not read directory \"%s\": %s\n", + fullparentpath, strerror(errno)); + if (closedir(xldir)) - { - fprintf(stderr, "could not close archive location \"%s\": %s\n", - fullparentpath, strerror(errno)); - exit(1); - } + pg_fatal("could not close archive location \"%s\": %s\n", + fullparentpath, strerror(errno)); } void @@ -190,22 +180,16 @@ open_target_file(const char *path, bool trunc) mode |= O_TRUNC; dstfd = open(dstpath, mode, 0600); if (dstfd < 0) - { - fprintf(stderr, "could not open destination file \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not open destination file \"%s\": %s\n", + dstpath, strerror(errno)); } void close_target_file(void) { if (close(dstfd) != 0) - { - fprintf(stderr, "error closing destination file \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("error closing destination file \"%s\": %s\n", + dstpath, strerror(errno)); dstfd = -1; /* fsync? */ @@ -221,11 +205,8 @@ write_file_range(char *buf, off_t begin, size_t size) return; if (lseek(dstfd, begin, SEEK_SET) == -1) - { - fprintf(stderr, "could not seek in destination file \"%s\": %s\n", + pg_fatal("could not seek in destination file \"%s\": %s\n", dstpath, strerror(errno)); - exit(1); - } writeleft = size; p = buf; @@ -235,11 +216,8 @@ write_file_range(char *buf, off_t begin, size_t size) writelen = write(dstfd, p, writeleft); if (writelen < 0) - { - fprintf(stderr, "could not write file \"%s\": %s\n", + pg_fatal("could not write file \"%s\": %s\n", dstpath, strerror(errno)); - exit(1); - } p += writelen; writeleft -= writelen; @@ -263,16 +241,11 @@ copy_file_range(const char *path, off_t begin, off_t end, bool trunc) srcfd = open(srcpath, O_RDONLY | PG_BINARY, 0); if (srcfd < 0) - { - fprintf(stderr, "could not open source file \"%s\": %s\n", srcpath, strerror(errno)); - exit(1); - } + pg_fatal("could not open source file \"%s\": %s\n", + srcpath, strerror(errno)); if (lseek(srcfd, begin, SEEK_SET) == -1) - { - fprintf(stderr, "could not seek in source file: %s\n", strerror(errno)); - exit(1); - } + pg_fatal("could not seek in source file: %s\n", strerror(errno)); open_target_file(path, trunc); @@ -289,22 +262,17 @@ copy_file_range(const char *path, off_t begin, off_t end, bool trunc) readlen = read(srcfd, buf, len); if (readlen < 0) - { - fprintf(stderr, "could not read file \"%s\": %s\n", srcpath, strerror(errno)); - exit(1); - } + pg_fatal("could not read file \"%s\": %s\n", + srcpath, strerror(errno)); else if (readlen == 0) - { - fprintf(stderr, "unexpected EOF while reading file \"%s\"\n", srcpath); - exit(1); - } + pg_fatal("unexpected EOF while reading file \"%s\"\n", srcpath); write_file_range(buf, begin, readlen); begin += readlen; } if (close(srcfd) != 0) - fprintf(stderr, "error closing file \"%s\": %s\n", srcpath, strerror(errno)); + pg_fatal("error closing file \"%s\": %s\n", srcpath, strerror(errno)); } /* @@ -319,22 +287,15 @@ check_samefile(int fd1, int fd2) statbuf2; if (fstat(fd1, &statbuf1) < 0) - { - fprintf(stderr, "fstat failed: %s\n", strerror(errno)); - exit(1); - } + pg_fatal("fstat failed: %s\n", strerror(errno)); if (fstat(fd2, &statbuf2) < 0) - { - fprintf(stderr, "fstat failed: %s\n", strerror(errno)); - exit(1); - } + pg_fatal("fstat failed: %s\n", strerror(errno)); if (statbuf1.st_dev == statbuf2.st_dev && statbuf1.st_ino == statbuf2.st_ino) { - fprintf(stderr, "old and new data directory are the same\n"); - exit(1); + pg_fatal("old and new data directory are the same\n"); } } @@ -424,8 +385,7 @@ create_target(file_entry_t *entry) case FILE_TYPE_REGULAR: /* can't happen */ - fprintf (stderr, "invalid action (CREATE) for regular file\n"); - exit(1); + pg_fatal("invalid action (CREATE) for regular file\n"); break; } } @@ -440,11 +400,8 @@ remove_target_file(const char *path) snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path); if (unlink(dstpath) != 0) - { - fprintf(stderr, "could not remove file \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not remove file \"%s\": %s\n", + dstpath, strerror(errno)); } void @@ -460,17 +417,13 @@ truncate_target_file(const char *path, off_t newsize) fd = open(path, O_WRONLY, 0); if (fd < 0) - { - fprintf(stderr, "could not open file \"%s\" for truncation: %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not open file \"%s\" for truncation: %s\n", + dstpath, strerror(errno)); + if (ftruncate(fd, newsize) != 0) - { - fprintf(stderr, "could not truncate file \"%s\" to %u bytes: %s\n", + pg_fatal("could not truncate file \"%s\" to %u bytes: %s\n", dstpath, (unsigned int) newsize, strerror(errno)); - exit(1); - } + close(fd); } @@ -484,11 +437,8 @@ create_target_dir(const char *path) snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path); if (mkdir(dstpath, S_IRWXU) != 0) - { - fprintf(stderr, "could not create directory \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not create directory \"%s\": %s\n", + dstpath, strerror(errno)); } static void @@ -501,11 +451,8 @@ remove_target_dir(const char *path) snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path); if (rmdir(dstpath) != 0) - { - fprintf(stderr, "could not remove directory \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not remove directory \"%s\": %s\n", + dstpath, strerror(errno)); } static void @@ -518,11 +465,8 @@ create_target_symlink(const char *path, const char *link) snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path); if (symlink(link, dstpath) != 0) - { - fprintf(stderr, "could not create symbolic link at \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not create symbolic link at \"%s\": %s\n", + dstpath, strerror(errno)); } static void @@ -535,11 +479,8 @@ remove_target_symlink(const char *path) snprintf(dstpath, sizeof(dstpath), "%s/%s", datadir_target, path); if (unlink(dstpath) != 0) - { - fprintf(stderr, "could not remove symbolic link \"%s\": %s\n", - dstpath, strerror(errno)); - exit(1); - } + pg_fatal("could not remove symbolic link \"%s\": %s\n", + dstpath, strerror(errno)); } @@ -585,29 +526,20 @@ slurpFile(const char *datadir, const char *path, size_t *filesize) snprintf(fullpath, sizeof(fullpath), "%s/%s", datadir, path); if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) == -1) - { - fprintf(stderr, _("could not open file \"%s\" for reading: %s\n"), - fullpath, strerror(errno)); - exit(2); - } + pg_fatal("could not open file \"%s\" for reading: %s\n", + fullpath, strerror(errno)); if (fstat(fd, &statbuf) < 0) - { - fprintf(stderr, _("could not open file \"%s\" for reading: %s\n"), - fullpath, strerror(errno)); - exit(2); - } + pg_fatal("could not open file \"%s\" for reading: %s\n", + fullpath, strerror(errno)); len = statbuf.st_size; buffer = pg_malloc(len + 1); if (read(fd, buffer, len) != len) - { - fprintf(stderr, _("could not read file \"%s\": %s\n"), - fullpath, strerror(errno)); - exit(2); - } + pg_fatal("could not read file \"%s\": %s\n", + fullpath, strerror(errno)); close(fd); /* Zero-terminate the buffer. */ diff --git a/contrib/pg_rewind/fetch.h b/contrib/pg_rewind/fetch.h index d3689dd37f..9ad0c5889b 100644 --- a/contrib/pg_rewind/fetch.h +++ b/contrib/pg_rewind/fetch.h @@ -29,6 +29,7 @@ extern void executeFileMap(void); /* in libpq_fetch.c */ extern void libpqConnect(const char *connstr); +extern XLogRecPtr libpqGetCurrentXlogInsertLocation(void); extern void libpqProcessFileList(void); extern void libpq_executeFileMap(filemap_t *map); extern void libpqGetChangedDataPages(datapagemap_t *pagemap); diff --git a/contrib/pg_rewind/filemap.c b/contrib/pg_rewind/filemap.c index 0e62d7971b..b79e8d6644 100644 --- a/contrib/pg_rewind/filemap.c +++ b/contrib/pg_rewind/filemap.c @@ -16,6 +16,7 @@ #include "datapagemap.h" #include "filemap.h" +#include "logging.h" #include "pg_rewind.h" #include "common/string.h" @@ -93,10 +94,7 @@ process_remote_file(const char *path, file_type_t type, size_t newsize, * regular file */ if (type != FILE_TYPE_REGULAR && isRelDataFile(path)) - { - fprintf(stderr, "data file in source \"%s\" is a directory\n", path); - exit(1); - } + pg_fatal("data file in source \"%s\" is a directory\n", path); snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path); @@ -105,11 +103,8 @@ process_remote_file(const char *path, file_type_t type, size_t newsize, { /* does not exist */ if (errno != ENOENT) - { - fprintf(stderr, "could not stat file \"%s\": %s", - localpath, strerror(errno)); - exit(1); - } + pg_fatal("could not stat file \"%s\": %s", + localpath, strerror(errno)); exists = false; } @@ -122,8 +117,7 @@ process_remote_file(const char *path, file_type_t type, size_t newsize, if (exists && !S_ISDIR(statbuf.st_mode)) { /* it's a directory in target, but not in source. Strange.. */ - fprintf(stderr, "\"%s\" is not a directory.\n", localpath); - exit(1); + pg_fatal("\"%s\" is not a directory.\n", localpath); } if (!exists) @@ -143,8 +137,7 @@ process_remote_file(const char *path, file_type_t type, size_t newsize, ) { /* it's a symbolic link in target, but not in source. Strange.. */ - fprintf(stderr, "\"%s\" is not a symbolic link.\n", localpath); - exit(1); + pg_fatal("\"%s\" is not a symbolic link.\n", localpath); } if (!exists) @@ -156,10 +149,7 @@ process_remote_file(const char *path, file_type_t type, size_t newsize, case FILE_TYPE_REGULAR: if (exists && !S_ISREG(statbuf.st_mode)) - { - fprintf(stderr, "\"%s\" is not a regular file.\n", localpath); - exit(1); - } + pg_fatal("\"%s\" is not a regular file.\n", localpath); if (!exists || !isRelDataFile(path)) { @@ -262,14 +252,11 @@ process_local_file(const char *path, file_type_t type, size_t oldsize, snprintf(localpath, sizeof(localpath), "%s/%s", datadir_target, path); if (lstat(localpath, &statbuf) < 0) { - if (errno == ENOENT) - exists = false; - else - { - fprintf(stderr, "could not stat file \"%s\": %s", - localpath, strerror(errno)); - exit(1); - } + if (errno != ENOENT) + pg_fatal("could not stat file \"%s\": %s", + localpath, strerror(errno)); + + exists = false; } if (map->array == NULL) @@ -278,8 +265,7 @@ process_local_file(const char *path, file_type_t type, size_t oldsize, if (map->nlist == 0) { /* should not happen */ - fprintf(stderr, "remote file list is empty\n"); - exit(1); + pg_fatal("remote file list is empty\n"); } filemap_list_to_array(); @@ -384,8 +370,7 @@ process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) return; case FILE_ACTION_CREATE: - fprintf(stderr, "unexpected page modification for directory or symbolic link \"%s\"", entry->path); - exit(1); + pg_fatal("unexpected page modification for directory or symbolic link \"%s\"", entry->path); } } else diff --git a/contrib/pg_rewind/libpq_fetch.c b/contrib/pg_rewind/libpq_fetch.c index a781b7aa4a..bf91306e43 100644 --- a/contrib/pg_rewind/libpq_fetch.c +++ b/contrib/pg_rewind/libpq_fetch.c @@ -20,9 +20,10 @@ #include #include "pg_rewind.h" +#include "datapagemap.h" #include "fetch.h" #include "filemap.h" -#include "datapagemap.h" +#include "logging.h" #include "libpq-fe.h" #include "catalog/catalog.h" @@ -43,14 +44,57 @@ libpqConnect(const char *connstr) { conn = PQconnectdb(connstr); if (PQstatus(conn) == CONNECTION_BAD) - { - fprintf(stderr, "could not connect to remote server: %s\n", - PQerrorMessage(conn)); - exit(1); - } + pg_fatal("could not connect to remote server: %s\n", + PQerrorMessage(conn)); + + pg_log(PG_VERBOSE, "connected to remote server\n"); +} + +/* + * Runs a query that returns a single value. + */ +static char * +libpqRunSimpleQuery(const char *sql) +{ + PGresult *res; + char *result; + + res = PQexec(conn, sql); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + pg_fatal("error running query (%s) in source server: %s\n", + sql, PQresultErrorMessage(res)); + + /* sanity check the result set */ + if (PQnfields(res) != 1 || PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) + pg_fatal("unexpected result set while running query\n"); + + result = pg_strdup(PQgetvalue(res, 0, 0)); + + PQclear(res); + + return result; +} + +/* + * Calls pg_current_xlog_insert_location() function + */ +XLogRecPtr +libpqGetCurrentXlogInsertLocation(void) +{ + XLogRecPtr result; + uint32 hi; + uint32 lo; + char *val; + + val = libpqRunSimpleQuery("SELECT pg_current_xlog_insert_location()"); + + if (sscanf(val, "%X/%X", &hi, &lo) != 2) + pg_fatal("unexpected result \"%s\" while fetching current XLOG insert location\n", val); + + result = ((uint64) hi) << 32 | lo; - if (verbose) - printf("connected to remote server\n"); + return result; } /* @@ -92,18 +136,12 @@ libpqProcessFileList(void) res = PQexec(conn, sql); if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - fprintf(stderr, "unexpected result while fetching file list: %s\n", - PQresultErrorMessage(res)); - exit(1); - } + pg_fatal("unexpected result while fetching file list: %s\n", + PQresultErrorMessage(res)); /* sanity check the result set */ - if (!(PQnfields(res) == 4)) - { - fprintf(stderr, "unexpected result set while fetching file list\n"); - exit(1); - } + if (PQnfields(res) != 4) + pg_fatal("unexpected result set while fetching file list\n"); /* Read result to local variables */ for (i = 0; i < PQntuples(res); i++) @@ -141,22 +179,14 @@ receiveFileChunks(const char *sql) PGresult *res; if (PQsendQueryParams(conn, sql, 0, NULL, NULL, NULL, NULL, 1) != 1) - { - fprintf(stderr, "could not send query: %s\n", PQerrorMessage(conn)); - exit(1); - } + pg_fatal("could not send query: %s\n", PQerrorMessage(conn)); - if (verbose) - fprintf(stderr, "getting chunks: %s\n", sql); + pg_log(PG_VERBOSE, "getting chunks: %s\n", sql); if (PQsetSingleRowMode(conn) != 1) - { - fprintf(stderr, "could not set libpq connection to single row mode\n"); - exit(1); - } + pg_fatal("could not set libpq connection to single row mode\n"); - if (verbose) - fprintf(stderr, "sent query\n"); + pg_log(PG_VERBOSE, "sent query\n"); while ((res = PQgetResult(conn)) != NULL) { @@ -173,37 +203,41 @@ receiveFileChunks(const char *sql) case PGRES_TUPLES_OK: continue; /* final zero-row result */ + default: - fprintf(stderr, "unexpected result while fetching remote files: %s\n", + pg_fatal("unexpected result while fetching remote files: %s\n", PQresultErrorMessage(res)); - exit(1); } /* sanity check the result set */ - if (!(PQnfields(res) == 3 && PQntuples(res) == 1)) - { - fprintf(stderr, "unexpected result set size while fetching remote files\n"); - exit(1); - } + if (PQnfields(res) != 3 || PQntuples(res) != 1) + pg_fatal("unexpected result set size while fetching remote files\n"); - if (!(PQftype(res, 0) == TEXTOID && PQftype(res, 1) == INT4OID && PQftype(res, 2) == BYTEAOID)) + if (PQftype(res, 0) != TEXTOID && + PQftype(res, 1) != INT4OID && + PQftype(res, 2) != BYTEAOID) { - fprintf(stderr, "unexpected data types in result set while fetching remote files: %u %u %u\n", PQftype(res, 0), PQftype(res, 1), PQftype(res, 2)); - exit(1); + pg_fatal("unexpected data types in result set while fetching remote files: %u %u %u\n", + PQftype(res, 0), PQftype(res, 1), PQftype(res, 2)); } - if (!(PQfformat(res, 0) == 1 && PQfformat(res, 1) == 1 && PQfformat(res, 2) == 1)) + + if (PQfformat(res, 0) != 1 && + PQfformat(res, 1) != 1 && + PQfformat(res, 2) != 1) { - fprintf(stderr, "unexpected result format while fetching remote files\n"); - exit(1); + pg_fatal("unexpected result format while fetching remote files\n"); } - if (!(!PQgetisnull(res, 0, 0) && !PQgetisnull(res, 0, 1) && !PQgetisnull(res, 0, 2) && - PQgetlength(res, 0, 1) == sizeof(int32))) + if (PQgetisnull(res, 0, 0) || + PQgetisnull(res, 0, 1) || + PQgetisnull(res, 0, 2)) { - fprintf(stderr, "unexpected result set while fetching remote files\n"); - exit(1); + pg_fatal("unexpected NULL result while fetching remote files\n"); } + if (PQgetlength(res, 0, 1) != sizeof(int32)) + pg_fatal("unexpected result length while fetching remote files\n"); + /* Read result set to local variables */ memcpy(&chunkoff, PQgetvalue(res, 0, 1), sizeof(int32)); chunkoff = ntohl(chunkoff); @@ -216,9 +250,8 @@ receiveFileChunks(const char *sql) chunk = PQgetvalue(res, 0, 2); - if (verbose) - fprintf(stderr, "received chunk for file \"%s\", off %d, len %d\n", - filename, chunkoff, chunksize); + pg_log(PG_VERBOSE, "received chunk for file \"%s\", off %d, len %d\n", + filename, chunkoff, chunksize); open_target_file(filename, false); @@ -242,20 +275,13 @@ libpqGetFile(const char *filename, size_t *filesize) 1, NULL, paramValues, NULL, NULL, 1); if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - fprintf(stderr, "unexpected result while fetching remote file \"%s\": %s\n", - filename, PQresultErrorMessage(res)); - exit(1); - } - + pg_fatal("unexpected result while fetching remote file \"%s\": %s\n", + filename, PQresultErrorMessage(res)); /* sanity check the result set */ - if (!(PQntuples(res) == 1 && !PQgetisnull(res, 0, 0))) - { - fprintf(stderr, "unexpected result set while fetching remote file \"%s\"\n", - filename); - exit(1); - } + if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) + pg_fatal("unexpected result set while fetching remote file \"%s\"\n", + filename); /* Read result to local variables */ len = PQgetlength(res, 0, 0); @@ -263,8 +289,7 @@ libpqGetFile(const char *filename, size_t *filesize) memcpy(result, PQgetvalue(res, 0, 0), len); result[len] = '\0'; - if (verbose) - printf("fetched file \"%s\", length %d\n", filename, len); + pg_log(PG_VERBOSE, "fetched file \"%s\", length %d\n", filename, len); if (filesize) *filesize = len; @@ -289,11 +314,9 @@ copy_file_range(const char *path, unsigned int begin, unsigned int end) snprintf(linebuf, sizeof(linebuf), "%s\t%u\t%u\n", path, begin, len); if (PQputCopyData(conn, linebuf, strlen(linebuf)) != 1) - { - fprintf(stderr, "error sending COPY data: %s\n", - PQerrorMessage(conn)); - exit(1); - } + pg_fatal("error sending COPY data: %s\n", + PQerrorMessage(conn)); + begin += len; } } @@ -317,21 +340,15 @@ libpq_executeFileMap(filemap_t *map) res = PQexec(conn, sql); if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "error creating temporary table: %s\n", - PQresultErrorMessage(res)); - exit(1); - } + pg_fatal("error creating temporary table: %s\n", + PQresultErrorMessage(res)); - sql = "copy fetchchunks from stdin"; + sql = "COPY fetchchunks FROM STDIN"; res = PQexec(conn, sql); if (PQresultStatus(res) != PGRES_COPY_IN) - { - fprintf(stderr, "unexpected result while sending file list: %s\n", - PQresultErrorMessage(res)); - exit(1); - } + pg_fatal("unexpected result while sending file list: %s\n", + PQresultErrorMessage(res)); for (i = 0; i < map->narray; i++) { @@ -369,20 +386,14 @@ libpq_executeFileMap(filemap_t *map) } if (PQputCopyEnd(conn, NULL) != 1) - { - fprintf(stderr, "error sending end-of-COPY: %s\n", - PQerrorMessage(conn)); - exit(1); - } + pg_fatal("error sending end-of-COPY: %s\n", + PQerrorMessage(conn)); while ((res = PQgetResult(conn)) != NULL) { if (PQresultStatus(res) != PGRES_COMMAND_OK) - { - fprintf(stderr, "unexpected result while sending file list: %s\n", - PQresultErrorMessage(res)); - exit(1); - } + pg_fatal("unexpected result while sending file list: %s\n", + PQresultErrorMessage(res)); } /* Ok, we've sent the file list. Now receive the files */ diff --git a/contrib/pg_rewind/parsexlog.c b/contrib/pg_rewind/parsexlog.c index f661245bed..045e1f2172 100644 --- a/contrib/pg_rewind/parsexlog.c +++ b/contrib/pg_rewind/parsexlog.c @@ -18,6 +18,7 @@ #include "pg_rewind.h" #include "filemap.h" +#include "logging.h" #include "access/rmgr.h" #include "access/xlog_internal.h" @@ -56,43 +57,48 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader, TimeLineID *pageTLI); /* - * Read all the WAL in the datadir/pg_xlog, starting from 'startpoint' on - * timeline 'tli'. Make note of the data blocks touched by the WAL records, - * and return them in a page map. + * Read WAL from the datadir/pg_xlog, starting from 'startpoint' on timeline + * 'tli', until 'endpoint'. Make note of the data blocks touched by the WAL + * records, and return them in a page map. */ void -extractPageMap(const char *datadir, XLogRecPtr startpoint, TimeLineID tli) +extractPageMap(const char *datadir, XLogRecPtr startpoint, TimeLineID tli, + XLogRecPtr endpoint) { XLogRecord *record; XLogReaderState *xlogreader; - char *errormsg; + char *errormsg; XLogPageReadPrivate private; private.datadir = datadir; private.tli = tli; xlogreader = XLogReaderAllocate(&SimpleXLogPageRead, &private); - record = XLogReadRecord(xlogreader, startpoint, &errormsg); - if (record == NULL) - { - fprintf(stderr, "could not read WAL starting at %X/%X", - (uint32) (startpoint >> 32), - (uint32) (startpoint)); - if (errormsg) - fprintf(stderr, ": %s", errormsg); - fprintf(stderr, "\n"); - exit(1); - } - do { + record = XLogReadRecord(xlogreader, startpoint, &errormsg); + + if (record == NULL) + { + XLogRecPtr errptr; + + errptr = startpoint ? startpoint : xlogreader->EndRecPtr; + + if (errormsg) + pg_fatal("error reading WAL at %X/%X: %s\n", + (uint32) (errptr >> 32), (uint32) (errptr), + errormsg); + else + pg_fatal("error reading WAL at %X/%X\n", + (uint32) (startpoint >> 32), + (uint32) (startpoint)); + } + extractPageInfo(xlogreader); - record = XLogReadRecord(xlogreader, InvalidXLogRecPtr, &errormsg); + startpoint = InvalidXLogRecPtr; /* continue reading at next record */ - if (errormsg) - fprintf(stderr, "error reading xlog record: %s\n", errormsg); - } while(record != NULL); + } while (xlogreader->ReadRecPtr != endpoint); XLogReaderFree(xlogreader); if (xlogreadfd != -1) @@ -104,7 +110,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, TimeLineID tli) /* * Reads one WAL record. Returns the end position of the record, without - * doing anything the record itself. + * doing anything with the record itself. */ XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, TimeLineID tli) @@ -122,12 +128,12 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, TimeLineID tli) record = XLogReadRecord(xlogreader, ptr, &errormsg); if (record == NULL) { - fprintf(stderr, "could not read WAL record at %X/%X", - (uint32) (ptr >> 32), (uint32) (ptr)); if (errormsg) - fprintf(stderr, ": %s", errormsg); - fprintf(stderr, "\n"); - exit(1); + pg_fatal("could not read WAL record at %X/%X: %s\n", + (uint32) (ptr >> 32), (uint32) (ptr), errormsg); + else + pg_fatal("could not read WAL record at %X/%X\n", + (uint32) (ptr >> 32), (uint32) (ptr)); } endptr = xlogreader->EndRecPtr; @@ -179,13 +185,13 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, TimeLineID tli, if (record == NULL) { - fprintf(stderr, "could not find previous WAL record at %X/%X", - (uint32) (searchptr >> 32), - (uint32) (searchptr)); if (errormsg) - fprintf(stderr, ": %s", errormsg); - fprintf(stderr, "\n"); - exit(1); + pg_fatal("could not find previous WAL record at %X/%X: %s\n", + (uint32) (searchptr >> 32), (uint32) (searchptr), + errormsg); + else + pg_fatal("could not find previous WAL record at %X/%X\n", + (uint32) (searchptr >> 32), (uint32) (searchptr)); } /* @@ -289,6 +295,9 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, return XLOG_BLCKSZ; } +/* + * Extract information on which blocks the current record modifies. + */ static void extractPageInfo(XLogReaderState *record) { @@ -345,11 +354,10 @@ extractPageInfo(XLogReaderState *record) * way, but we don't recognize the type. That's bad - we don't * know how to track that change. */ - fprintf(stderr, "WAL record modifies a relation, but record type is not recognized\n"); - fprintf(stderr, "lsn: %X/%X, rmgr: %s, info: %02X\n", - (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), - RmgrNames[rmid], info); - exit(1); + pg_fatal("WAL record modifies a relation, but record type is not recognized\n" + "lsn: %X/%X, rmgr: %s, info: %02X\n", + (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), + RmgrNames[rmid], info); } for (block_id = 0; block_id <= record->max_block_id; block_id++) diff --git a/contrib/pg_rewind/pg_rewind.c b/contrib/pg_rewind/pg_rewind.c index 6accfa8343..ff7ea2133f 100644 --- a/contrib/pg_rewind/pg_rewind.c +++ b/contrib/pg_rewind/pg_rewind.c @@ -17,6 +17,7 @@ #include "pg_rewind.h" #include "fetch.h" #include "filemap.h" +#include "logging.h" #include "access/timeline.h" #include "access/xlog_internal.h" @@ -92,7 +93,9 @@ main(int argc, char **argv) size_t size; char *buffer; bool rewind_needed; - ControlFileData ControlFile; + XLogRecPtr endrec; + TimeLineID endtli; + ControlFileData ControlFile_new; progname = get_progname(argv[0]); @@ -147,28 +150,26 @@ main(int argc, char **argv) /* No source given? Show usage */ if (datadir_source == NULL && connstr_source == NULL) { - fprintf(stderr, "%s: no source specified\n", progname); - fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); + pg_fatal("no source specified (--source-pgdata or --source-server)\n"); + pg_fatal("Try \"%s --help\" for more information.\n", progname); exit(1); } if (datadir_target == NULL) { - fprintf(stderr, "%s: no target data directory specified\n", progname); + pg_fatal("no target data directory specified (--target-pgdata)\n"); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } if (argc != optind) { - fprintf(stderr, "%s: invalid arguments\n", progname); + pg_fatal("%s: invalid arguments\n", progname); fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname); exit(1); } - /* - * Connect to remote server - */ + /* Connect to remote server */ if (connstr_source) libpqConnect(connstr_source); @@ -191,10 +192,7 @@ main(int argc, char **argv) * to do. */ if (ControlFile_target.checkPointCopy.ThisTimeLineID == ControlFile_source.checkPointCopy.ThisTimeLineID) - { - fprintf(stderr, "source and target cluster are both on the same timeline.\n"); - exit(1); - } + pg_fatal("source and target cluster are both on the same timeline.\n"); findCommonAncestorTimeline(&divergerec, &lastcommontli); printf("The servers diverged at WAL position %X/%X on timeline %u.\n", @@ -235,9 +233,10 @@ main(int argc, char **argv) printf("No rewind required.\n"); exit(0); } + findLastCheckpoint(datadir_target, divergerec, lastcommontli, &chkptrec, &chkpttli, &chkptredo); - printf("Rewinding from Last common checkpoint at %X/%X on timeline %u\n", + printf("Rewinding from last common checkpoint at %X/%X on timeline %u\n", (uint32) (chkptrec >> 32), (uint32) chkptrec, chkpttli); @@ -251,9 +250,12 @@ main(int argc, char **argv) /* * Read the target WAL from last checkpoint before the point of fork, * to extract all the pages that were modified on the target cluster - * after the fork. + * after the fork. We can stop reading after reaching the final shutdown + * record. XXX: If we supported rewinding a server that was not shut + * down cleanly, we would need to replay until the end of WAL here. */ - extractPageMap(datadir_target, chkptrec, lastcommontli); + extractPageMap(datadir_target, chkptrec, lastcommontli, + ControlFile_target.checkPoint); filemap_finalize(); @@ -261,20 +263,38 @@ main(int argc, char **argv) if (verbose) print_filemap(); - /* Ok, we're ready to start copying things over. */ + /* + * Ok, we're ready to start copying things over. Once we start modifying + * things, there is no turning back! + */ executeFileMap(); createBackupLabel(chkptredo, chkpttli, chkptrec); /* - * Update control file of target file and make it ready to - * perform archive recovery when restarting. + * Update control file of target. Make it ready to perform archive + * recovery when restarting. + * + * minRecoveryPoint is set to the current WAL insert location in the + * source server. Like in an online backup, it's important that we recover + * all the WAL that was generated while we copied the files over. */ - memcpy(&ControlFile, &ControlFile_source, sizeof(ControlFileData)); - ControlFile.minRecoveryPoint = divergerec; - ControlFile.minRecoveryPointTLI = ControlFile_target.checkPointCopy.ThisTimeLineID; - ControlFile.state = DB_IN_ARCHIVE_RECOVERY; - updateControlFile(&ControlFile, datadir_target); + memcpy(&ControlFile_new, &ControlFile_source, sizeof(ControlFileData)); + + if (connstr_source) + { + endrec = libpqGetCurrentXlogInsertLocation(); + endtli = ControlFile_source.checkPointCopy.ThisTimeLineID; + } + else + { + endrec = ControlFile_source.checkPoint; + endtli = ControlFile_source.checkPointCopy.ThisTimeLineID; + } + ControlFile_new.minRecoveryPoint = endrec; + ControlFile_new.minRecoveryPointTLI = endtli; + ControlFile_new.state = DB_IN_ARCHIVE_RECOVERY; + updateControlFile(&ControlFile_new, datadir_target); printf("Done!\n"); @@ -287,18 +307,15 @@ sanityChecks(void) /* Check that there's no backup_label in either cluster */ /* Check system_id match */ if (ControlFile_target.system_identifier != ControlFile_source.system_identifier) - { - fprintf(stderr, "source and target clusters are from different systems\n"); - exit(1); - } + pg_fatal("source and target clusters are from different systems\n"); + /* check version */ if (ControlFile_target.pg_control_version != PG_CONTROL_VERSION || ControlFile_source.pg_control_version != PG_CONTROL_VERSION || ControlFile_target.catalog_version_no != CATALOG_VERSION_NO || ControlFile_source.catalog_version_no != CATALOG_VERSION_NO) { - fprintf(stderr, "clusters are not compatible with this version of pg_rewind\n"); - exit(1); + pg_fatal("clusters are not compatible with this version of pg_rewind\n"); } /* @@ -308,8 +325,7 @@ sanityChecks(void) if (ControlFile_target.data_checksum_version != PG_DATA_CHECKSUM_VERSION && !ControlFile_target.wal_log_hints) { - fprintf(stderr, "target master need to use either data checksums or \"wal_log_hints = on\".\n"); - exit(1); + pg_fatal("target master need to use either data checksums or \"wal_log_hints = on\"\n"); } /* @@ -319,10 +335,7 @@ sanityChecks(void) * long as it isn't running at the moment. */ if (ControlFile_target.state != DB_SHUTDOWNED) - { - fprintf(stderr, "target master must be shut down cleanly.\n"); - exit(1); - } + pg_fatal("target master must be shut down cleanly\n"); } /* @@ -388,8 +401,7 @@ findCommonAncestorTimeline(XLogRecPtr *recptr, TimeLineID *tli) } } - fprintf(stderr, "could not find common ancestor of the source and target cluster's timelines\n"); - exit(1); + pg_fatal("could not find common ancestor of the source and target cluster's timelines\n"); } @@ -440,11 +452,8 @@ createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli, XLogRecPtr checkpo /* omit LABEL: line */ if (fclose(fp) != 0) - { - fprintf(stderr, _("could not write backup label file \"%s\": %s\n"), + pg_fatal("could not write backup label file \"%s\": %s\n", BackupLabelFilePath, strerror(errno)); - exit(2); - } } /* @@ -464,10 +473,7 @@ checkControlFile(ControlFileData *ControlFile) /* And simply compare it */ if (!EQ_CRC32C(crc, ControlFile->crc)) - { - fprintf(stderr, "unexpected control file CRC\n"); - exit(1); - } + pg_fatal("unexpected control file CRC\n"); } /* @@ -477,11 +483,9 @@ static void digestControlFile(ControlFileData *ControlFile, char *src, size_t size) { if (size != PG_CONTROL_SIZE) - { - fprintf(stderr, "unexpected control file size %d, expected %d\n", + pg_fatal("unexpected control file size %d, expected %d\n", (int) size, PG_CONTROL_SIZE); - exit(1); - } + memcpy(ControlFile, src, sizeof(ControlFileData)); /* Additional checks on control file */ @@ -494,9 +498,9 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) static void updateControlFile(ControlFileData *ControlFile, char *datadir) { - char path[MAXPGPATH]; - char buffer[PG_CONTROL_SIZE]; - FILE *fp; + char path[MAXPGPATH]; + char buffer[PG_CONTROL_SIZE]; + FILE *fp; if (dry_run) return; @@ -516,25 +520,14 @@ updateControlFile(ControlFileData *ControlFile, char *datadir) memset(buffer, 0, PG_CONTROL_SIZE); memcpy(buffer, ControlFile, sizeof(ControlFileData)); - snprintf(path, MAXPGPATH, - "%s/global/pg_control", datadir); + snprintf(path, MAXPGPATH, "%s/global/pg_control", datadir); if ((fp = fopen(path, "wb")) == NULL) - { - fprintf(stderr,"Could not open the pg_control file\n"); - exit(1); - } + pg_fatal("could not open pg_control file\n"); - if (fwrite(buffer, 1, - PG_CONTROL_SIZE, fp) != PG_CONTROL_SIZE) - { - fprintf(stderr,"Could not write the pg_control file\n"); - exit(1); - } + if (fwrite(buffer, 1, PG_CONTROL_SIZE, fp) != PG_CONTROL_SIZE) + pg_fatal("could not write the pg_control file\n"); if (fclose(fp)) - { - fprintf(stderr,"Could not close the pg_control file\n"); - exit(1); - } + pg_fatal("could not close the pg_control file\n"); } diff --git a/contrib/pg_rewind/pg_rewind.h b/contrib/pg_rewind/pg_rewind.h index 5a6dbce053..2676153e53 100644 --- a/contrib/pg_rewind/pg_rewind.h +++ b/contrib/pg_rewind/pg_rewind.h @@ -26,14 +26,15 @@ extern char *connstr_source; extern int verbose; extern int dry_run; - /* in parsexlog.c */ -extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, TimeLineID tli); -extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr, TimeLineID tli, +extern void extractPageMap(const char *datadir, XLogRecPtr startpoint, + TimeLineID tli, XLogRecPtr endpoint); +extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr, + TimeLineID tli, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli, XLogRecPtr *lastchkptredo); -extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, TimeLineID tli); - +extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr, + TimeLineID tli); /* in timeline.c */ extern TimeLineHistoryEntry *rewind_parseTimeLineHistory(char *buffer, -- 2.39.5