Bug#20691930 5.7.6 CRASH WHEN RUNNING MYSQL_UPGRADE AFTER BINARY UPGRADE

This is a regression from
Bug#17345513 CHECKING FIL_PAGE_TYPE BREAKS COMPATIBILITY WITH
OLD INNODB DATA FILES

This affected a production environment where the data files were
originally created with MySQL 5.0 or earlier. Originally, InnoDB only
initialized FIL_PAGE_TYPE on two types of pages:

#define FIL_PAGE_INDEX		17855	/*!< B-tree node */
#define FIL_PAGE_UNDO_LOG	2	/*!< Undo log page */

When files were allocated in the file system, the field was
initialized to 0. When a page was initialized in the buffer pool, the
field would be left uninitialized, reusing whatever value happened to
be at that address (typically one of the 3 values).

In the originally reported incident, page 32768 in the system
tablespace is an allocation bitmap page, but the uninitialized
FIL_PAGE_TYPE field on it happened to be FIL_PAGE_INDEX, which caused
the flush-time check to fail.

Our fix comprises the following parts:

1. Reset wrong page type on allocation bitmap pages and change buffer bitmap
pages based on the page number, without checking the page contents and
without writing redo log.

#define FIL_PAGE_IBUF_BITMAP	5	/*!< Insert buffer bitmap */
#define FIL_PAGE_TYPE_FSP_HDR	8	/*!< File space header */
#define FIL_PAGE_TYPE_XDES	9	/*!< Extent descriptor page */

2. On database startup, reset the page types on the following pages
in the system tablespace, writing redo log:

#define FSP_IBUF_HEADER_PAGE_NO	3	// init to 6=FIL_PAGE_TYPE_SYS
#define FSP_TRX_SYS_PAGE_NO	5	// init to 7=FIL_PAGE_TYPE_TRX_SYS
#define FSP_FIRST_RSEG_PAGE_NO	6	// init to 6=FIL_PAGE_TYPE_SYS
#define FSP_DICT_HDR_PAGE_NO	7	// init to 6=FIL_PAGE_TYPE_SYS

3. Whenever we modify other types of pages, we reset the FIL_PAGE_TYPE
within the same mini-transaction, to one of the following values:

#define FIL_PAGE_INODE		3	/*!< Index node */
#define FIL_PAGE_TYPE_SYS	6	/*!< System page */
#define FIL_PAGE_TYPE_FSP_HDR	8	/*!< File space header */
#define FIL_PAGE_TYPE_XDES	9	/*!< Extent descriptor page */

Note: Some page types are initialized immediately after page
allocation, and the pages are not modified further without changing
the page type first.  Nothing needs to be done for these page types,
if the requirement is to have valid page type when we are writing back
pages from the buffer pool to files.

#define FIL_PAGE_IBUF_FREE_LIST	4	/*!< Insert buffer free list */
#define FIL_PAGE_TYPE_BLOB	10	/*!< Uncompressed BLOB page */
#define FIL_PAGE_TYPE_ZBLOB	11	/*!< First compressed BLOB page */
#define FIL_PAGE_TYPE_ZBLOB2	12	/*!< Subsequent compressed BLOB page */

Because MySQL does not officially support upgrade following by a
server crash, there should be no legitimate usage scenario where such
pages with an incorrect page type would be written out as a result of
applying redo log during crash recovery.

BLOB pages created before MySQL 5.1 could carry any page type
(including FIL_PAGE_INDEX), but this should not be an issue, because
existing BLOB pages are never updated in place. The BLOB columns are
always updated by copy-on-write, with a valid FIL_PAGE_TYPE.

Note: InnoDB never modifies BLOB pages in place. If BLOB data is modified,
entirely new pages will be initialized and rewritten. Thus, no logic
is implemented to update FIL_PAGE_TYPE on BLOB pages. This means that
even after this fix, subsequent versions of MySQL must be prepared
to read BLOB pages that contain anything in FIL_PAGE_TYPE.

RB: 8314
Reviewed-by: Kevin Lewis <[email protected]>
Reviewed-by: Vasil Dimov <[email protected]>
Reviewed-by: Sunny Bains <[email protected]>
(cherry picked from commit fc1f91f396bc73fcea72f17d6c22c174ba057e9b)

Author: Marko Mäkelä

mysql-test/suite/innodb/r/file_format_upgrade_16k.result

@@ -0,0 +1,15 @@
+#
+# Bug#20691930 5.7.6 CRASH WHEN RUNNING MYSQL_UPGRADE
+# AFTER BINARY UPGRADE
+#
+CREATE TABLE t (a int primary key, b char(255) not null default '', index(b))
+ENGINE=InnoDB;
+INSERT t(a) VALUES(1),(2),(3),(4),(5),(6),(7),(8);
+# restart
+INSERT t(a) SELECT a+8 FROM t;
+INSERT t(a) SELECT a+16 FROM t;
+INSERT t(a) SELECT a+32 FROM t;
+INSERT t(a) SELECT a+64 FROM t;
+# Page types: 00080005000345bf(8)
+# restart
+DROP TABLE t;

mysql-test/suite/innodb/t/file_format_upgrade_16k.test

@@ -0,0 +1,59 @@
+--source include/have_innodb.inc
+# In old versions that wrote garbage into files, the page size was 16k.
+--source include/have_innodb_16k.inc
+# Embedded server does not support restarting.
+--source include/not_embedded.inc
+
+--echo #
+--echo # Bug#20691930 5.7.6 CRASH WHEN RUNNING MYSQL_UPGRADE
+--echo # AFTER BINARY UPGRADE
+--echo #
+
+CREATE TABLE t (a int primary key, b char(255) not null default '', index(b))
+ENGINE=InnoDB;
+
+INSERT t(a) VALUES(1),(2),(3),(4),(5),(6),(7),(8);
+let $MYSQLD_DATADIR=`select @@datadir`;
+let IBD_FILE=$MYSQLD_DATADIR/test/t.ibd;
+let IBD_PAGE_SIZE=`select @@innodb_page_size`;
+
+--source include/shutdown_mysqld.inc
+perl;
+use strict;
+use warnings;
+use Fcntl qw(:DEFAULT :seek);
+my $file = $ENV{IBD_FILE};
+my $size = $ENV{IBD_PAGE_SIZE};
+sysopen FILE, $file, O_RDWR || die "Unable to open $file: $!";
+die "Unable to read $file: $!" unless sysread(FILE, $_, $size*3) == $size*3;
+my $ck = pack("H*", "DEADBEEF");
+my $type = pack("H*", "45BF"); # FIL_PAGE_INDEX
+my $payload = $size-26-8;
+# Change each page type to FIL_PAGE_INDEX, and rewrite the checksum.
+s/....(.{16})(....)..(.{$payload})......../$ck.$1.$2.$type.$3.$ck.$2/ges;
+sysseek(FILE, 0, SEEK_SET) || die "Unable to seek $file: $!";
+syswrite(FILE, $_, $size * 3) || warn "Unable to write $file: $!";
+close(FILE) || die "Unable to close $file: $!";
+EOF
+
+--source include/start_mysqld.inc
+
+INSERT t(a) SELECT a+8 FROM t;
+INSERT t(a) SELECT a+16 FROM t;
+INSERT t(a) SELECT a+32 FROM t;
+INSERT t(a) SELECT a+64 FROM t;
+--source include/shutdown_mysqld.inc
+perl;
+use strict;
+use warnings;
+my $file = $ENV{IBD_FILE};
+my $size = $ENV{IBD_PAGE_SIZE};
+open(FILE, "<$file") || die "Unable to open $file: $!";
+die "Unable to read $file: $!" unless read(FILE, $_, $size*4) == $size*4;
+my $payload = $size-26;
+s/.{24}(..).{$payload}/$1/ges;
+print "# Page types: ", unpack("H*", substr($_, 0, 8)), "(", length($_), ")\n";
+close(FILE);
+EOF
+--source include/start_mysqld.inc
+DROP TABLE t;

storage/innobase/buf/buf0dblwr.cc

@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -732,17 +732,6 @@ buf_dblwr_check_block(
 		return;
 	}
 
-	/* On uncompressed pages, it is possible that invalid
-	FIL_PAGE_TYPE was left behind by an older version of InnoDB
-	that did not initialize FIL_PAGE_TYPE on other pages than
-	B-tree pages. Here, we will reset invalid page types to
-	FIL_PAGE_TYPE_UNKNOWN when flushing. */
-	bool	reset_invalid = block->page.zip.data == NULL;
-
-	if (reset_invalid) {
-		buf_dblwr_check_page_lsn(block->frame);
-	}
-
 	switch (fil_page_get_type(block->frame)) {
 	case FIL_PAGE_INDEX:
 	case FIL_PAGE_RTREE:
@@ -753,24 +742,21 @@ buf_dblwr_check_block(
 		} else if (page_simple_validate_old(block->frame)) {
 			return;
 		}
-		/* Do not attempt to fix the page type. While it is
-		possible that this is not an index page but just
-		happens to have this value in the uninitialized
-		FIL_PAGE_TYPE field, it is also possible that we
-		caught genuine corruption of an index page, and want
-		to prevent the corruption from reaching the file
-		system. */
-		reset_invalid = false;
+		/* While it is possible that this is not an index page
+		but just happens to have wrongly set FIL_PAGE_TYPE,
+		such pages should never be modified to without also
+		adjusting the page type during page allocation or
+		buf_flush_init_for_writing() or fil_page_reset_type(). */
 		break;
+	case FIL_PAGE_TYPE_FSP_HDR:
+	case FIL_PAGE_IBUF_BITMAP:
 	case FIL_PAGE_TYPE_UNKNOWN:
 		/* Do not complain again, we already reset this field. */
 	case FIL_PAGE_UNDO_LOG:
 	case FIL_PAGE_INODE:
 	case FIL_PAGE_IBUF_FREE_LIST:
-	case FIL_PAGE_IBUF_BITMAP:
 	case FIL_PAGE_TYPE_SYS:
 	case FIL_PAGE_TYPE_TRX_SYS:
-	case FIL_PAGE_TYPE_FSP_HDR:
 	case FIL_PAGE_TYPE_XDES:
 	case FIL_PAGE_TYPE_BLOB:
 	case FIL_PAGE_TYPE_ZBLOB:
@@ -780,33 +766,9 @@ buf_dblwr_check_block(
 	case FIL_PAGE_TYPE_ALLOCATED:
 		/* empty pages should never be flushed */
 		break;
-	default:
-		break;
 	}
 
-	if (reset_invalid
-	    && fil_space_get_flags(block->page.id.space()) == 0) {
-		ib::warn()
-			<< "Resetting unknown page "
-			<< block->page.id << " type "
-			<< fil_page_get_type(block->frame)
-			<< " to " << FIL_PAGE_TYPE_UNKNOWN
-			<< " before writing to data file.";
-		/* We are skipping redo logging here, because this is
-		a special case. Flushing is covered by previously
-		generated redo log records (write-ahead log). */
-		mach_write_to_2(block->frame
-				+ FIL_PAGE_TYPE, FIL_PAGE_TYPE_UNKNOWN);
-	} else {
-		/* Only in tablespaces that were created before MySQL
-		5.1, some pages could be created with garbage in the
-		FIL_PAGE_TYPE field. These tablespaces always carried
-		flags==0, indicating ROW_FORMAT=REDUNDANT or
-		ROW_FORMAT=COMPACT. For all other tablespaces,
-		FIL_PAGE_TYPE must always be valid, already during
-		page creation. */
-		buf_dblwr_assert_on_corrupt_block(block);
-	}
+	buf_dblwr_assert_on_corrupt_block(block);
 }
 
 /********************************************************************//**

storage/innobase/buf/buf0flu.cc

@@ -801,19 +801,25 @@ buf_flush_update_zip_checksum(
 	mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
 }
 
-/********************************************************************//**
-Initializes a page for writing to the tablespace. */
+/** Initialize a page for writing to the tablespace.
+@param[in]	block		buffer block; NULL if bypassing the buffer pool
+@param[in,out]	page		page frame
+@param[in,out]	page_zip_	compressed page, or NULL if uncompressed
+@param[in]	newest_lsn	newest modification LSN to the page
+@param[in]	skip_checksum	whether to disable the page checksum */
 void
 buf_flush_init_for_writing(
-/*=======================*/
-	byte*	page,		/*!< in/out: page */
-	void*	page_zip_,	/*!< in/out: compressed page, or NULL */
-	lsn_t	newest_lsn,	/*!< in: newest modification lsn
-				to the page */
-	bool	skip_checksum)	/*!< in: if true, disable/skip checksum. */
+	const buf_block_t*	block,
+	byte*			page,
+	void*			page_zip_,
+	lsn_t			newest_lsn,
+	bool			skip_checksum)
 {
 	ib_uint32_t	checksum = BUF_NO_CHECKSUM_MAGIC;
 
+	ut_ad(block == NULL || block->frame == page);
+	ut_ad(block == NULL || page_zip_ == NULL
+	      || &block->page.zip == page_zip_);
 	ut_ad(page);
 
 	if (page_zip_) {
@@ -865,6 +871,58 @@ buf_flush_init_for_writing(
 	if (skip_checksum) {
 		mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
 	} else {
+		if (block != NULL && UNIV_PAGE_SIZE == 16384) {
+			/* The page type could be garbage in old files
+			created before MySQL 5.5. Such files always
+			had a page size of 16 kilobytes. */
+			ulint	page_type = fil_page_get_type(page);
+			ulint	reset_type = page_type;
+
+			switch (block->page.id.page_no() % 16384) {
+			case 0:
+				reset_type = block->page.id.page_no() == 0
+					? FIL_PAGE_TYPE_FSP_HDR
+					: FIL_PAGE_TYPE_XDES;
+				break;
+			case 1:
+				reset_type = FIL_PAGE_IBUF_BITMAP;
+				break;
+			default:
+				switch (page_type) {
+				case FIL_PAGE_INDEX:
+				case FIL_PAGE_RTREE:
+				case FIL_PAGE_UNDO_LOG:
+				case FIL_PAGE_INODE:
+				case FIL_PAGE_IBUF_FREE_LIST:
+				case FIL_PAGE_TYPE_ALLOCATED:
+				case FIL_PAGE_TYPE_SYS:
+				case FIL_PAGE_TYPE_TRX_SYS:
+				case FIL_PAGE_TYPE_BLOB:
+				case FIL_PAGE_TYPE_ZBLOB:
+				case FIL_PAGE_TYPE_ZBLOB2:
+					break;
+				case FIL_PAGE_TYPE_FSP_HDR:
+				case FIL_PAGE_TYPE_XDES:
+				case FIL_PAGE_IBUF_BITMAP:
+					/* These pages should have
+					predetermined page numbers
+					(see above). */
+				default:
+					reset_type = FIL_PAGE_TYPE_UNKNOWN;
+					break;
+				}
+			}
+
+			if (UNIV_UNLIKELY(page_type != reset_type)) {
+				ib::info()
+					<< "Resetting invalid page "
+					<< block->page.id << " type "
+					<< page_type << " to "
+					<< reset_type << " when flushing.";
+				fil_page_set_type(page, reset_type);
+			}
+		}
+
 		switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) {
 		case SRV_CHECKSUM_ALGORITHM_CRC32:
 		case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
@@ -979,12 +1037,12 @@ buf_flush_write_block_low(
 			frame = ((buf_block_t*) bpage)->frame;
 		}
 
-		buf_flush_init_for_writing(((buf_block_t*) bpage)->frame,
-					   bpage->zip.data
-					   ? &bpage->zip : NULL,
-					   bpage->newest_modification,
-					   fsp_is_checksum_disabled(
-						bpage->id.space()));
+		buf_flush_init_for_writing(
+			reinterpret_cast<const buf_block_t*>(bpage),
+			reinterpret_cast<const buf_block_t*>(bpage)->frame,
+			bpage->zip.data ? &bpage->zip : NULL,
+			bpage->newest_modification,
+			fsp_is_checksum_disabled(bpage->id.space()));
 		break;
 	}
 

storage/innobase/fil/fil0fil.cc

@@ -2198,7 +2198,8 @@ fil_recreate_tablespace(
 #endif /* UNIV_DEBUG */
 		page_zip.m_end = page_zip.m_nonempty = page_zip.n_blobs = 0;
 		buf_flush_init_for_writing(
-			page, &page_zip, 0, fsp_is_checksum_disabled(space_id));
+			NULL, page, &page_zip, 0,
+			fsp_is_checksum_disabled(space_id));
 
 		err = fil_write(page_id_t(space_id, 0), page_size, 0,
 				page_size.physical(), page_zip.data);
@@ -2262,7 +2263,7 @@ fil_recreate_tablespace(
 			ut_ad(!page_size.is_compressed());
 
 			buf_flush_init_for_writing(
-				page, NULL, recv_lsn,
+				block, page, NULL, recv_lsn,
 				fsp_is_checksum_disabled(space_id));
 
 			err = fil_write(cur_page_id, page_size, 0,
@@ -2277,7 +2278,7 @@ fil_recreate_tablespace(
 					buf_block_get_page_zip(block);
 
 				buf_flush_init_for_writing(
-					page, page_zip, recv_lsn,
+					block, page, page_zip, recv_lsn,
 					fsp_is_checksum_disabled(space_id));
 
 				err = fil_write(cur_page_id, page_size, 0,
@@ -3445,7 +3446,8 @@ fil_ibd_create(
 
 	if (!page_size.is_compressed()) {
 		buf_flush_init_for_writing(
-			page, NULL, 0, fsp_is_checksum_disabled(space_id));
+			NULL, page, NULL, 0,
+			fsp_is_checksum_disabled(space_id));
 		success = os_file_write(path, file, page, 0,
 					page_size.physical());
 	} else {
@@ -3460,7 +3462,8 @@ fil_ibd_create(
 			page_zip.n_blobs = 0;
 
 		buf_flush_init_for_writing(
-			page, &page_zip, 0, fsp_is_checksum_disabled(space_id));
+			NULL, page, &page_zip, 0,
+			fsp_is_checksum_disabled(space_id));
 		success = os_file_write(path, file, page_zip.data, 0,
 					page_size.physical());
 	}
@@ -5566,18 +5569,25 @@ fil_page_set_type(
 	mach_write_to_2(page + FIL_PAGE_TYPE, type);
 }
 
-/*********************************************************************//**
-Gets the file page type.
-@return type; NOTE that if the type has not been written to page, the
-return value not defined */
-ulint
-fil_page_get_type(
-/*==============*/
-	const byte*	page)	/*!< in: file page */
+/** Reset the page type.
+Data files created before MySQL 5.1 may contain garbage in FIL_PAGE_TYPE.
+In MySQL 3.23.53, only undo log pages and index pages were tagged.
+Any other pages were written with unitialized bytes in FIL_PAGE_TYPE.
+@param[in]	page_id	page number
+@param[in,out]	page	page with invalid FIL_PAGE_TYPE
+@param[in]	type	expected page type
+@param[in,out]	mtr	mini-transaction */
+void
+fil_page_reset_type(
+	const page_id_t&	page_id,
+	byte*			page,
+	ulint			type,
+	mtr_t*			mtr)
 {
-	ut_ad(page);
-
-	return(mach_read_from_2(page + FIL_PAGE_TYPE));
+	ib::info()
+		<< "Resetting invalid page " << page_id << " type "
+		<< fil_page_get_type(page) << " to " << type << ".";
+	mlog_write_ulint(page + FIL_PAGE_TYPE, type, MLOG_2BYTES, mtr);
 }
 
 /****************************************************************//**