Bug#34929814 Inconsistent FTS state in concurrent scenarios

Andrzej Jarzabek · zmur · commit a7e0c4190cb3 · 2024-05-31T12:31:41.000+02:00
Bug#36347647  Contribution by Tencent: Resiliency issue in fts_sync_commit

Symptoms:
During various operations on tables containing FTS indexes
the state of FTS as comitted to database may become
inconsistent, affecting the following scenarios:
 - the server terminates when synchronizing the FTS cache,
 - synchronization of FTS cache occurs concurrently with
   another FTS operation
This inconsistency may lead to various negative effects
including incorrect query results.
An example operation which forces the synchronization of
FTS cach is OPTIMIZE TABLE with
innodb_optimize_fulltext_only set to ON.

Root Cause:
Function 'fts_cmp_set_sync_doc_id' and 'fts_sql_commit' use
different  trx_t objects in function 'fts_sync_commit'.
This causes a scenario where 'synced_doc_id' in the config
table is already committed, but remaining FTS data isn't yet,
leading to issues in the scenarios described above - the
server terminating between the commits, or concurrent access
getting the intermediate state.

Fix:
When 'fts_cmp_set_sync_doc_id' is called from 'fts_sync_commit'
it will use the transaction provided by the caller.

Patch based on contribution by Tencent.

Change-Id: I65fa5702db5e7b6b2004a7311a6b0aa97449034f
diff --git a/mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result b/mysql-test/suite/innodb_fts/r/fts_sync_commit_resiliency.result
@@ -0,0 +1,63 @@
+CREATE TABLE opening_lines (
+id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
+opening_line TEXT(500),
+author VARCHAR(200),
+title VARCHAR(200)
+) ENGINE=InnoDB;
+CREATE FULLTEXT INDEX idx ON opening_lines(opening_line);
+Warnings:
+Warning	124	InnoDB rebuilding table to add column FTS_DOC_ID
+CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title);
+INSERT INTO opening_lines(opening_line,author,title) VALUES
+('Call me Ishmael.','Herman Melville','Moby Dick'),
+('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'),
+       ('I am an invisible man.','Ralph Ellison','Invisible Man'),
+       ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'),
+       ('It was love at first sight.','Joseph Heller','Catch-22'),
+       ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'),
+       ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'),
+       ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451');
+SET GLOBAL innodb_ft_aux_table='test/opening_lines';
+SELECT * FROM information_schema.innodb_ft_config;
+KEY	VALUE
+optimize_checkpoint_limit	180
+synced_doc_id	0
+stopword_table_name	
+use_stopword	1
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+id	opening_line	author	title
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+SELECT * FROM opening_lines;
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+2	A screaming comes across the sky.	Thomas Pynchon	Gravity's Rainbow
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+4	Where now? Who now? When now?	Samuel Beckett	The Unnamable
+5	It was love at first sight.	Joseph Heller	Catch-22
+6	All this happened, more or less.	Kurt Vonnegut	Slaughterhouse-Five
+7	Mrs. Dalloway said she would buy the flowers herself.	Virginia Woolf	Mrs. Dalloway
+8	It was a pleasure to burn.	Ray Bradbury	Fahrenheit 451
+SET GLOBAL innodb_optimize_fulltext_only=ON;
+SET GLOBAL debug='+d,fts_crash_before_commit_sync';
+OPTIMIZE TABLE opening_lines;
+ERROR HY000: Lost connection to MySQL server during query
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+id	opening_line	author	title
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+SELECT * FROM opening_lines;
+id	opening_line	author	title
+1	Call me Ishmael.	Herman Melville	Moby Dick
+2	A screaming comes across the sky.	Thomas Pynchon	Gravity's Rainbow
+3	I am an invisible man.	Ralph Ellison	Invisible Man
+4	Where now? Who now? When now?	Samuel Beckett	The Unnamable
+5	It was love at first sight.	Joseph Heller	Catch-22
+6	All this happened, more or less.	Kurt Vonnegut	Slaughterhouse-Five
+7	Mrs. Dalloway said she would buy the flowers herself.	Virginia Woolf	Mrs. Dalloway
+8	It was a pleasure to burn.	Ray Bradbury	Fahrenheit 451
+DROP TABLE opening_lines;
diff --git a/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test b/mysql-test/suite/innodb_fts/t/fts_sync_commit_resiliency.test
@@ -0,0 +1,47 @@
+# Test database resiliency against scenario where the server crashes
+# right before fts_sync_commit commits its transaction
+
+source include/have_debug.inc;
+source include/not_valgrind.inc;
+
+CREATE TABLE opening_lines (
+       id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY,
+       opening_line TEXT(500),
+       author VARCHAR(200),
+       title VARCHAR(200)
+       ) ENGINE=InnoDB;
+
+CREATE FULLTEXT INDEX idx ON opening_lines(opening_line);
+CREATE FULLTEXT INDEX ft_idx1 ON opening_lines(title);
+
+INSERT INTO opening_lines(opening_line,author,title) VALUES
+       ('Call me Ishmael.','Herman Melville','Moby Dick'),
+       ('A screaming comes across the sky.','Thomas Pynchon','Gravity\'s Rainbow'),
+       ('I am an invisible man.','Ralph Ellison','Invisible Man'),
+       ('Where now? Who now? When now?','Samuel Beckett','The Unnamable'),
+       ('It was love at first sight.','Joseph Heller','Catch-22'),
+       ('All this happened, more or less.','Kurt Vonnegut','Slaughterhouse-Five'),
+       ('Mrs. Dalloway said she would buy the flowers herself.','Virginia Woolf','Mrs. Dalloway'),
+       ('It was a pleasure to burn.','Ray Bradbury','Fahrenheit 451');
+
+SET GLOBAL innodb_ft_aux_table='test/opening_lines';
+SELECT * FROM information_schema.innodb_ft_config;
+
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+SELECT * FROM opening_lines;
+
+SET GLOBAL innodb_optimize_fulltext_only=ON;
+SET GLOBAL debug='+d,fts_crash_before_commit_sync';
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--error 2013
+OPTIMIZE TABLE opening_lines;
+
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('Ishmael');
+SELECT * FROM opening_lines WHERE MATCH(opening_line) AGAINST('invisible');
+SELECT * FROM opening_lines;
+
+DROP TABLE opening_lines;
diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc
@@ -1,6 +1,6 @@
 /*****************************************************************************
 
-Copyright (c) 2011, 2023, Oracle and/or its affiliates.
+Copyright (c) 2011, 2024, Oracle and/or its affiliates.
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License, version 2.0,
@@ -2778,26 +2778,28 @@ fts_get_next_doc_id(
 This function fetch the Doc ID from CONFIG table, and compare with
 the Doc ID supplied. And store the larger one to the CONFIG table.
 @return DB_SUCCESS if OK */
-static MY_ATTRIBUTE((nonnull))
+static
 dberr_t
 fts_cmp_set_sync_doc_id(
 /*====================*/
 	const dict_table_t*	table,		/*!< in: table */
 	doc_id_t		doc_id_cmp,	/*!< in: Doc ID to compare */
 	ibool			read_only,	/*!< in: TRUE if read the
 						synced_doc_id only */
-	doc_id_t*		doc_id)		/*!< out: larger document id
+	doc_id_t*		doc_id,		/*!< out: larger document id
 						after comparing "doc_id_cmp"
 						to the one stored in CONFIG
 						table */
+        trx_t*                  trx = NULL)
 {
-	trx_t*		trx;
 	pars_info_t*	info;
 	dberr_t		error;
 	fts_table_t	fts_table;
 	que_t*		graph = NULL;
 	fts_cache_t*	cache = table->fts->cache;
 	char		table_name[MAX_FULL_NAME_LEN];
+        bool           trx_allocated;
+        trx_savept_t    savept;
 retry:
 	ut_a(table->fts->doc_col != ULINT_UNDEFINED);
 
@@ -2808,7 +2810,13 @@ fts_cmp_set_sync_doc_id(
 
 	fts_table.parent = table->name.m_name;
 
-	trx = trx_allocate_for_background();
+        trx_allocated = false;
+        if (trx == NULL) {
+                trx = trx_allocate_for_background();
+                trx_allocated = true;
+        } else {
+                savept = trx_savept_take(trx);
+        }
 
 	trx->op_info = "update the next FTS document id";
 
@@ -2875,21 +2883,34 @@ fts_cmp_set_sync_doc_id(
 func_exit:
 
 	if (error == DB_SUCCESS) {
-		fts_sql_commit(trx);
+                if (trx_allocated) {
+        		fts_sql_commit(trx);
+                }
 	} else {
 		*doc_id = 0;
 
 		ib::error() << "(" << ut_strerr(error) << ") while getting"
 			" next doc id.";
-		fts_sql_rollback(trx);
+                if (trx_allocated) {
+		        fts_sql_rollback(trx);
+                } else {
+                        trx_rollback_to_savepoint(trx, &savept);
+                }
 
 		if (error == DB_DEADLOCK) {
 			os_thread_sleep(FTS_DEADLOCK_RETRY_WAIT);
+                        if (trx_allocated) {
+                                /* free trx before retry */
+                                trx_free_for_background(trx);
+                                trx = NULL;
+                        }
 			goto retry;
 		}
 	}
 
-	trx_free_for_background(trx);
+        if (trx_allocated) {
+	        trx_free_for_background(trx);
+        }
 
 	return(error);
 }
@@ -4618,7 +4639,7 @@ fts_sync_commit(
 	/* After each Sync, update the CONFIG table about the max doc id
 	we just sync-ed to index table */
 	error = fts_cmp_set_sync_doc_id(sync->table, sync->max_doc_id, FALSE,
-					&last_doc_id);
+					&last_doc_id, trx);
 
 	/* Get the list of deleted documents that are either in the
 	cache or were headed there but were deleted before the add
@@ -4639,6 +4660,8 @@ fts_sync_commit(
 
 	if (error == DB_SUCCESS) {
 
+                DBUG_EXECUTE_IF("fts_crash_before_commit_sync", { DBUG_SUICIDE(); });
+
 		fts_sql_commit(trx);
 
 	} else if (error != DB_SUCCESS) {
