Bug #30220791: INCORRECT RESULT WHEN THE ITERATOR EXECUTOR IS USED

Steinar H. Gunderson · dahlerlend · commit 9a14d83383b9 · 2019-09-10T13:16:27.000+02:00
Handle first match "split jumps" correctly in the iterator executor,
by way of our usual fallback strategy of rewriting them to weedout.
These are set up whenever the join optimizer inserts otherwise irrelevant
tables in the middle of a first match segment. Not handling them leads
to potentially wrong result in such queries (depending on row order in
the input tables), removing rows that should have been there.

Change-Id: I44e600d40bed62af520ca1f80d153784d3281412
diff --git a/mysql-test/r/explain_tree.result b/mysql-test/r/explain_tree.result
@@ -747,3 +747,38 @@ EXPLAIN
             -> Index scan on t1 using PRIMARY  (cost=0.55 rows=3)
 
 DROP TABLE t1;
+CREATE TABLE t1 (
+b INTEGER,
+UNIQUE KEY ix1 (b)
+);
+INSERT INTO t1 VALUES (5),(7);
+CREATE TABLE t2 (
+a INTEGER NOT NULL,
+b INTEGER DEFAULT NULL,
+KEY ix1 (b)
+);
+INSERT INTO t2 VALUES (40, 1), (45, 12), (11, 23), (10, 7);
+ANALYZE TABLE t1, t2;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	OK
+test.t2	analyze	status	OK
+EXPLAIN SELECT * FROM t1 AS table1
+WHERE EXISTS ( SELECT * FROM t1 LEFT JOIN t2 USING (b) WHERE t2.a = t2.a AND t2.b <= 7 );
+id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t2	NULL	range	ix1	ix1	5	NULL	2	25.00	Using index condition; Using where; Using MRR; FirstMatch
+1	SIMPLE	table1	NULL	index	NULL	ix1	5	NULL	2	100.00	Using index
+1	SIMPLE	t1	NULL	eq_ref	ix1	ix1	5	test.t2.b	1	100.00	Using index; FirstMatch(table1)
+Warnings:
+Note	1003	/* select#1 */ select `test`.`table1`.`b` AS `b` from `test`.`t1` `table1` semi join (`test`.`t1` join `test`.`t2`) where ((`test`.`t1`.`b` = `test`.`t2`.`b`) and (`test`.`t2`.`a` = `test`.`t2`.`a`) and (`test`.`t2`.`b` <= 7))
+EXPLAIN FORMAT=tree SELECT * FROM t1 AS table1
+WHERE EXISTS ( SELECT * FROM t1 LEFT JOIN t2 USING (b) WHERE t2.a = t2.a AND t2.b <= 7 );
+EXPLAIN
+-> Remove duplicate table1 rows using temporary table (weedout)
+    -> Nested loop inner join  (cost=1.55 rows=1)
+        -> Nested loop inner join  (cost=1.32 rows=1)
+            -> Filter: (t2.a = t2.a)  (cost=1.10 rows=0)
+                -> Index range scan on t2 using ix1, with index condition: ((t2.b <= 7) and (t2.b is not null))  (cost=1.10 rows=2)
+            -> Index scan on table1 using ix1  (cost=0.65 rows=2)
+        -> Single-row index lookup on t1 using ix1 (b=t2.b)  (cost=0.11 rows=1)
+
+DROP TABLE t1, t2;
diff --git a/mysql-test/r/subquery_bugs.result b/mysql-test/r/subquery_bugs.result
@@ -631,8 +631,8 @@ test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 LEFT JOIN t1 AS t3 ON t1.pk = t3.pk AND t1.f2 IN ( SELECT f2 FROM t2 )
 WHERE EXISTS ( SELECT * FROM t1 AS t4, t2 AS t5 ) AND t1.f1 = 80 AND t1.pk > t1.f1;
 EXPLAIN
--> Remove duplicate (t1, t3, t4) rows using temporary table (weedout)
-    -> Nested loop semijoin  (cost=1.04 rows=1)
+-> Remove duplicate (t1, t3) rows using temporary table (weedout)
+    -> Nested loop inner join  (cost=1.04 rows=1)
         -> Nested loop left join  (cost=0.81 rows=0)
             -> Nested loop inner join  (cost=0.64 rows=0)
                 -> Filter: ((t1.f1 = 80) and (t1.pk > 80))  (cost=0.46 rows=0)
diff --git a/mysql-test/r/subquery_sj_all.result b/mysql-test/r/subquery_sj_all.result
@@ -12896,12 +12896,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_all_bka.result b/mysql-test/r/subquery_sj_all_bka.result
@@ -12902,12 +12902,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_all_bka_nixbnl.result b/mysql-test/r/subquery_sj_all_bka_nixbnl.result
@@ -12905,12 +12905,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_all_bkaunique.result b/mysql-test/r/subquery_sj_all_bkaunique.result
@@ -12838,12 +12838,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_firstmatch.result b/mysql-test/r/subquery_sj_firstmatch.result
@@ -12758,12 +12758,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_firstmatch_bka.result b/mysql-test/r/subquery_sj_firstmatch_bka.result
@@ -12759,12 +12759,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_firstmatch_bka_nixbnl.result b/mysql-test/r/subquery_sj_firstmatch_bka_nixbnl.result
@@ -12749,12 +12749,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/r/subquery_sj_firstmatch_bkaunique.result b/mysql-test/r/subquery_sj_firstmatch_bkaunique.result
@@ -12760,12 +12760,11 @@ test.t1	analyze	status	OK
 test.t2	analyze	status	OK
 EXPLAIN FORMAT=tree SELECT * FROM t1 WHERE EXISTS ( SELECT alias1.f2 FROM t2 LEFT JOIN t1 ON 1 LEFT JOIN t1 AS alias1 ON 1 );
 EXPLAIN
--> Remove duplicate (t2, t1) rows using temporary table (weedout)
-    -> Nested loop left join
-        -> Nested loop left join
-            -> Nested loop inner join
-                -> Limit: 1 row(s)
-                    -> Table scan on t2  (cost=0.45 rows=2)
+-> Remove duplicate t1 rows using temporary table (weedout)
+    -> Nested loop left join  (cost=14.75 rows=54)
+        -> Nested loop left join  (cost=4.85 rows=18)
+            -> Nested loop inner join  (cost=1.55 rows=6)
+                -> Table scan on t2  (cost=0.45 rows=2)
                 -> Table scan on t1  (cost=0.40 rows=3)
             -> Filter: true  (cost=0.30 rows=3)
                 -> Table scan on t1  (cost=0.30 rows=3)
diff --git a/mysql-test/t/explain_tree.test b/mysql-test/t/explain_tree.test
@@ -427,3 +427,43 @@ EXPLAIN FORMAT=tree SELECT * FROM t1, ( SELECT f1 FROM t1 UNION SELECT f1 + 10 F
 EXPLAIN FORMAT=tree SELECT * FROM t1, ( SELECT f1 FROM t1 UNION ALL SELECT f1 + 10 FROM t1 ) d1;
 
 DROP TABLE t1;
+
+#
+# Bug #30220791: INCORRECT RESULT WHEN THE ITERATOR EXECUTOR IS USED
+#
+
+# This query happens to set up a “split jump” first match (firstmatch_return
+# is set on two different tables for what's logically a single jump back,
+# since the tables are non-contiguous).
+#
+# Non-tree EXPLAIN doesn't show this very clearly (it shows FirstMatch on both
+# the first and third tables), but for correctness of execution, we need to
+# to rewrite it into a single semijoin block, ie., we do a weedout at the end
+# (and no LIMIT 1 in the middle). Otherwise, the results would depend on the
+# order of rows coming from t2.
+
+CREATE TABLE t1 (
+  b INTEGER,
+  UNIQUE KEY ix1 (b)
+);
+
+INSERT INTO t1 VALUES (5),(7);
+
+CREATE TABLE t2 (
+  a INTEGER NOT NULL,
+  b INTEGER DEFAULT NULL,
+  KEY ix1 (b)
+);
+
+INSERT INTO t2 VALUES (40, 1), (45, 12), (11, 23), (10, 7);
+
+ANALYZE TABLE t1, t2;
+
+let $query =
+  SELECT * FROM t1 AS table1
+    WHERE EXISTS ( SELECT * FROM t1 LEFT JOIN t2 USING (b) WHERE t2.a = t2.a AND t2.b <= 7 );
+
+eval EXPLAIN $query;
+eval EXPLAIN FORMAT=tree $query;
+
+DROP TABLE t1, t2;
diff --git a/sql/sql_executor.cc b/sql/sql_executor.cc
@@ -1529,6 +1529,77 @@ static unique_ptr_destroy_only<RowIterator> CreateWeedoutIteratorForTables(
   return CreateWeedoutIterator(thd, move(iterator), sjtbl);
 }
 
+/**
+  Find out whether there is a first-match jump from "last_idx" to before
+  "first_idx".
+
+  This is made a bit trickier by the existence of “split jumps”. A split jump is
+  set up when there is a semijoin against two or more inner tables, but the join
+  optimizer has decided to put more tables in-between. Consider e.g. the query
+
+    (A sj (B ij C)) ij D
+
+  where the join optimizer has decided that the order should be A, B, D, C!
+  In this case, there's no direct jump from C to A, but a jump first from C to
+  D, and then when D has been processed, a jump back from B to A. As with other
+  non-hierarchical semijoin setups, we don't try to execute these directly,
+  but rather set them up as weedouts, by adding elements to
+  "unhandled_duplicates".
+
+  @return true if there is a first match jump, split or not, from "last_idx" to
+    just before "split_idx". If it's a split jump, "is_split" is set to true.
+    (If the function returns false, "is_split" is undefined.)
+ */
+static bool FirstMatchBetween(QEP_TAB *qep_tabs, const plan_idx first_idx,
+                              const plan_idx last_idx, bool *is_split,
+                              vector<QEP_TAB *> *unhandled_duplicates) {
+  if (qep_tabs[last_idx].match_tab != last_idx) {
+    // last_idx doesn't contain a first match, or its first match is
+    // the middle part of a split jump. Ignore.
+    return false;
+  }
+  if (qep_tabs[last_idx].firstmatch_return == first_idx - 1) {
+    for (plan_idx i = 0; i < first_idx; ++i) {
+      if (qep_tabs[i].firstmatch_return != NO_PLAN_IDX &&
+          qep_tabs[i].match_tab == last_idx) {
+        // The jump from <last_idx> back to <first_idx> is the last part of a
+        // larger, split jump. Ignore.
+        return false;
+      }
+    }
+
+    // Regular, non-split first match jump.
+    *is_split = false;
+    return true;
+  }
+
+  // See if there's a first match jump chain that starts at <last_idx>
+  // and ends up jumping to before <first_idx>.
+  for (plan_idx i = first_idx; i < last_idx; ++i) {
+    if (qep_tabs[i].firstmatch_return == first_idx - 1 &&
+        qep_tabs[i].match_tab == last_idx) {
+      // OK, there is. We will be solving it by adding these tables to
+      // unhandled_duplicates, so they will be solved with weedout at
+      // the very top. Go through all the different segments of the split jump
+      // to figure out which tables are supposed to be part of this semijoin.
+      plan_idx this_segment_end = MAX_TABLES + 1;
+      for (plan_idx j = last_idx; j >= first_idx; --j) {
+        if (qep_tabs[j].match_tab == last_idx) {
+          this_segment_end = qep_tabs[j].firstmatch_return;
+        }
+        if (j > this_segment_end) {
+          unhandled_duplicates->push_back(&qep_tabs[j]);
+        }
+      }
+
+      *is_split = true;
+      return true;
+    }
+  }
+
+  return false;
+}
+
 enum class Substructure { NONE, OUTER_JOIN, SEMIJOIN, WEEDOUT };
 
 /**
@@ -1563,10 +1634,19 @@ static Substructure FindSubstructure(
   bool is_semijoin = false;
   plan_idx semijoin_end = NO_PLAN_IDX;
   for (plan_idx j = this_idx; j < last_idx; ++j) {
-    if (qep_tabs[j].firstmatch_return == this_idx - 1) {
-      is_semijoin = true;
-      semijoin_end = j + 1;
-      break;
+    bool is_split;
+    if (FirstMatchBetween(qep_tabs, this_idx, j, &is_split,
+                          unhandled_duplicates)) {
+      if (is_split) {
+        // Split first-match jumps are fully handled by adding to
+        // <unhandled_duplicates>, so don't communicate the semijoin upwards;
+        // keep looking for other substructures instead (including smaller
+        // semijoins).
+      } else {
+        is_semijoin = true;
+        semijoin_end = j + 1;
+        break;
+      }
     }
   }
 
