Merge branch 'mysql-8.0' into mysql-trunk

Change-Id: I0a4748ab944766e822c7e93a3a461dc6a73d1b2b

Author: roylyseng

mysql-test/r/subquery_sj_mat_nosj.result

@@ -11561,7 +11561,7 @@ WHERE (alias1.col_varchar_key, alias1.col_varchar_nokey) IN
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	<derived2>	NULL	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
 1	PRIMARY	alias2	NULL	ALL	NULL	NULL	NULL	NULL	11	10.00	Using where; Using join buffer (hash join)
-4	SUBQUERY	<derived7>	NULL	ALL	<auto_distinct_key>	NULL	NULL	NULL	2	100.00	NULL
+4	SUBQUERY	<derived7>	NULL	ALL	NULL	NULL	NULL	NULL	2	100.00	NULL
 7	DERIVED	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 8	UNION	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
 9	UNION RESULT	<union7,8>	NULL	ALL	NULL	NULL	NULL	NULL	NULL	NULL	Using temporary

sql/sql_optimizer.cc

@@ -9166,153 +9166,163 @@ void JOIN::finalize_derived_keys() {
   assert(query_block->materialized_derived_table_count);
   ASSERT_BEST_REF_IN_JOIN_ORDER(this);
 
-  bool adjust_key_count = false;
   table_map processed_tables = 0;
 
   for (uint i = 0; i < tables; i++) {
-    JOIN_TAB *tab = best_ref[i];
-    TABLE *table = tab->table();
-    Table_ref *table_ref = tab->table_ref;
+    TABLE *const table = best_ref[i]->table();
+    Table_ref *const tr = best_ref[i]->table_ref;
     /*
-     Save chosen key description if:
-     1) it's a materialized derived table
-     2) it's not yet instantiated
-     3) some keys are defined for it
+      Process the table's key definitions if:
+      1) it is a materialized derived table, and
+      2) it is not yet instantiated, and
+      3) it has some keys defined, and
+      4) it has not yet been processed (may happen if there are more than one
+         local references to the same CTE, which are processed on seeing the
+         first reference).
     */
-    if (table && table_ref->uses_materialization() &&  // (1)
-        !table->is_created() &&                        // (2)
-        table->s->keys > 0)                            // (3)
-    {
-      /*
-        If there are two local references to the same CTE, and they use
-        the same key, the iteration for the second reference is unnecessary.
-      */
-      if (processed_tables & table_ref->map()) continue;
-
-      adjust_key_count = true;
+    if (table == nullptr || !tr->uses_materialization() ||  // (1)
+        table->is_created() ||                              // (2)
+        table->s->keys == 0 ||                              // (3)
+        (processed_tables & tr->map())) {                   // (4)
+      continue;
+    }
+    /*
+      Collect all used keys before starting to shuffle them:
+      First create a map from key number to the table using the key:
+    */
+    assert(table->s->keys <= MAX_INDEXES);
+    TABLE *table_map[MAX_INDEXES];
+    for (uint j = 0; j < table->s->keys; j++) {
+      table_map[j] = nullptr;
+    }
 
-      Key_map used_keys;
+    Key_map used_keys;
 
-      // Mark all unique indexes as in use, since they have an effect
-      // (deduplication) whether any expression refers to them or not.
-      // In particular, they are used if we want to materialize a UNION DISTINCT
-      // directly into the derived table.
-      for (uint key_idx = 0; key_idx < table->s->keys; ++key_idx) {
-        if (table->key_info[key_idx].flags & HA_NOSAME) {
-          used_keys.set_bit(key_idx);
-        }
+    // Mark all unique indexes as in use, since they have an effect
+    // (deduplication) whether any expression refers to them or not.
+    // In particular, they are used if we want to materialize a UNION DISTINCT
+    // directly into the derived table.
+    for (uint key_idx = 0; key_idx < table->s->keys; ++key_idx) {
+      if (table->key_info[key_idx].flags & HA_NOSAME) {
+        used_keys.set_bit(key_idx);
       }
+    }
+    // Same for the hash key used for manual deduplication, if any.
+    // (It always has index 0 if it exists.)
+    if (table->hash_field) {
+      used_keys.set_bit(0);
+    }
 
-      // Same for the hash key used for manual deduplication, if any. (It always
-      // has index 0 if it exists.)
-      if (table->hash_field) {
-        used_keys.set_bit(0);
-      }
+    Derived_refs_iterator it(tr);
 
-      Key_use *const keyuse = tab->position()->key;
-      if (keyuse == nullptr && used_keys.is_clear_all()) {
-        // Nothing uses any keys.
-        tab->keys().clear_all();
-        tab->const_keys.clear_all();
+    while (TABLE *t = it.get_next()) {
+      if (t->pos_in_table_list->query_block != query_block) {
         continue;
       }
+      JOIN_TAB *jtab = t->reginfo.join_tab;
+      Key_use *const keyuse = jtab->position()->key;
+      if (keyuse != nullptr) {
+        used_keys.set_bit(keyuse->key);
+        table_map[keyuse->key] = t;
+      } else {
+        jtab->keys().clear_all();
+        jtab->const_keys.clear_all();
+        processed_tables |= t->pos_in_table_list->map();
+      }
+    }
+    /*
+      This call is required to establish the initial value for
+      TABLE_SHARE::first_unused_tmp_key.
+    */
+    (void)table->s->find_first_unused_tmp_key(used_keys);
 
-      Derived_refs_iterator it(table_ref);
-      while (TABLE *t = it.get_next()) {
-        /*
-          Eliminate possible keys created by this JOIN and which it
-          doesn't use.
-          Collect all keys of this table which are used by any reference in
-          this query block. Any other query block doesn't matter as:
-          - either it was optimized before, so it's not using a key we may
+    // Process keys in increasing key order
+    for (uint j = 0; j < table->s->keys; j++) {
+      TABLE *const t = table_map[j];
+      if (t == nullptr) continue;
+
+      /*
+        Eliminate possible keys created by this JOIN and which it doesn't use.
+        Collect all keys of this table which are used by any reference in
+        this query block. Any other query block doesn't matter as:
+        - either it was optimized before, so it's not using a key we may
           want to drop.
-          - or it was optimized in this same window, so:
-            * either we own the window, then any key we may want to
+        - or it was optimized in this same window, so:
+          * either we own the window, then any key we may want to
             drop is not visible to it.
-            * or it owns the window, then we are using only existing
-            keys.
-          - or it will be optimized after, so it's not using any key yet.
-
-          used_keys is a mix of possible used keys and existing used keys.
-        */
-        if (t->pos_in_table_list->query_block == query_block) {
-          JOIN_TAB *jtab = t->reginfo.join_tab;
-          Key_use *keyuse_1 = jtab->position()->key;
-          if (keyuse_1) used_keys.set_bit(keyuse_1->key);
-        }
-      }
+          * or it owns the window, then we are using only existing keys.
+        - or it will be optimized after, so it's not using any key yet.
 
-      uint new_idx = table->s->find_first_unused_tmp_key(
-          used_keys);  // Also updates table->s->first_unused_tmp_key.
-      if (keyuse == nullptr) {
+        used_keys is a mix of possible used keys and existing used keys.
+      */
+      if (t->pos_in_table_list->query_block != query_block) {
         continue;
       }
 
+      JOIN_TAB *jtab = t->reginfo.join_tab;
+      Key_use *const keyuse = jtab->position()->key;
+      assert(keyuse != nullptr);
+
+      // Also updates table->s->first_unused_tmp_key.
+      uint new_idx = t->s->find_first_unused_tmp_key(used_keys);
+
       const uint old_idx = keyuse->key;
       assert(old_idx != new_idx);
 
       if (old_idx > new_idx) {
-        assert(table->s->owner_of_possible_tmp_keys == query_block);
-        it.rewind();
-        while (TABLE *t = it.get_next()) {
+        assert(t->s->owner_of_possible_tmp_keys == query_block);
+        Derived_refs_iterator it1(tr);
+        while (TABLE *t1 = it1.get_next()) {
           /*
             Unlike the collection of used_keys, references from other query
             blocks must be considered here, as they need a key_info array
             consistent with the to-be-changed table->s->keys.
           */
-          t->move_tmp_key(old_idx, it.is_first());
+          t1->move_tmp_key(old_idx, it1.is_first());
         }
-      } else
+        used_keys.clear_bit(old_idx);
+        used_keys.set_bit(new_idx);
+      } else {
         new_idx = old_idx;  // Index stays at same slot
+      }
 
       /*
         If the key was created by earlier-optimized query blocks, and is
         already used by nonlocal references, those don't need any further
-        update: they are already setup to use it and we're not moving the
-        key.
+        update: they are already setup to use it and we're not moving the key.
         If the key was created by this query block, nonlocal references cannot
         possibly be referencing it.
         In both cases, only local references need to update their Key_use.
       */
-      it.rewind();
-      while (TABLE *t = it.get_next()) {
-        if (t->pos_in_table_list->query_block != query_block) continue;
-        JOIN_TAB *jtab = t->reginfo.join_tab;
-        Key_use *keyuse_1 = jtab->position()->key;
-        if (keyuse_1 && keyuse_1->key == old_idx) {
-          processed_tables |= t->pos_in_table_list->map();
-          const bool key_is_const = jtab->const_keys.is_set(old_idx);
+      Derived_refs_iterator it2(tr);
+      while (TABLE *t2 = it2.get_next()) {
+        if (t2->pos_in_table_list->query_block != query_block) continue;
+        JOIN_TAB *jt2 = t2->reginfo.join_tab;
+        Key_use *ku2 = jt2->position()->key;
+        if (ku2 != nullptr && ku2->key == old_idx) {
+          processed_tables |= t2->pos_in_table_list->map();
+          const bool key_is_const = jt2->const_keys.is_set(old_idx);
           // tab->keys() was never set, so must be set
-          jtab->keys().clear_all();
-          jtab->keys().set_bit(new_idx);
-          jtab->const_keys.clear_all();
-          if (key_is_const) tab->const_keys.set_bit(new_idx);
-          for (Key_use *kit = keyuse_1;
-               kit->table_ref == jtab->table_ref && kit->key == old_idx; kit++)
+          jt2->keys().clear_all();
+          jt2->keys().set_bit(new_idx);
+          jt2->const_keys.clear_all();
+          if (key_is_const) jt2->const_keys.set_bit(new_idx);
+          for (Key_use *kit = ku2;
+               kit->table_ref == jt2->table_ref && kit->key == old_idx; kit++) {
             kit->key = new_idx;
+          }
         }
       }
     }
-  }
 
-  if (!adjust_key_count) return;
+    // Finally, we know how many keys remain in the table.
+    if (table->s->owner_of_possible_tmp_keys != query_block) continue;
 
-  // Finally we know how many keys remain in the table.
-  for (uint i = 0; i < tables; i++) {
-    JOIN_TAB *tab = best_ref[i];
-    TABLE *table = tab->table();
-    Table_ref *table_ref = tab->table_ref;
-    if (table && table_ref->uses_materialization() && !table->is_created() &&
-        table->s->keys > 0) {
-      if (table->s->owner_of_possible_tmp_keys != query_block) continue;
-      /*
-        Release lock. As a bonus, avoid double work when this loop
-        later processes another local reference to the same table (similar to
-        the processed_tables map in the first loop).
-      */
-      table->s->owner_of_possible_tmp_keys = nullptr;
-      Derived_refs_iterator it(table_ref);
-      while (TABLE *t = it.get_next()) t->drop_unused_tmp_keys(it.is_first());
+    // Release lock:
+    table->s->owner_of_possible_tmp_keys = nullptr;
+    it.rewind();
+    while (TABLE *t = it.get_next()) {
+      t->drop_unused_tmp_keys(it.is_first());
     }
   }
 }