WL#6035: Native InnoDB Partitioning.

Implementing partitioning natively in InnoDB, so it does not need
ha_partition generic partition engine for supporting partitioning.

ha_partition uses one ha_innobase handler for each partition which
does not share common data with each other resulting in high resource
usage which this WL fix.

It also makes it easier to support other InnoDB features that partitioning
currently does not support.

A new handler ha_innopart is added, inheriting both ha_innobase (for InnoDB
access) and Partition_helper (for partitioning support, see wl#4807).

And to avoid a proxy object for Partition_handler (see wl#4807) it is also
inherited.

Also ha_partition is changed to inherit Partition_handler directly.

As a result of this there is no longer any need for .par files
for partitioned InnoDB tables, since InnoDB can use its internal
data dictionary for finding partitions during rename and delete.

The optimizer estimate is also changed, especially for
records_in_range, where all used partitions are checked instead
of only the biggest ones.

rb#7636
Approved by Krunal and Annamalai

Author: Mattias Jonsson

mysql-test/include/have_log_bin.inc

@@ -1,5 +1,5 @@
 # ==== Purpose ====
-# 
+#
 # Ensure that the server is running with binlogging on
 #
 # ==== Usage ====

mysql-test/include/partition_default_functions.inc

@@ -15,25 +15,20 @@ FLUSH STATUS;
 SET TIMESTAMP = 1234567890;
 INSERT INTO t3 (a) VALUES (NULL);
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 }
 FLUSH STATUS;
 SET TIMESTAMP = 1234567891;
 INSERT INTO t3 VALUES ();
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 FLUSH STATUS;
 INSERT INTO t3 (a) VALUES ('2011-01-01 00:00:00');
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 FLUSH STATUS;
 INSERT INTO t3 (a) VALUES ('2011-01-01 00:00:01');
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 FLUSH STATUS;
 INSERT INTO t3 (a) VALUES ('2011-01-01 00:00:02'), ('2011-01-01 00:00:03');
 --source include/get_handler_status_counts.inc
---echo # 6 locks (1 table, 2 partition lock/unlock)
 --echo # 2 writes
 FLUSH STATUS;
 SET TIMESTAMP = 1234567892;
@@ -42,22 +37,19 @@ ON DUPLICATE KEY UPDATE b = "DUP_KEY";
 --source include/get_handler_status_counts.inc
 if ($default_update)
 {
---echo # 8 locks (1 table, 3 partition lock/unlock) 
 --echo # No pruning due to DEFAULT function on partitioning column
 --echo # 1 read_key + 1 delete + 2 write (1 failed + 1 ok)
 --echo # 1 delete + 1 write due to moved to different partition
 }
 if (!$default_update)
 {
---echo # 4 locks (1 table, 1 partition lock/unlock) 
 --echo # 1 read_key + 1 update (same partition)
 --echo # 1 (failed) write
 }
 FLUSH STATUS;
 INSERT INTO t3 (a) VALUES ('2011-01-01 00:00:01')
 ON DUPLICATE KEY UPDATE a = '2011-01-01 00:00:05', b = "DUP_KEY2";
 --source include/get_handler_status_counts.inc
---echo # 8 locks (1 table, 3 partition lock/unlock)
 --echo # No pruning due to updating partitioning field.
 --echo # 1 read_key + 1 delete + 2 write (1 failed + 1 ok)
 --echo # 1 delete + 1 write due to moved to different partition
@@ -67,7 +59,6 @@ UPDATE t3 SET b = 'Updated' WHERE a = '2011-01-01 00:00:02';
 --source include/get_handler_status_counts.inc
 if ($default_update)
 {
---echo # 8 locks (1 table, 3 partition lock/unlock) 
 --echo # No pruning due to DEFAULT function on partitioning column
 --echo # 2 read_key + 1 read_rnd (1 read_key due to index lookup,
 --echo # 1 read_rnd + 1 read_key due to positioning before update)
@@ -76,18 +67,15 @@ if ($default_update)
 }
 if (!$default_update)
 {
---echo # 4 locks (1 table, 1 partition lock/unlock) 
 --echo # 1 read_key + 1 update (same partition)
 --echo # 1 (failed) write
 }
 FLUSH STATUS;
 REPLACE INTO t3 VALUES ('2011-01-01 00:00:04', 'Replace1');
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 FLUSH STATUS;
 REPLACE INTO t3 VALUES ('2011-01-01 00:00:04', 'Replace2');
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 --echo # 1 read_key + 1 update + 1 failed write
 
 --echo #
@@ -98,12 +86,10 @@ FLUSH STATUS;
 SET TIMESTAMP = 1234567894;
 REPLACE INTO t3 (b) VALUES ('Replace3');
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 FLUSH STATUS;
 SET TIMESTAMP = 1234567894;
 REPLACE INTO t3 (b) VALUES ('Replace4');
 --source include/get_handler_status_counts.inc
---echo # 4 locks (1 table, 1 partition lock/unlock)
 --echo # 1 read_key + 1 update + 1 failed write
 --sorted_result
 SELECT * FROM t3;

mysql-test/r/partition_column.result

@@ -101,7 +101,7 @@ Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
 explain partitions select hex(a) from t1 where a like 'A_';
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	p0	ALL	NULL	NULL	NULL	NULL	2	50.00	Using where
+1	SIMPLE	t1	p0	ALL	NULL	NULL	NULL	NULL	1	100.00	Using where
 Warnings:
 Warning	1681	'PARTITIONS' is deprecated and will be removed in a future release.
 Note	1003	/* select#1 */ select hex(`test`.`t1`.`a`) AS `hex(a)` from `test`.`t1` where (`test`.`t1`.`a` like 'A_')
@@ -513,7 +513,7 @@ Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
 explain partitions select * from t1 where a = 1 AND b <= 'A' and c = 1;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	p0,p1	ALL	NULL	NULL	NULL	NULL	2	12.50	Using where
+1	SIMPLE	t1	p0,p1	ALL	NULL	NULL	NULL	NULL	1	100.00	Using where
 Warnings:
 Warning	1681	'PARTITIONS' is deprecated and will be removed in a future release.
 Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c` from `test`.`t1` where ((`test`.`t1`.`c` = 1) and (`test`.`t1`.`a` = 1) and (`test`.`t1`.`b` <= 'A'))
@@ -683,7 +683,7 @@ Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
 explain partitions select * from t1 where b < 4;
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	p0,p11,p12,p21	ALL	NULL	NULL	NULL	NULL	7	33.33	Using where
+1	SIMPLE	t1	p0,p11,p12,p21	ALL	NULL	NULL	NULL	NULL	6	33.33	Using where
 Warnings:
 Warning	1681	'PARTITIONS' is deprecated and will be removed in a future release.
 Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b` from `test`.`t1` where (`test`.`t1`.`b` < 4)

mysql-test/r/partition_column_prune.result

@@ -6,11 +6,10 @@ insert into t1 values ('a', NULL, 'd');
 analyze table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	analyze	status	OK
-explain partitions select * from t1 where a = 'a' AND c = 'd';
+explain select * from t1 where a = 'a' AND c = 'd';
 id	select_type	table	partitions	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t1	p0	ALL	NULL	NULL	NULL	NULL	2	25.00	Using where
+1	SIMPLE	t1	p0	ALL	NULL	NULL	NULL	NULL	1	100.00	Using where
 Warnings:
-Warning	1681	'PARTITIONS' is deprecated and will be removed in a future release.
 Note	1003	/* select#1 */ select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c` from `test`.`t1` where ((`test`.`t1`.`c` = 'd') and (`test`.`t1`.`a` = 'a'))
 select * from t1 where a = 'a' AND c = 'd';
 a	b	c

mysql-test/r/partition_debug_sync.result

@@ -130,4 +130,44 @@ SET DEBUG_SYNC="now SIGNAL release";
 SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
 AUTO_INCREMENT
 15
-DROP TABLE t1;
+#
+# Test with INSERT SELECT
+#
+TRUNCATE TABLE t1;
+CREATE TABLE t2 (a INT);
+INSERT INTO t1 VALUES (10);
+INSERT INTO t2 VALUES (3), (NULL), (4);
+SET DEBUG_SYNC="release_auto_increment SIGNAL auto_inc_held WAIT_FOR release";
+INSERT INTO t1 SELECT * FROM t2;
+# Con1
+SET DEBUG_SYNC="now WAIT_FOR auto_inc_held";
+INSERT INTO t1 VALUES (5);
+SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
+AUTO_INCREMENT
+12
+SET DEBUG_SYNC="now SIGNAL release";
+# Default
+# Reaping: INSERT INTO t1 SELECT * FROM t2;
+SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
+AUTO_INCREMENT
+12
+# Test with generating an auto inc value in Con1
+TRUNCATE TABLE t1;
+INSERT INTO t1 VALUES (10);
+SET DEBUG_SYNC="release_auto_increment SIGNAL auto_inc_held WAIT_FOR release";
+INSERT INTO t1 SELECT * FROM t2;
+# Con1
+SET DEBUG_SYNC="now WAIT_FOR auto_inc_held";
+INSERT INTO t1 VALUES (NULL);
+# Not really a bug, since no row has been assigned the number between
+# this and the next auto_inc value.
+SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
+AUTO_INCREMENT
+13
+SET DEBUG_SYNC="now SIGNAL release";
+# Default
+# Reaping: INSERT INTO t1 SELECT * FROM t2;
+SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
+AUTO_INCREMENT
+13
+DROP TABLE t1, t2;

mysql-test/r/partition_debug_sync_stmt.result

@@ -86,3 +86,4 @@ SELECT AUTO_INCREMENT FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1';
 AUTO_INCREMENT
 13
 DROP TABLE t1, t2;
+SET DEBUG_SYNC='RESET';

mysql-test/r/partition_error.result

@@ -742,8 +742,8 @@ CREATE TABLE t1 (id INT, purchased DATE)
 PARTITION BY RANGE(YEAR(purchased))
 SUBPARTITION BY HASH(TO_DAYS(purchased))
 (PARTITION p0 VALUES LESS THAN MAXVALUE
-DATA DIRECTORY = '/tmp/not-existing' 
-INDEX DIRECTORY = '/tmp/not-existing');
+DATA DIRECTORY = '/tmp/not-existing'
+  INDEX DIRECTORY = '/tmp/not-existing');
 Warnings:
 Warning	1618	<DATA DIRECTORY> option ignored
 Warning	1618	<INDEX DIRECTORY> option ignored
@@ -764,8 +764,8 @@ PARTITION BY RANGE(YEAR(purchased))
 SUBPARTITION BY HASH(TO_DAYS(purchased)) SUBPARTITIONS 2
 (PARTITION p0 VALUES LESS THAN MAXVALUE
 (SUBPARTITION sp0
-DATA DIRECTORY = '/tmp/not-existing' 
-INDEX DIRECTORY = '/tmp/not-existing',
+DATA DIRECTORY = '/tmp/not-existing'
+   INDEX DIRECTORY = '/tmp/not-existing',
 SUBPARTITION sp1));
 Warnings:
 Warning	1618	<DATA DIRECTORY> option ignored
@@ -785,8 +785,8 @@ DROP TABLE t1;
 CREATE TABLE t1 (id INT, purchased DATE)
 PARTITION BY RANGE(YEAR(purchased))
 (PARTITION p0 VALUES LESS THAN MAXVALUE
-DATA DIRECTORY = '/tmp/not-existing' 
-INDEX DIRECTORY = '/tmp/not-existing');
+DATA DIRECTORY = '/tmp/not-existing'
+  INDEX DIRECTORY = '/tmp/not-existing');
 Warnings:
 Warning	1618	<DATA DIRECTORY> option ignored
 Warning	1618	<INDEX DIRECTORY> option ignored
@@ -1173,7 +1173,7 @@ partitions 2
 (partition x1 values less than (4) tablespace ts1,
 partition x2 values less than (8) tablespace ts2);
 ERROR HY000: Constant, random or timezone-dependent expressions in (sub)partitioning function are not allowed
-CREATE TABLE t1 ( 
+CREATE TABLE t1 (
 a int not null,
 b int not null,
 c int not null,
@@ -1294,21 +1294,21 @@ a int not null,
 b int not null,
 c int not null,
 primary key (a,b))
-partition by key (a) 
+partition by key (a)
 subpartition by list (a+b)
 ( partition x1
 ( subpartition x11,
 subpartition x12),
-partition x2 
+partition x2
 ( subpartition x21,
 subpartition x22)
 );
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'list (a+b)
 ( partition x1
 ( subpartition x11,
 subpartition x12),
-partition x2 
-(' at line 7
+partition x2
+( ' at line 7
 CREATE TABLE t1 (
 a int not null,
 b int not null,
@@ -1340,7 +1340,7 @@ b int not null,
 c int not null,
 primary key(a,b))
 partition by list (3+4)
-partitions 2 
+partitions 2
 (partition x1 values in (4) tablespace ts1,
 partition x2 values in (8) tablespace ts2);
 ERROR HY000: Constant, random or timezone-dependent expressions in (sub)partitioning function are not allowed
@@ -1745,7 +1745,7 @@ ERROR HY000: Constant, random or timezone-dependent expressions in (sub)partitio
 DROP TABLE old;
 #
 # Bug #56709: Memory leaks at running the 5.1 test suite
-# 
+#
 CREATE TABLE t1 (a TIMESTAMP NOT NULL PRIMARY KEY);
 ALTER TABLE t1
 PARTITION BY RANGE (EXTRACT(DAY FROM a)) (
@@ -1851,3 +1851,39 @@ pMax	pMaxsp0	This is a long comment (2050 ascii characters)   50 pMax partition
 pMax	pMaxsp1	This is a long comment (2050 ascii characters)   50 pMax partition comment .80-!
 DROP TABLE t1;
 SET sql_mode = default;
+#
+# Test for increased coverage of misplaced record handling.
+#
+CREATE TABLE t1 (a int)
+PARTITION BY LIST (a)
+(PARTITION p1 VALUES IN (1),
+PARTITION p3 VALUES IN (3,5),
+PARTITION p6 VALUES IN (6,7));
+# Intentionally insert illegal rows into p1.
+CREATE TABLE t2 LIKE t1;
+ALTER TABLE t2 REMOVE PARTITIONING;
+INSERT INTO t2 VALUES (2),(3),(4),(6);
+ALTER TABLE t1 EXCHANGE PARTITION p1 WITH TABLE t2 WITHOUT VALIDATION;
+# These should fail gracefully (WHERE used to not trigger pruning!)
+CALL mtr.add_suppression(" corrupted: row in wrong partition: ");
+INSERT INTO t1 VALUES (2);
+ERROR HY000: Table has no partition for value 2
+INSERT INTO t1 PARTITION (p1) VALUES (3);
+ERROR HY000: Found a row not matching the given partition set
+UPDATE t1 SET a = 3 WHERE (a % 3) != 0;
+ERROR HY000: Table has no partition for value 2
+UPDATE t1 PARTITION (p1) SET a = 3 WHERE (a % 3) != 0;
+ERROR HY000: Table has no partition for value 2
+UPDATE t1 PARTITION (p1) SET a = 5 WHERE (a % 3) = 0;
+ERROR HY000: Found a row not matching the given partition set
+UPDATE t1 PARTITION (p1) SET a = 3 WHERE (a % 5) > 3;
+ERROR HY000: Table has no partition for value 4
+UPDATE t1 PARTITION (p1,p6) SET a = 7 WHERE (a % 7) > 5;
+ERROR HY000: Found a row in wrong partition 0. Correct is 2 a:6
+DELETE FROM t1 PARTITION (p1) WHERE (a % 7) > 2;
+ERROR HY000: Found a row not matching the given partition set
+DELETE FROM t1 PARTITION (p1,p6) WHERE (a % 7) > 5;
+ERROR HY000: Found a row in wrong partition 0. Correct is 2 a:6
+DELETE FROM t1 WHERE a > 0;
+ERROR HY000: Table has no partition for value 2
+DROP TABLE t1, t2;

mysql-test/r/partition_exchange.result

@@ -1515,7 +1515,7 @@ a	b
 163	163
 164	164
 200	Two hundred, end of values
-call mtr.add_suppression("Cannot find index PRIMARY in InnoDB index translation table.");
+call mtr.add_suppression("Cannot find index .PRIMARY. in InnoDB index translation table.");
 ALTER TABLE tsp REBUILD PARTITION p1;
 Got one of the listed errors
 ALTER TABLE tsp REORGANIZE PARTITION p1 INTO