*: speed up the operation of "admin check table" (pingcap#8572, pingcap#1156… (pingcap#11676)

Author: zimulala

ddl/db_test.go

@@ -4616,6 +4616,22 @@ func (s *testDBSuite) TestAddIndexForGeneratedColumn(c *C) {
 	s.mustExec(c, "delete from t where y = 2155")
 	s.mustExec(c, "alter table t add index idx_y(y1)")
 	s.mustExec(c, "alter table t drop index idx_y")
+
+	// Fix issue 9311.
+	s.tk.MustExec("create table gcai_table (id int primary key);")
+	s.tk.MustExec("insert into gcai_table values(1);")
+	s.tk.MustExec("ALTER TABLE gcai_table ADD COLUMN d date DEFAULT '9999-12-31';")
+	s.tk.MustExec("ALTER TABLE gcai_table ADD COLUMN d1 date as (DATE_SUB(d, INTERVAL 31 DAY));")
+	s.tk.MustExec("ALTER TABLE gcai_table ADD INDEX idx(d1);")
+	s.tk.MustQuery("select * from gcai_table").Check(testkit.Rows("1 9999-12-31 9999-11-30"))
+	s.tk.MustQuery("select d1 from gcai_table use index(idx)").Check(testkit.Rows("9999-11-30"))
+	s.tk.MustExec("admin check table gcai_table")
+	// The column is PKIsHandle in generated column expression.
+	s.tk.MustExec("ALTER TABLE gcai_table ADD COLUMN id1 int as (id+5);")
+	s.tk.MustExec("ALTER TABLE gcai_table ADD INDEX idx1(id1);")
+	s.tk.MustQuery("select * from gcai_table").Check(testkit.Rows("1 9999-12-31 9999-11-30 6"))
+	s.tk.MustQuery("select id1 from gcai_table use index(idx1)").Check(testkit.Rows("6"))
+	s.tk.MustExec("admin check table gcai_table")
 }
 
 func (s *testDBSuite) TestModifyColumnCharset(c *C) {

ddl/index.go

@@ -519,7 +519,7 @@ func mergeAddIndexCtxToResult(taskCtx *addIndexTaskContext, result *addIndexResu
 
 func newAddIndexWorker(sessCtx sessionctx.Context, worker *worker, id int, t table.PhysicalTable, indexInfo *model.IndexInfo, decodeColMap map[int64]decoder.Column) *addIndexWorker {
 	index := tables.NewIndex(t.GetPhysicalID(), t.Meta(), indexInfo)
-	rowDecoder := decoder.NewRowDecoder(t.Cols(), decodeColMap)
+	rowDecoder := decoder.NewRowDecoder(t, decodeColMap)
 	return &addIndexWorker{
 		id:          id,
 		ddlWorker:   worker,
@@ -549,7 +549,7 @@ func (w *addIndexWorker) getIndexRecord(handle int64, recordKey []byte, rawRecor
 	cols := t.Cols()
 	idxInfo := w.index.Meta()
 	sysZone := timeutil.SystemLocation()
-	_, err := w.rowDecoder.DecodeAndEvalRowWithMap(w.sessCtx, rawRecord, time.UTC, sysZone, w.rowMap)
+	_, err := w.rowDecoder.DecodeAndEvalRowWithMap(w.sessCtx, handle, rawRecord, time.UTC, sysZone, w.rowMap)
 	if err != nil {
 		return nil, errors.Trace(errCantDecodeIndex.GenWithStackByArgs(err))
 	}

executor/admin_test.go

@@ -101,10 +101,10 @@ func (s *testSuite) TestAdminRecoverIndex(c *C) {
 	c.Assert(err, IsNil)
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
-	_, err = tk.Exec("admin check table admin_test")
+	err = tk.ExecToErr("admin check table admin_test")
 	c.Assert(err, NotNil)
 	c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue)
-	_, err = tk.Exec("admin check index admin_test c2")
+	err = tk.ExecToErr("admin check index admin_test c2")
 	c.Assert(err, NotNil)
 
 	r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
@@ -125,7 +125,7 @@ func (s *testSuite) TestAdminRecoverIndex(c *C) {
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
 
-	_, err = tk.Exec("admin check index admin_test c2")
+	err = tk.ExecToErr("admin check index admin_test c2")
 	c.Assert(err, NotNil)
 	r = tk.MustQuery("admin recover index admin_test c2")
 	r.Check(testkit.Rows("1 5"))
@@ -147,9 +147,9 @@ func (s *testSuite) TestAdminRecoverIndex(c *C) {
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
 
-	_, err = tk.Exec("admin check table admin_test")
+	err = tk.ExecToErr("admin check table admin_test")
 	c.Assert(err, NotNil)
-	_, err = tk.Exec("admin check index admin_test c2")
+	err = tk.ExecToErr("admin check index admin_test c2")
 	c.Assert(err, NotNil)
 
 	r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
@@ -271,9 +271,9 @@ func (s *testSuite) TestAdminCleanupIndex(c *C) {
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
 
-	_, err = tk.Exec("admin check table admin_test")
+	err = tk.ExecToErr("admin check table admin_test")
 	c.Assert(err, NotNil)
-	_, err = tk.Exec("admin check index admin_test c2")
+	err = tk.ExecToErr("admin check index admin_test c2")
 	c.Assert(err, NotNil)
 	r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c2)")
 	r.Check(testkit.Rows("11"))
@@ -283,9 +283,9 @@ func (s *testSuite) TestAdminCleanupIndex(c *C) {
 	r.Check(testkit.Rows("6"))
 	tk.MustExec("admin check index admin_test c2")
 
-	_, err = tk.Exec("admin check table admin_test")
+	err = tk.ExecToErr("admin check table admin_test")
 	c.Assert(err, NotNil)
-	_, err = tk.Exec("admin check index admin_test c3")
+	err = tk.ExecToErr("admin check index admin_test c3")
 	c.Assert(err, NotNil)
 	r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(c3)")
 	r.Check(testkit.Rows("9"))
@@ -332,9 +332,9 @@ func (s *testSuite) TestAdminCleanupIndexPKNotHandle(c *C) {
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
 
-	_, err = tk.Exec("admin check table admin_test")
+	err = tk.ExecToErr("admin check table admin_test")
 	c.Assert(err, NotNil)
-	_, err = tk.Exec("admin check index admin_test `primary`")
+	err = tk.ExecToErr("admin check index admin_test `primary`")
 	c.Assert(err, NotNil)
 	r = tk.MustQuery("SELECT COUNT(*) FROM admin_test USE INDEX(`primary`)")
 	r.Check(testkit.Rows("6"))
@@ -384,11 +384,11 @@ func (s *testSuite) TestAdminCleanupIndexMore(c *C) {
 	err = txn.Commit(context.Background())
 	c.Assert(err, IsNil)
 
-	_, err = tk.Exec("admin check table admin_test")
+	err = tk.ExecToErr("admin check table admin_test")
 	c.Assert(err, NotNil)
-	_, err = tk.Exec("admin check index admin_test c1")
+	err = tk.ExecToErr("admin check index admin_test c1")
 	c.Assert(err, NotNil)
-	_, err = tk.Exec("admin check index admin_test c2")
+	err = tk.ExecToErr("admin check index admin_test c2")
 	c.Assert(err, NotNil)
 	r := tk.MustQuery("SELECT COUNT(*) FROM admin_test")
 	r.Check(testkit.Rows("3"))
@@ -409,6 +409,112 @@ func (s *testSuite) TestAdminCleanupIndexMore(c *C) {
 	tk.MustExec("admin check table admin_test")
 }
 
+func (s *testSuite) TestAdminCheckTableFailed(c *C) {
+	tk := testkit.NewTestKit(c, s.store)
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists admin_test")
+	tk.MustExec("create table admin_test (c1 int, c2 int, c3 varchar(255) default '1', primary key(c1), key(c3), unique key(c2), key(c2, c3))")
+	tk.MustExec("insert admin_test (c1, c2, c3) values (-10, -20, 'y'), (-1, -10, 'z'), (1, 11, 'a'), (2, 12, 'b'), (5, 15, 'c'), (10, 20, 'd'), (20, 30, 'e')")
+
+	// Make some corrupted index. Build the index information.
+	s.ctx = mock.NewContext()
+	s.ctx.Store = s.store
+	is := s.domain.InfoSchema()
+	dbName := model.NewCIStr("test")
+	tblName := model.NewCIStr("admin_test")
+	tbl, err := is.TableByName(dbName, tblName)
+	c.Assert(err, IsNil)
+	tblInfo := tbl.Meta()
+	idxInfo := tblInfo.Indices[1]
+	indexOpr := tables.NewIndex(tblInfo.ID, tblInfo, idxInfo)
+	sc := s.ctx.GetSessionVars().StmtCtx
+	tk.Se.GetSessionVars().IndexLookupSize = 3
+	tk.Se.GetSessionVars().MaxChunkSize = 3
+
+	// Reduce one row of index.
+	// Table count > index count.
+	// Index c2 is missing 11.
+	txn, err := s.store.Begin()
+	c.Assert(err, IsNil)
+	err = indexOpr.Delete(sc, txn, types.MakeDatums(-10), -1)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	err = tk.ExecToErr("admin check table admin_test")
+	c.Assert(err.Error(), Equals,
+		"[executor:8003]admin_test err:[admin:1]index:<nil> != record:&admin.RecordData{Handle:-1, Values:[]types.Datum{types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:-10, b:[]uint8(nil), x:interface {}(nil)}}}")
+	c.Assert(executor.ErrAdminCheckTable.Equal(err), IsTrue)
+	r := tk.MustQuery("admin recover index admin_test c2")
+	r.Check(testkit.Rows("1 7"))
+	tk.MustExec("admin check table admin_test")
+
+	// Add one row of index.
+	// Table count < index count.
+	// Index c2 has one more values ​​than table data: 0, and the handle 0 hasn't correlative record.
+	txn, err = s.store.Begin()
+	c.Assert(err, IsNil)
+	_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(0), 0)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	err = tk.ExecToErr("admin check table admin_test")
+	c.Assert(err.Error(), Equals, "handle 0, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:0, b:[]uint8(nil), x:interface {}(nil)} != record:<nil>")
+
+	// Add one row of index.
+	// Table count < index count.
+	// Index c2 has two more values ​​than table data: 10, 13, and these handles have correlative record.
+	txn, err = s.store.Begin()
+	c.Assert(err, IsNil)
+	err = indexOpr.Delete(sc, txn, types.MakeDatums(0), 0)
+	c.Assert(err, IsNil)
+	// Make sure the index value "19" is smaller "21". Then we scan to "19" before "21".
+	_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(19), 10)
+	c.Assert(err, IsNil)
+	_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(13), 2)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	err = tk.ExecToErr("admin check table admin_test")
+	c.Assert(err.Error(), Equals, "col c2, handle 2, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:13, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:12, b:[]uint8(nil), x:interface {}(nil)}")
+
+	// Table count = index count.
+	// Two indices have the same handle.
+	txn, err = s.store.Begin()
+	c.Assert(err, IsNil)
+	err = indexOpr.Delete(sc, txn, types.MakeDatums(13), 2)
+	c.Assert(err, IsNil)
+	err = indexOpr.Delete(sc, txn, types.MakeDatums(12), 2)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	err = tk.ExecToErr("admin check table admin_test")
+	c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}")
+
+	// Table count = index count.
+	// Index c2 has one line of data is 19, the corresponding table data is 20.
+	txn, err = s.store.Begin()
+	c.Assert(err, IsNil)
+	_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(12), 2)
+	c.Assert(err, IsNil)
+	err = indexOpr.Delete(sc, txn, types.MakeDatums(20), 10)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	err = tk.ExecToErr("admin check table admin_test")
+	c.Assert(err.Error(), Equals, "col c2, handle 10, index:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:19, b:[]uint8(nil), x:interface {}(nil)} != record:types.Datum{k:0x1, collation:0x0, decimal:0x0, length:0x0, i:20, b:[]uint8(nil), x:interface {}(nil)}")
+
+	// Recover records.
+	txn, err = s.store.Begin()
+	c.Assert(err, IsNil)
+	err = indexOpr.Delete(sc, txn, types.MakeDatums(19), 10)
+	c.Assert(err, IsNil)
+	_, err = indexOpr.Create(s.ctx, txn, types.MakeDatums(20), 10)
+	c.Assert(err, IsNil)
+	err = txn.Commit(context.Background())
+	c.Assert(err, IsNil)
+	tk.MustExec("admin check table admin_test")
+}
+
 func (s *testSuite) TestAdminCheckTable(c *C) {
 	// test NULL value.
 	tk := testkit.NewTestKit(c, s.store)
@@ -466,22 +572,22 @@ func (s *testSuite) TestAdminCheckTable(c *C) {
 
 	// Test index in virtual generated column.
 	tk.MustExec(`drop table if exists test`)
-	tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')) , index idxc(c));`)
+	tk.MustExec(`create table test ( b json , c int as (JSON_EXTRACT(b,'$.d')), index idxc(c));`)
 	tk.MustExec(`INSERT INTO test set b='{"d": 100}';`)
 	tk.MustExec(`admin check table test;`)
 	// Test prefix index.
 	tk.MustExec(`drop table if exists t`)
 	tk.MustExec(`CREATE TABLE t (
-  			ID CHAR(32) NOT NULL,
-  			name CHAR(32) NOT NULL,
-  			value CHAR(255),
-  			INDEX indexIDname (ID(8),name(8)));`)
+	  			ID CHAR(32) NOT NULL,
+	  			name CHAR(32) NOT NULL,
+	  			value CHAR(255),
+	  			INDEX indexIDname (ID(8),name(8)));`)
 	tk.MustExec(`INSERT INTO t VALUES ('keyword','urlprefix','text/ /text');`)
 	tk.MustExec(`admin check table t;`)
 
 	tk.MustExec("use mysql")
 	tk.MustExec(`admin check table test.t;`)
-	_, err := tk.Exec("admin check table t")
+	err := tk.ExecToErr("admin check table t")
 	c.Assert(err, NotNil)
 
 	// test add index on time type column which have default value
@@ -521,7 +627,7 @@ func (s *testSuite) TestAdminCheckTable(c *C) {
 	tk.MustExec(`drop table if exists t1`)
 	tk.MustExec(`create table t1 (a decimal(2,1), index(a))`)
 	tk.MustExec(`insert into t1 set a='1.9'`)
-	_, err = tk.Exec(`alter table t1 modify column a decimal(3,2);`)
+	err = tk.ExecToErr(`alter table t1 modify column a decimal(3,2);`)
 	c.Assert(err, NotNil)
 	c.Assert(err.Error(), Equals, "[ddl:203]unsupported modify decimal column precision")
 	tk.MustExec(`delete from t1;`)

executor/builder.go

@@ -271,8 +271,8 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor {
 		b.err = errors.Trace(err)
 		return nil
 	}
-	readerExec.ranges = ranger.FullRange()
-	readerExec.isCheckOp = true
+
+	buildIndexLookUpChecker(b, v.IndexLookUpReader, readerExec)
 
 	e := &CheckIndexExec{
 		baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()),
@@ -285,12 +285,59 @@ func (b *executorBuilder) buildCheckIndex(v *plannercore.CheckIndex) Executor {
 	return e
 }
 
+// buildIndexLookUpChecker builds check information to IndexLookUpReader.
+func buildIndexLookUpChecker(b *executorBuilder, readerPlan *plannercore.PhysicalIndexLookUpReader,
+	readerExec *IndexLookUpExecutor) {
+	is := readerPlan.IndexPlans[0].(*plannercore.PhysicalIndexScan)
+	readerExec.dagPB.OutputOffsets = make([]uint32, 0, len(is.Index.Columns))
+	for i := 0; i <= len(is.Index.Columns); i++ {
+		readerExec.dagPB.OutputOffsets = append(readerExec.dagPB.OutputOffsets, uint32(i))
+	}
+	readerExec.ranges = ranger.FullRange()
+	ts := readerPlan.TablePlans[0].(*plannercore.PhysicalTableScan)
+	readerExec.handleIdx = ts.HandleIdx
+
+	tps := make([]*types.FieldType, 0, len(is.Columns)+1)
+	for _, col := range is.Columns {
+		tps = append(tps, &col.FieldType)
+	}
+	tps = append(tps, types.NewFieldType(mysql.TypeLonglong))
+	readerExec.checkIndexValue = &checkIndexValue{genExprs: is.GenExprs, idxColTps: tps}
+
+	colNames := make([]string, 0, len(is.Columns))
+	for _, col := range is.Columns {
+		colNames = append(colNames, col.Name.O)
+	}
+	var err error
+	readerExec.idxTblCols, err = table.FindCols(readerExec.table.Cols(), colNames, true)
+	if err != nil {
+		b.err = errors.Trace(err)
+		return
+	}
+}
+
 func (b *executorBuilder) buildCheckTable(v *plannercore.CheckTable) Executor {
+	readerExecs := make([]*IndexLookUpExecutor, 0, len(v.IndexLookUpReaders))
+	for _, readerPlan := range v.IndexLookUpReaders {
+		readerExec, err := buildNoRangeIndexLookUpReader(b, readerPlan)
+		if err != nil {
+			b.err = errors.Trace(err)
+			return nil
+		}
+		buildIndexLookUpChecker(b, readerPlan, readerExec)
+
+		readerExecs = append(readerExecs, readerExec)
+	}
+
 	e := &CheckTableExec{
 		baseExecutor: newBaseExecutor(b.ctx, v.Schema(), v.ExplainID()),
-		tables:       v.Tables,
+		dbName:       v.DBName,
+		tblInfo:      v.TblInfo,
+		indices:      v.Indices,
 		is:           b.is,
-		genExprs:     v.GenExprs,
+		srcs:         readerExecs,
+		exitCh:       make(chan struct{}),
+		retCh:        make(chan error, len(v.Indices)),
 	}
 	return e
 }