WL#16264: InnoDB: Add support for bulk load of large objects

Support for bulk load of large objects. This means that when column data
size exceeds a threshold, then it will be stored externally (external to
the leaf pages of the B-tree), only storing a blob reference in the leaf
pages.

The data types whose data can be stored externally are - TINYBLOB, BLOB,
MEDIUMBLOB, LONGBLOB, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, JSON, POINT,
LINESTRING, POLYGON, MULTIPOINT, MULTILINESTRING, MULTIPOLYGON,
GEOMETRYCOLLECTION, GEOMETRY, VARCHAR, VARBINARY.

Change-Id: I7a56e68e524e4c482ec466d65bb6b305701c2e05

Author: gurusami

include/mysql/components/services/bulk_data_service.h

@@ -0,0 +1,2 @@
+# Run the bulk_load test suite that is not part of the default
+perl mysql-test-run.pl --force --timer --comment=bulk --vardir=var-bulk --suite=innodb,bulk_load

mysql-test/collections/mysql-trunk-win.push

@@ -0,0 +1,2 @@
+# Run the bulk_load test suite that is not part of the default
+perl mysql-test-run.pl --force --timer --comment=bulk --vardir=var-bulk --suite=innodb,bulk_load

mysql-test/collections/mysql-trunk-wl16264.push

@@ -10494,6 +10494,12 @@ ER_NON_DML_DYNAMIC_PARAMETERS
 ER_DEPRECATE_NON_COMPOSABLE_MULTIPLE_ENGINE
   eng "Combining the storage engines %.64s and %.64s is deprecated, but the statement or transaction updates both the %.64s table %.64s.%.64s and the %.64s table %.64s.%.64s."
 
+ER_BULK_JSON_INVALID
+  eng "Bulk load reports invalid JSON: error='%s', table='%s', field='%s'"
+
+ER_BULK_BLOB_TOO_BIG
+  eng "Column data bigger than %zu bytes for column='%s' of table='%s'"
+
 #
 #  End of 9.x error messages (server-to-client).
 #

share/messages_to_clients.txt

@@ -12585,7 +12585,7 @@ ER_IB_ERR_CORRUPT_TABLESPACE_UNRECOVERABLE
   eng "Tablespace '%u' mentioned in the redo log is corrupted in a way it is unrecoverable by double-write buffer, so further redo log recovery is not possible!"
 
 ER_IB_BULK_LOAD_STATS_WARN
-  eng "%s: Failed to save statistics for table=%s err=%zu"
+  eng "%s: Failed to save statistics for table=%s err=%zu trx_err=%zu"
 
 ER_COMPONENT_MASKING_INVALID_FLUSH_INTERVAL_VALUE
    eng "Invalid flush interval specified: %lu. Valid values are 0 (off) or greater than or equal to %d. Adjusting to %d."
@@ -12746,6 +12746,12 @@ ER_DD_UPDATE_DATADIR_FLAG_FAIL
 ER_IB_MSG_FIL_STATE_MOVED_PREV
   eng "%s DD ID: %llu - Tablespace %u, name '%s', found at '%s' outside default data directory, but DD misses info about DATA DIRECTORY"
 
+ER_LOAD_BULK_JSON_ERROR
+  eng "Bulk CSV parser reports JSON error: %s at %zu"
+
+ER_IB_BULK_LOAD_STATS_INFO
+  eng "%s: %s: Saving statistics for table=%s err=%zu trx_err=%zu"
+
 # DO NOT add server-to-client messages here;
 # they go in messages_to_clients.txt
 # in the same directory as this file.

share/messages_to_error_log.txt

@@ -4575,6 +4575,7 @@ class handler {
 
  public:
   typedef ulonglong Table_flags;
+  using Blob_context = void *;
 
  protected:
   TABLE_SHARE *table_share;          /* The table definition */
@@ -5102,6 +5103,53 @@ class handler {
     return HA_ERR_UNSUPPORTED;
   }
 
+  /** Open a blob for write operation.
+  @param[in,out]  thd         user session
+  @param[in,out]  load_ctx    load execution context
+  @param[in]      thread_idx  index of the thread executing
+  @param[out]     blob_ctx    a blob context
+  @param[out]     blobref     a blob reference to be placed in the record.
+  @return 0 on success, error code on failure */
+  virtual int open_blob(THD *thd [[maybe_unused]],
+                        void *load_ctx [[maybe_unused]],
+                        size_t thread_idx [[maybe_unused]],
+                        Blob_context &blob_ctx [[maybe_unused]],
+                        unsigned char *blobref [[maybe_unused]]) {
+    return HA_ERR_UNSUPPORTED;
+  }
+
+  /** Write to a blob
+  @param[in,out]  thd         user session
+  @param[in,out]  load_ctx    load execution context
+  @param[in]      thread_idx  index of the thread executing
+  @param[in]      blob_ctx    a blob context
+  @param[in]      data        data to be written to blob.
+  @param[in]      data_len    length of data to be written in bytes.
+  @return 0 on success, error code on failure */
+  virtual int write_blob(THD *thd [[maybe_unused]],
+                         void *load_ctx [[maybe_unused]],
+                         size_t thread_idx [[maybe_unused]],
+                         Blob_context blob_ctx [[maybe_unused]],
+                         unsigned char *blobref [[maybe_unused]],
+                         const unsigned char *data [[maybe_unused]],
+                         size_t data_len [[maybe_unused]]) {
+    return HA_ERR_UNSUPPORTED;
+  }
+
+  /** Close the blob
+  @param[in,out]  thd         user session
+  @param[in,out]  load_ctx    load execution context
+  @param[in]      thread_idx  index of the thread executing
+  @param[in]      blob_ctx    a blob context
+  @return 0 on success, error code on failure */
+  virtual int close_blob(THD *thd [[maybe_unused]],
+                         void *load_ctx [[maybe_unused]],
+                         size_t thread_idx [[maybe_unused]],
+                         Blob_context blob_ctx [[maybe_unused]],
+                         unsigned char *blobref [[maybe_unused]]) {
+    return HA_ERR_UNSUPPORTED;
+  }
+
   /** End bulk load operation. Must be called after all execution threads have
   completed. Must be called even if the bulk load execution failed.
   @param[in,out]  thd       user session