Merge pull request osheroff#41 from gunnarmorling/40

osheroff#40 Adding support for column visibility as supported by MySQL 8.0.23

Author: osheroff

src/main/java/com/github/shyiko/mysql/binlog/event/TableMapEventMetadata.java

@@ -36,6 +36,7 @@ public class TableMapEventMetadata implements EventData {
     private Map<Integer, Integer> primaryKeysWithPrefix;
     private DefaultCharset enumAndSetDefaultCharset;
     private List<Integer> enumAndSetColumnCharsets;
+    private BitSet visibility;
 
     public BitSet getSignedness() {
         return signedness;
@@ -125,6 +126,14 @@ public void setEnumAndSetColumnCharsets(List<Integer> enumAndSetColumnCharsets)
         this.enumAndSetColumnCharsets = enumAndSetColumnCharsets;
     }
 
+    public BitSet getVisibility() {
+        return visibility;
+    }
+
+    public void setVisibility(BitSet visibility) {
+        this.visibility = visibility;
+    }
+
     @Override
     public String toString() {
         final StringBuilder sb = new StringBuilder();
@@ -163,6 +172,8 @@ public String toString() {
         sb.append(", enumAndSetColumnCharsets=").append(enumAndSetColumnCharsets == null ? "null" : "");
         appendList(sb, enumAndSetColumnCharsets);
 
+        sb.append(",visibility=").append(visibility);
+
         sb.append('}');
         return sb.toString();
     }

src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventDataDeserializer.java

@@ -46,6 +46,7 @@ public TableMapEventData deserialize(ByteArrayInputStream inputStream) throws IO
         if (metadataLength > 0) {
             metadata = metadataDeserializer.deserialize(
                 new ByteArrayInputStream(inputStream.read(metadataLength)),
+                eventData.getColumnTypes().length,
                 numericColumnCount(eventData.getColumnTypes())
             );
         }

src/main/java/com/github/shyiko/mysql/binlog/event/deserialization/TableMapEventMetadataDeserializer.java

@@ -26,13 +26,17 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * @author <a href="mailto:[email protected]">Ahmed Abdul Hamid</a>
  */
 public class TableMapEventMetadataDeserializer {
 
-    public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int nIntColumns) throws IOException {
+    private final Logger logger = Logger.getLogger(getClass().getName());
+
+    public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int nColumns, int nNumericColumns) throws IOException {
         int remainingBytes = inputStream.available();
         if (remainingBytes <= 0) {
             return null;
@@ -41,15 +45,21 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
         TableMapEventMetadata result = new TableMapEventMetadata();
 
         for (; remainingBytes > 0; inputStream.enterBlock(remainingBytes)) {
-            MetadataFieldType fieldType = MetadataFieldType.byCode(inputStream.readInteger(1));
+            int code = inputStream.readInteger(1);
+
+            MetadataFieldType fieldType = MetadataFieldType.byCode(code);
+            if (fieldType == null) {
+                throw new IOException("Unsupported table metadata field type " + code);
+            }
+
             int fieldLength = inputStream.readPackedInteger();
 
             remainingBytes = inputStream.available();
             inputStream.enterBlock(fieldLength);
 
             switch (fieldType) {
                 case SIGNEDNESS:
-                    result.setSignedness(readSignedness(inputStream, nIntColumns));
+                    result.setSignedness(readBooleanList(inputStream, nNumericColumns));
                     break;
                 case DEFAULT_CHARSET:
                     result.setDefaultCharset(readDefaultCharset(inputStream));
@@ -80,17 +90,24 @@ public TableMapEventMetadata deserialize(ByteArrayInputStream inputStream, int n
                     break;
                 case ENUM_AND_SET_COLUMN_CHARSET:
                     result.setEnumAndSetColumnCharsets(readIntegers(inputStream));
+                case VISIBILITY:
+                    result.setVisibility(readBooleanList(inputStream, nColumns));
+                    break;
+                case UNKNOWN_METADATA_FIELD_TYPE:
+                    if (logger.isLoggable(Level.FINE)) {
+                        logger.fine("Received metadata field of unknown type");
+                    }
                     break;
                 default:
                     inputStream.enterBlock(remainingBytes);
-                    throw new IOException("Unsupported table metadata field type " + fieldType);
+                    throw new IOException("Unsupported table metadata field type " + code);
             }
             remainingBytes -= fieldLength;
         }
         return result;
     }
 
-    private static BitSet readSignedness(ByteArrayInputStream inputStream, int length) throws IOException {
+    private static BitSet readBooleanList(ByteArrayInputStream inputStream, int length) throws IOException {
         BitSet result = new BitSet();
         // according to MySQL internals the amount of storage required for N columns is INT((N+7)/8) bytes
         byte[] bytes = inputStream.read((length + 7) >> 3);
@@ -162,7 +179,9 @@ private enum MetadataFieldType {
         SIMPLE_PRIMARY_KEY(8),              // The primary key without any prefix
         PRIMARY_KEY_WITH_PREFIX(9),         // The primary key with some prefix
         ENUM_AND_SET_DEFAULT_CHARSET(10),   // Charsets of ENUM and SET columns
-        ENUM_AND_SET_COLUMN_CHARSET(11);    // Charsets of ENUM and SET columns
+        ENUM_AND_SET_COLUMN_CHARSET(11),    // Charsets of ENUM and SET columns
+        VISIBILITY(12),                     // Column visibility (8.0.23 and newer)
+        UNKNOWN_METADATA_FIELD_TYPE(128);   // Returned with binlog-row-metadata=FULL from MySQL 8.0 in some cases
 
         private final int code;
 

src/test/java/com/github/shyiko/mysql/binlog/BinaryLogClientIntegrationTest.java

@@ -115,7 +115,9 @@ public class BinaryLogClientIntegrationTest {
     protected MysqlVersion mysqlVersion;
 
     protected MysqlOnetimeServerOptions getOptions() {
-        return null;
+        MysqlOnetimeServerOptions options = new MysqlOnetimeServerOptions();
+        options.fullRowMetaData = true;
+        return options;
     }
 
     @BeforeClass
@@ -1140,6 +1142,21 @@ public void execute(final ResultSet rs) throws SQLException {
         });
     }
 
+    @Test
+    public void testMySQL8InvisibleColumn() throws Exception {
+        if ( !mysqlVersion.atLeast(8, 0) )
+            throw new SkipException("skipping mysql8 invisible column test");
+
+        master.execute("drop table if exists test_invisible_column");
+        master.execute("create table test_invisible_column (\n"
+                + "id BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,\n"
+                + "name varchar(100) not null,\n"
+                + "created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP INVISIBLE\n"
+                + ");");
+        master.execute("insert into test_invisible_column (name) values ('User 1')");
+        eventListener.waitFor(WriteRowsEventData.class, 1, DEFAULT_TIMEOUT);
+    }
+
     @AfterMethod
     public void afterEachTest() throws Exception {
         final CountDownLatch latch = new CountDownLatch(1);

src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServer.java

@@ -63,6 +63,11 @@ public void boot() throws Exception {
 			authPlugin = "--default-authentication-plugin=mysql_native_password";
 		}
 
+		String fullRowMetaData = "";
+		if ( getVersion().atLeast(8, 0) && options.fullRowMetaData ) {
+			fullRowMetaData = "--binlog-row-metadata=FULL";
+		}
+
 		ProcessBuilder pb = new ProcessBuilder(
 			dir + "/src/test/onetimeserver",
 			"--mysql-version=" + getVersionString(),
@@ -74,6 +79,7 @@ public void boot() throws Exception {
 			"--character-set-server=utf8",
 			"--sync_binlog=0",
 			"--default-time-zone=+00:00",
+			fullRowMetaData,
 			isRoot ? "--user=root" : "",
 			authPlugin,
 			gtidParams

src/test/java/com/github/shyiko/mysql/binlog/MysqlOnetimeServerOptions.java

@@ -5,4 +5,5 @@ public class MysqlOnetimeServerOptions {
     public boolean gtid = false;
     public MysqlOnetimeServer masterServer;
     public String extraParams;
+    public boolean fullRowMetaData;
 }