Online DDL/VReplication: support non-UTF8 character sets

go/mysql/constants.go

@@ -16,7 +16,13 @@ limitations under the License.
 
 package mysql
 
-import "strings"
+import (
+	"strings"
+
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/simplifiedchinese"
+)
 
 const (
 	// MaxPacketSize is the maximum payload length of a packet
@@ -626,6 +632,41 @@ var CharacterSetMap = map[string]uint8{
 	"eucjpms":  97,
 }
 
+// CharacterSetEncoding maps a charset name to a golang encoder.
+// golang does not support encoders for all MySQL charsets.
+// A charset not in this map is unsupported.
+// A trivial encoding (e.g. utf8) has a `nil` encoder
+var CharacterSetEncoding = map[string]encoding.Encoding{
+	"cp850":   charmap.CodePage850,
+	"koi8r":   charmap.KOI8R,
+	"latin1":  charmap.Windows1252,
+	"latin2":  charmap.ISO8859_2,
+	"ascii":   nil,
+	"hebrew":  charmap.ISO8859_8,
+	"greek":   charmap.ISO8859_7,
+	"cp1250":  charmap.Windows1250,
+	"gbk":     simplifiedchinese.GBK,
+	"latin5":  charmap.ISO8859_9,
+	"utf8":    nil,
+	"cp866":   charmap.CodePage866,
+	"cp852":   charmap.CodePage852,
+	"latin7":  charmap.ISO8859_13,
+	"utf8mb4": nil,
+	"cp1251":  charmap.Windows1251,
+	"cp1256":  charmap.Windows1256,
+	"cp1257":  charmap.Windows1257,
+	"binary":  nil,
+}
+
+// ReverseCharacterSetMap maps a charset integer code to charset name
+var ReverseCharacterSetMap = map[uint8]string{}
+
+func init() {
+	for c, i := range CharacterSetMap {
+		ReverseCharacterSetMap[i] = c
+	}
+}
+
 // IsNum returns true if a MySQL type is a numeric value.
 // It is the same as IS_NUM defined in mysql.h.
 func IsNum(typ uint8) bool {

go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8-80/create.sql

@@ -0,0 +1,30 @@
+drop table if exists onlineddl_test;
+create table onlineddl_test (
+  id int auto_increment,
+  t1 varchar(128)  charset latin1 collate latin1_swedish_ci,
+  t2 varchar(128)  charset latin1 collate latin1_swedish_ci,
+  tutf8 varchar(128) charset utf8,
+  tutf8mb4 varchar(128) charset utf8mb4,
+  tlatin1 varchar(128)  charset latin1 collate latin1_swedish_ci,
+  primary key(id)
+) auto_increment=1;
+
+insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
+insert into onlineddl_test values (null, 'átesting', 'átesting', 'átesting', 'átesting', 'átesting');
+insert into onlineddl_test values (null, 'testátest', 'testátest', 'testátest', '🍻😀', 'átesting');
+
+drop event if exists onlineddl_test;
+delimiter ;;
+create event onlineddl_test
+  on schedule every 1 second
+  starts current_timestamp
+  ends current_timestamp + interval 60 second
+  on completion not preserve
+  enable
+  do
+begin
+  insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
+  insert into onlineddl_test values (null, 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog');
+  insert into onlineddl_test values (null, 'testátest-binlog', 'testátest-binlog', 'testátest-binlog', '🍻😀', 'átesting-binlog');
+  insert into onlineddl_test values (null, 'átesting-bnull', 'átesting-bnull', 'átesting-bnull', null, null);
+end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8-80/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6|5.7)

go/test/endtoend/onlineddl/vrepl_suite/untestdata/alter-charset-all-dml/create.sql → go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8/create.sql

@@ -5,10 +5,13 @@ create table onlineddl_test (
   t2 varchar(128)  charset latin1 collate latin1_swedish_ci,
   tutf8 varchar(128) charset utf8,
   tutf8mb4 varchar(128) charset utf8mb4,
-  random_value varchar(128) charset ascii,
+  tlatin1 varchar(128)  charset latin1 collate latin1_swedish_ci,
   primary key(id)
 ) auto_increment=1;
 
+insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
+insert into onlineddl_test values (null, 'átesting', 'átesting', 'átesting', 'átesting', 'átesting');
+
 drop event if exists onlineddl_test;
 delimiter ;;
 create event onlineddl_test
@@ -20,9 +23,6 @@ create event onlineddl_test
   do
 begin
   insert into onlineddl_test values (null, md5(rand()), md5(rand()), md5(rand()), md5(rand()), md5(rand()));
-  insert into onlineddl_test values (null, 'átesting', 'átesting', 'átesting', 'átesting', md5(rand()));
-  insert into onlineddl_test values (null, 'átesting_del', 'átesting', 'átesting', 'átesting', md5(rand()));
-  insert into onlineddl_test values (null, 'testátest', 'testátest', 'testátest', '🍻😀', md5(rand()));
-  update onlineddl_test set t1='átesting2' where t1='átesting' order by id desc limit 1;
-  delete from onlineddl_test where t1='átesting_del' order by id desc limit 1;
+  insert into onlineddl_test values (null, 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog', 'átesting-binlog');
+  insert into onlineddl_test values (null, 'átesting-bnull', 'átesting-bnull', 'átesting-bnull', null, null);
 end ;;

go/test/endtoend/onlineddl/vrepl_suite/testdata/alter-charset-non-utf8/ignore_versions

@@ -0,0 +1 @@
+(5.5|5.6)

go/test/endtoend/onlineddl/vrepl_suite/untestdata/convert-utf8mb4/create.sql → go/test/endtoend/onlineddl/vrepl_suite/testdata/convert-utf8mb4/create.sql

@@ -7,9 +7,6 @@ create table onlineddl_test (
   primary key(id)
 ) auto_increment=1;
 
-insert into onlineddl_test values (null, 'átesting');
-
-
 insert into onlineddl_test values (null, 'Hello world, Καλημέρα κόσμε, コンニチハ', 'átesting0', 'initial');
 
 drop event if exists onlineddl_test;