From 66dd93a0a9e9a4a4a7927361461d0c5e867a1a47 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Thu, 10 Aug 2023 14:30:01 +0200 Subject: [PATCH] collations: Refactor to separate basic collation information from data This refactor separates the full collation data and tables from the basic collation information like names etc. With this separation, tools like `mysqlctld` and `vtbackup` can run without having to compile collation data into the binary at all, since they only depend on basic name functionality for the MySQL protocol bits but they don't need the full collations data package. Reduces the binary size significantly and removes it from in-memory in the process. Before ``` -rwxr-xr-x 1 dirkjan staff 41336562 Aug 10 14:25 mysqlctl* -rwxr-xr-x 1 dirkjan staff 42197394 Aug 10 14:25 mysqlctld* -rwxr-xr-x 1 dirkjan staff 67138226 Aug 10 14:25 vtbackup* ``` After ``` -rwxr-xr-x 1 dirkjan staff 35649490 Aug 10 14:25 mysqlctl* -rwxr-xr-x 1 dirkjan staff 36509970 Aug 10 14:25 mysqlctld* -rwxr-xr-x 1 dirkjan staff 61432706 Aug 10 14:25 vtbackup* ``` This step enforces the separation between these packages which means that lazy loading of actually used collation information also becomes possible and much simpler with this refactor. Signed-off-by: Dirkjan Bussink --- go.mod | 16 +- go.sum | 31 +- go/mysql/collations/coercion.go | 212 +---- go/mysql/collations/collation.go | 150 ---- go/mysql/collations/{ => colldata}/8bit.go | 15 +- .../collations/{ => colldata}/cached_size.go | 2 +- go/mysql/collations/colldata/collation.go | 374 ++++++++ go/mysql/collations/{ => colldata}/fuzz.go | 2 +- .../collations/{ => colldata}/fuzz_test.go | 6 +- go/mysql/collations/colldata/golden_test.go | 82 ++ .../collations/{ => colldata}/multibyte.go | 7 +- .../collations/{ => colldata}/mysqldata.go | 18 +- .../{ => colldata}/mysqlucadata.bin | Bin .../collations/{ => colldata}/mysqlucadata.go | 18 +- go/mysql/collations/{ => colldata}/uca.go | 13 +- .../{ => colldata}/uca_contraction_test.go | 2 +- .../{ => colldata}/uca_tables_test.go | 2 +- .../collations/{ => colldata}/uca_test.go | 2 +- go/mysql/collations/{ => colldata}/unicase.go | 2 +- go/mysql/collations/{ => colldata}/unicode.go | 11 +- .../collations/{ => colldata}/wildcard.go | 2 +- .../{ => colldata}/wildcard_test.go | 2 +- go/mysql/collations/env.go | 140 +-- go/mysql/collations/golden_test.go | 145 +--- .../collations/integration/charset_test.go | 4 +- .../collations/integration/coercion_test.go | 52 +- .../collations/integration/collations_test.go | 6 +- .../collations/integration/helpers_test.go | 10 +- .../integration/weight_string_test.go | 17 +- .../collations/integration/wildcard_test.go | 3 +- .../collations/internal/uca/fasttables.go | 16 + go/mysql/collations/mysqlversion.go | 807 +++++++++--------- go/mysql/collations/remote/collation.go | 13 +- go/mysql/collations/supported.go | 294 +++++++ ...ions_MySQL80.csv => collations_MySQL8.csv} | 0 .../tools/makecolldata/codegen/codegen.go | 20 + .../collations/tools/makecolldata/main.go | 4 +- .../tools/makecolldata/maketables.go | 6 +- .../tools/makecolldata/mysqldata.go | 22 +- .../tools/makecolldata/mysqlversions.go | 12 +- .../tools/maketestdata/maketestdata.go | 23 +- go/mysql/endtoend/query_test.go | 7 +- go/vt/dbconfigs/dbconfigs.go | 4 +- go/vt/schemadiff/table.go | 8 +- go/vt/vtgate/engine/aggregations.go | 2 +- go/vt/vtgate/engine/distinct.go | 2 +- go/vt/vtgate/engine/filter_test.go | 2 +- go/vt/vtgate/engine/hash_join.go | 6 +- go/vt/vtgate/engine/ordered_aggregate.go | 3 +- go/vt/vtgate/engine/route.go | 3 +- go/vt/vtgate/evalengine/api_compare.go | 3 +- go/vt/vtgate/evalengine/api_compare_test.go | 4 +- go/vt/vtgate/evalengine/api_hash.go | 7 +- go/vt/vtgate/evalengine/api_hash_test.go | 4 +- go/vt/vtgate/evalengine/cached_size.go | 2 +- go/vt/vtgate/evalengine/compare.go | 5 +- go/vt/vtgate/evalengine/compiler.go | 13 +- go/vt/vtgate/evalengine/compiler_asm.go | 45 +- go/vt/vtgate/evalengine/eval_bytes.go | 9 +- go/vt/vtgate/evalengine/eval_json.go | 5 +- go/vt/vtgate/evalengine/eval_result.go | 3 +- go/vt/vtgate/evalengine/expr_collate.go | 7 +- go/vt/vtgate/evalengine/expr_compare.go | 15 +- go/vt/vtgate/evalengine/expr_convert.go | 3 +- go/vt/vtgate/evalengine/fn_compare.go | 5 +- go/vt/vtgate/evalengine/fn_regexp.go | 42 +- go/vt/vtgate/evalengine/fn_string.go | 47 +- go/vt/vtgate/evalengine/format.go | 10 +- .../evalengine/integration/comparison_test.go | 2 +- .../evalengine/integration/fuzz_test.go | 9 +- go/vt/vtgate/evalengine/mysql_test.go | 2 +- go/vt/vtgate/evalengine/translate.go | 8 +- go/vt/vtgate/evalengine/translate_convert.go | 14 +- go/vt/vtgate/evalengine/translate_simplify.go | 4 +- go/vt/vtgate/evalengine/weights.go | 5 +- go/vt/vtgate/executor_test.go | 4 +- go/vt/vtgate/planbuilder/collations_test.go | 2 +- go/vt/vtgate/planbuilder/show_test.go | 4 +- go/vt/vtgate/vcursor_impl.go | 2 +- go/vt/vttablet/onlineddl/vrepl.go | 11 +- .../tabletmanager/vdiff/table_differ.go | 11 +- go/vt/vttablet/tabletmanager/vdiff/utils.go | 4 +- .../vdiff/workflow_differ_test.go | 5 +- .../vreplication/replicator_plan.go | 5 +- go/vt/wrangler/vdiff.go | 25 +- go/vt/wrangler/vdiff_test.go | 196 ++--- 86 files changed, 1818 insertions(+), 1324 deletions(-) rename go/mysql/collations/{ => colldata}/8bit.go (96%) rename go/mysql/collations/{ => colldata}/cached_size.go (98%) create mode 100644 go/mysql/collations/colldata/collation.go rename go/mysql/collations/{ => colldata}/fuzz.go (98%) rename go/mysql/collations/{ => colldata}/fuzz_test.go (96%) create mode 100644 go/mysql/collations/colldata/golden_test.go rename go/mysql/collations/{ => colldata}/multibyte.go (96%) rename go/mysql/collations/{ => colldata}/mysqldata.go (99%) rename go/mysql/collations/{ => colldata}/mysqlucadata.bin (100%) rename go/mysql/collations/{ => colldata}/mysqlucadata.go (99%) rename go/mysql/collations/{ => colldata}/uca.go (96%) rename go/mysql/collations/{ => colldata}/uca_contraction_test.go (99%) rename go/mysql/collations/{ => colldata}/uca_tables_test.go (99%) rename go/mysql/collations/{ => colldata}/uca_test.go (99%) rename go/mysql/collations/{ => colldata}/unicase.go (99%) rename go/mysql/collations/{ => colldata}/unicode.go (97%) rename go/mysql/collations/{ => colldata}/wildcard.go (99%) rename go/mysql/collations/{ => colldata}/wildcard_test.go (99%) create mode 100644 go/mysql/collations/supported.go rename go/mysql/collations/testdata/versions/{collations_MySQL80.csv => collations_MySQL8.csv} (100%) diff --git a/go.mod b/go.mod index 078b4db8922..26c9a81d84a 100644 --- a/go.mod +++ b/go.mod @@ -72,15 +72,15 @@ require ( go.etcd.io/etcd/client/pkg/v3 v3.5.8 go.etcd.io/etcd/client/v3 v3.5.8 go.uber.org/mock v0.2.0 - golang.org/x/crypto v0.8.0 // indirect - golang.org/x/mod v0.11.0 // indirect - golang.org/x/net v0.9.0 + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/mod v0.12.0 // indirect + golang.org/x/net v0.14.0 golang.org/x/oauth2 v0.7.0 - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 - golang.org/x/text v0.9.0 + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 + golang.org/x/text v0.12.0 golang.org/x/time v0.3.0 - golang.org/x/tools v0.8.0 + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 google.golang.org/api v0.121.0 google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect google.golang.org/grpc v1.55.0-dev @@ -107,7 +107,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 github.com/xlab/treeprint v1.2.0 go.uber.org/goleak v1.2.1 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.3.0 modernc.org/sqlite v1.20.3 ) diff --git a/go.sum b/go.sum index 5a3fa32201a..b7cd1967093 100644 --- a/go.sum +++ b/go.sum @@ -667,8 +667,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= -golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -705,8 +705,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= -golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -759,8 +759,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -784,8 +784,9 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -856,14 +857,14 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -875,8 +876,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -936,8 +937,8 @@ golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= -golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go/mysql/collations/coercion.go b/go/mysql/collations/coercion.go index 8e72ebf3c37..8b66c818cc0 100644 --- a/go/mysql/collations/coercion.go +++ b/go/mysql/collations/coercion.go @@ -19,8 +19,6 @@ package collations import ( "fmt" "unsafe" - - "vitess.io/vitess/go/mysql/collations/charset" ) func init() { @@ -95,11 +93,6 @@ const ( RepertoireUnicode ) -// Coercion is a function that will transform either the given argument -// arguments of the function into a specific character set. The `dst` argument -// will be used as the destination of the coerced argument, but it can be nil. -type Coercion func(dst, in []byte) ([]byte, error) - // TypedCollation is the Collation of a SQL expression, including its coercibility // and repertoire. type TypedCollation struct { @@ -112,208 +105,13 @@ func (tc TypedCollation) Valid() bool { return tc.Collation != Unknown } -func checkCompatibleCollations( - left Collation, leftCoercibility Coercibility, leftRepertoire Repertoire, - right Collation, rightCoercibility Coercibility, rightRepertoire Repertoire, -) bool { - leftCS := left.Charset() - rightCS := right.Charset() - - switch leftCS.(type) { - case charset.Charset_utf8mb4: - if leftCoercibility <= rightCoercibility { - return true - } - - case charset.Charset_utf32: - switch { - case leftCoercibility < rightCoercibility: - return true - case leftCoercibility == rightCoercibility: - if !charset.IsUnicode(rightCS) { - return true - } - if !left.IsBinary() { - return true - } - } - - case charset.Charset_utf8mb3, charset.Charset_ucs2, charset.Charset_utf16, charset.Charset_utf16le: - switch { - case leftCoercibility < rightCoercibility: - return true - case leftCoercibility == rightCoercibility: - if !charset.IsUnicode(rightCS) { - return true - } - } - } - - if rightRepertoire == RepertoireASCII { - switch { - case leftCoercibility < rightCoercibility: - return true - case leftCoercibility == rightCoercibility: - if leftRepertoire == RepertoireUnicode { - return true - } - } - } - - return false -} - -// CoercionOptions is used to configure how aggressive the algorithm can be -// when merging two different collations by transcoding them. -type CoercionOptions struct { - // ConvertToSuperset allows merging two different collations as long - // as the charset of one of them is a strict superset of the other. In - // order to operate on the two expressions, one of them will need to - // be transcoded. This transcoding will always be safe because the string - // with the smallest repertoire will be transcoded to its superset, which - // cannot fail. - ConvertToSuperset bool - - // ConvertWithCoercion allows merging two different collations by forcing - // a coercion as long as the coercibility of the two sides is lax enough. - // This will force a transcoding of one of the expressions even if their - // respective charsets are not a strict superset, so the resulting transcoding - // CAN fail depending on the content of their strings. - ConvertWithCoercion bool -} - -// MergeCollations returns a Coercion function for a pair of TypedCollation based -// on their coercibility. -// -// The function takes the typed collations for the two sides of a text operation -// (namely, a comparison or concatenation of two textual expressions). These typed -// collations includes the actual collation for the expression on each size, their -// coercibility values (see: Coercibility) and their respective repertoires, -// and returns the target collation (i.e. the collation into which the two expressions -// must be coerced, and a Coercion function. The Coercion function can be called repeatedly -// with the different values for the two expressions and will transcode either -// the left-hand or right-hand value to the appropriate charset so it can be -// collated against the other value. -// -// If the collations for both sides of the expressions are the same, the returned -// Coercion function will be a no-op. Likewise, if the two collations are not the same, -// but they are compatible and have the same charset, the Coercion function will also -// be a no-op. -// -// If the collations for both sides of the expression are not compatible, an error -// will be returned and the returned TypedCollation and Coercion will be nil. -func (env *Environment) MergeCollations(left, right TypedCollation, opt CoercionOptions) (TypedCollation, Coercion, Coercion, error) { - leftColl := left.Collation.Get() - rightColl := right.Collation.Get() - if leftColl == nil || rightColl == nil { - return TypedCollation{}, nil, nil, fmt.Errorf("unsupported TypeCollationID: %v / %v", left.Collation, right.Collation) - } - - leftCS := leftColl.Charset() - rightCS := rightColl.Charset() - - if left.Coercibility == CoerceExplicit && right.Coercibility == CoerceExplicit { - if left.Collation != right.Collation { - goto cannotCoerce - } - } - - if leftCS.Name() == rightCS.Name() { - switch { - case left.Coercibility < right.Coercibility: - left.Repertoire |= right.Repertoire - return left, nil, nil, nil - - case left.Coercibility > right.Coercibility: - right.Repertoire |= left.Repertoire - return right, nil, nil, nil - - case left.Collation == right.Collation: - left.Repertoire |= right.Repertoire - return left, nil, nil, nil - } - - if left.Coercibility == CoerceExplicit { - goto cannotCoerce - } - - leftCsBin := leftColl.IsBinary() - rightCsBin := rightColl.IsBinary() - - switch { - case leftCsBin && rightCsBin: - left.Coercibility = CoerceNone - return left, nil, nil, nil - - case leftCsBin: - return left, nil, nil, nil - - case rightCsBin: - return right, nil, nil, nil - } - - defaults := env.byCharset[leftCS.Name()] - return TypedCollation{ - Collation: defaults.Binary.ID(), - Coercibility: CoerceNone, - Repertoire: left.Repertoire | right.Repertoire, - }, nil, nil, nil - } - - if _, leftIsBinary := leftColl.(*Collation_binary); leftIsBinary { - if left.Coercibility <= right.Coercibility { - return left, nil, nil, nil - } - goto coerceToRight - } - if _, rightIsBinary := rightColl.(*Collation_binary); rightIsBinary { - if left.Coercibility >= right.Coercibility { - return right, nil, nil, nil - } - goto coerceToLeft - } - - if opt.ConvertToSuperset { - if checkCompatibleCollations(leftColl, left.Coercibility, left.Repertoire, rightColl, right.Coercibility, right.Repertoire) { - goto coerceToLeft - } - if checkCompatibleCollations(rightColl, right.Coercibility, right.Repertoire, leftColl, left.Coercibility, left.Repertoire) { - goto coerceToRight - } - } - - if opt.ConvertWithCoercion { - if left.Coercibility < right.Coercibility && right.Coercibility > CoerceImplicit { - goto coerceToLeft - } - if right.Coercibility < left.Coercibility && left.Coercibility > CoerceImplicit { - goto coerceToRight - } - } - -cannotCoerce: - return TypedCollation{}, nil, nil, fmt.Errorf("Illegal mix of collations (%s,%s) and (%s,%s)", - leftColl.Name(), left.Coercibility, rightColl.Name(), right.Coercibility) - -coerceToLeft: - return left, nil, - func(dst, in []byte) ([]byte, error) { - return charset.Convert(dst, leftCS, in, rightCS) - }, nil - -coerceToRight: - return right, - func(dst, in []byte) ([]byte, error) { - return charset.Convert(dst, rightCS, in, leftCS) - }, nil, nil -} - func (env *Environment) EnsureCollate(fromID, toID ID) error { // these two lookups should never fail - from := fromID.Get() - to := toID.Get() - if from.Charset().Name() != to.Charset().Name() { - return fmt.Errorf("COLLATION '%s' is not valid for CHARACTER SET '%s'", to.Name(), from.Charset().Name()) + fromCharsetName := env.LookupCharsetName(fromID) + toCharsetName := env.LookupCharsetName(toID) + if fromCharsetName != toCharsetName { + toCollName := env.LookupName(toID) + return fmt.Errorf("COLLATION '%s' is not valid for CHARACTER SET '%s'", toCollName, fromCharsetName) } return nil } diff --git a/go/mysql/collations/collation.go b/go/mysql/collations/collation.go index 1cfb5bb4671..aebc4dc9646 100644 --- a/go/mysql/collations/collation.go +++ b/go/mysql/collations/collation.go @@ -16,160 +16,10 @@ limitations under the License. package collations -import ( - "math" - - "vitess.io/vitess/go/mysql/collations/charset" - "vitess.io/vitess/go/vt/vthash" -) - //go:generate go run ./tools/makecolldata/ --embed=true -// CaseAwareCollation implements lowercase and uppercase conventions for collations. -type CaseAwareCollation interface { - Collation - ToUpper(dst []byte, src []byte) []byte - ToLower(dst []byte, src []byte) []byte -} - // ID is a numeric identifier for a collation. These identifiers are defined by MySQL, not by Vitess. type ID uint16 -// Get returns the Collation identified by this ID. If the ID is invalid, this returns nil -func (i ID) Get() Collation { - if int(i) < len(collationsById) { - return collationsById[i] - } - return nil -} - -// Valid returns whether this Collation ID is valid (i.e. identifies a valid collation) -func (i ID) Valid() bool { - return int(i) < len(collationsById) && collationsById[i] != nil -} - // Unknown is the default ID for an unknown collation. const Unknown ID = 0 - -// Collation implements a MySQL-compatible collation. It defines how to compare -// for sorting order and equality two strings with the same encoding. -type Collation interface { - // ID returns the numerical identifier for this collation. This is the same - // value that is returned by MySQL in a query's headers to identify the collation - // for a given column - ID() ID - - // Name is the full name of this collation, in the form of "ENCODING_LANG_SENSITIVITY" - Name() string - - // Collate compares two strings using this collation. `left` and `right` must be the - // two strings encoded in the proper encoding for this collation. If `isPrefix` is true, - // the function instead behaves equivalently to `strings.HasPrefix(left, right)`, but - // being collation-aware. - // It returns a numeric value like a normal comparison function: <0 if left < right, - // 0 if left == right, >0 if left > right - Collate(left, right []byte, isPrefix bool) int - - // WeightString returns a weight string for the given `src` string. A weight string - // is a binary representation of the weights for the given string, that can be - // compared byte-wise to return identical results to collating this string. - // - // This means: - // bytes.Compare(WeightString(left), WeightString(right)) == Collate(left, right) - // - // The semantics of this API have been carefully designed to match MySQL's behavior - // in its `strnxfrm` API. Most notably, the `numCodepoints` argument implies different - // behaviors depending on the collation's padding mode: - // - // - For collations that pad WITH SPACE (this is, all legacy collations in MySQL except - // for the newly introduced UCA v9.0.0 utf8mb4 collations in MySQL 8.0), `numCodepoints` - // can have the following values: - // - // - if `numCodepoints` is any integer greater than zero, this treats the `src` string - // as if it were in a `CHAR(numCodepoints)` column in MySQL, meaning that the resulting - // weight string will be padded with the weight for the SPACE character until it becomes - // wide enough to fill the `CHAR` column. This is necessary to perform weight comparisons - // in fixed-`CHAR` columns. If `numCodepoints` is smaller than the actual amount of - // codepoints stored in `src`, the result is unspecified. - // - // - if `numCodepoints` is zero, this is equivalent to `numCodepoints = RuneCount(src)`, - // meaning that the resulting weight string will have no padding at the end: it'll only have - // the weight values for the exact amount of codepoints contained in `src`. This is the - // behavior required to sort `VARCHAR` columns. - // - // - if `numCodepoints` is the special constant PadToMax, then the `dst` slice must be - // pre-allocated to a zero-length slice with enough capacity to hold the complete weight - // string, and any remaining capacity in `dst` will be filled by the weights for the - // padding character, repeatedly. This is a special flag used by MySQL when performing - // filesorts, where all the sorting keys must have identical sizes, even for `VARCHAR` - // columns. - // - // - For collations that have NO PAD (this is, the newly introduced UCA v9.0.0 utf8mb4 collations - // in MySQL 8.0), `numCodepoints` can only have the special constant `PadToMax`, which will make - // the weight string padding equivalent to a PAD SPACE collation (as explained in the previous - // section). All other values for `numCodepoints` are ignored, because NO PAD collations always - // return the weights for the codepoints in their strings, with no further padding at the end. - // - // The resulting weight string is written to `dst`, which can be pre-allocated to - // WeightStringLen() bytes to prevent growing the slice. `dst` can also be nil, in which - // case it will grow dynamically. If `numCodepoints` has the special PadToMax value explained - // earlier, `dst` MUST be pre-allocated to the target size or the function will return an - // empty slice. - WeightString(dst, src []byte, numCodepoints int) []byte - - // WeightStringLen returns a size (in bytes) that would fit any weight strings for a string - // with `numCodepoints` using this collation. Note that this is a higher bound for the size - // of the string, and in practice weight strings can be significantly smaller than the - // returned value. - WeightStringLen(numCodepoints int) int - - // Hash returns a 32 or 64 bit identifier (depending on the platform) that uniquely identifies - // the given string based on this collation. It is functionally equivalent to calling WeightString - // and then hashing the result. - // - // Consequently, if the hashes for two strings are different, then the two strings are considered - // different according to this collation. If the hashes for two strings are equal, the two strings - // may or may not be considered equal according to this collation, because hashes can collide unlike - // weight strings. - // - // The numCodepoints argument has the same behavior as in WeightString: if this collation uses PAD SPACE, - // the hash will interpret the source string as if it were stored in a `CHAR(n)` column. If the value of - // numCodepoints is 0, this is equivalent to setting `numCodepoints = RuneCount(src)`. - // For collations with NO PAD, the numCodepoint argument is ignored. - Hash(hasher *vthash.Hasher, src []byte, numCodepoints int) - - // Wildcard returns a matcher for the given wildcard pattern. The matcher can be used to repeatedly - // test different strings to check if they match the pattern. The pattern must be a traditional wildcard - // pattern, which may contain the provided special characters for matching one character or several characters. - // The provided `escape` character will be used as an escape sequence in front of the other special characters. - // - // This method is fully collation aware; the matching will be performed according to the underlying collation. - // I.e. if this is a case-insensitive collation, matching will be case-insensitive. - // - // The returned WildcardPattern is always valid, but if the provided special characters do not exist in this - // collation's repertoire, the returned pattern will not match any strings. Likewise, if the provided pattern - // has invalid syntax, the returned pattern will not match any strings. - // - // If the provided special characters are 0, the defaults to parse an SQL 'LIKE' statement will be used. - // This is, '_' for matching one character, '%' for matching many and '\\' for escape. - // - // This method can also be used for Shell-like matching with '?', '*' and '\\' as their respective special - // characters. - Wildcard(pat []byte, matchOne, matchMany, escape rune) WildcardPattern - - // Charset returns the Charset with which this collation is encoded - Charset() Charset - - // IsBinary returns whether this collation is a binary collation - IsBinary() bool -} - -// WildcardPattern is a matcher for a wildcard pattern, constructed from a given collation -type WildcardPattern interface { - // Match returns whether the given string matches this pattern - Match(in []byte) bool -} - -type Charset = charset.Charset - -const PadToMax = math.MaxInt32 diff --git a/go/mysql/collations/8bit.go b/go/mysql/collations/colldata/8bit.go similarity index 96% rename from go/mysql/collations/8bit.go rename to go/mysql/collations/colldata/8bit.go index b587fab6d6c..2355888bbab 100644 --- a/go/mysql/collations/8bit.go +++ b/go/mysql/collations/colldata/8bit.go @@ -14,9 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/vt/vthash" ) @@ -42,7 +43,7 @@ type simpletables struct { } type Collation_8bit_bin struct { - id ID + id collations.ID name string simpletables charset charset.Charset @@ -52,7 +53,7 @@ func (c *Collation_8bit_bin) Name() string { return c.name } -func (c *Collation_8bit_bin) ID() ID { +func (c *Collation_8bit_bin) ID() collations.ID { return c.id } @@ -129,7 +130,7 @@ func (c *Collation_8bit_bin) ToUpper(dst, src []byte) []byte { } type Collation_8bit_simple_ci struct { - id ID + id collations.ID name string simpletables charset charset.Charset @@ -139,7 +140,7 @@ func (c *Collation_8bit_simple_ci) Name() string { return c.name } -func (c *Collation_8bit_simple_ci) ID() ID { +func (c *Collation_8bit_simple_ci) ID() collations.ID { return c.id } @@ -251,8 +252,8 @@ func (c *Collation_8bit_simple_ci) ToUpper(dst, src []byte) []byte { type Collation_binary struct{} -func (c *Collation_binary) ID() ID { - return CollationBinaryID +func (c *Collation_binary) ID() collations.ID { + return collations.CollationBinaryID } func (c *Collation_binary) Name() string { diff --git a/go/mysql/collations/cached_size.go b/go/mysql/collations/colldata/cached_size.go similarity index 98% rename from go/mysql/collations/cached_size.go rename to go/mysql/collations/colldata/cached_size.go index 6b5e901dffd..36167c69d6d 100644 --- a/go/mysql/collations/cached_size.go +++ b/go/mysql/collations/colldata/cached_size.go @@ -15,7 +15,7 @@ limitations under the License. */ // Code generated by Sizegen. DO NOT EDIT. -package collations +package colldata import hack "vitess.io/vitess/go/hack" diff --git a/go/mysql/collations/colldata/collation.go b/go/mysql/collations/colldata/collation.go new file mode 100644 index 00000000000..ec66fc09b58 --- /dev/null +++ b/go/mysql/collations/colldata/collation.go @@ -0,0 +1,374 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package colldata + +import ( + "fmt" + "math" + + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/vt/vthash" +) + +type Charset = charset.Charset + +// Collation implements a MySQL-compatible collation. It defines how to compare +// for sorting order and equality two strings with the same encoding. +type Collation interface { + // ID returns the numerical identifier for this collation. This is the same + // value that is returned by MySQL in a query's headers to identify the collation + // for a given column + ID() collations.ID + + // Name is the full name of this collation, in the form of "ENCODING_LANG_SENSITIVITY" + Name() string + + // Collate compares two strings using this collation. `left` and `right` must be the + // two strings encoded in the proper encoding for this collation. If `isPrefix` is true, + // the function instead behaves equivalently to `strings.HasPrefix(left, right)`, but + // being collation-aware. + // It returns a numeric value like a normal comparison function: <0 if left < right, + // 0 if left == right, >0 if left > right + Collate(left, right []byte, isPrefix bool) int + + // WeightString returns a weight string for the given `src` string. A weight string + // is a binary representation of the weights for the given string, that can be + // compared byte-wise to return identical results to collating this string. + // + // This means: + // bytes.Compare(WeightString(left), WeightString(right)) == Collate(left, right) + // + // The semantics of this API have been carefully designed to match MySQL's behavior + // in its `strnxfrm` API. Most notably, the `numCodepoints` argument implies different + // behaviors depending on the collation's padding mode: + // + // - For collations that pad WITH SPACE (this is, all legacy collations in MySQL except + // for the newly introduced UCA v9.0.0 utf8mb4 collations in MySQL 8.0), `numCodepoints` + // can have the following values: + // + // - if `numCodepoints` is any integer greater than zero, this treats the `src` string + // as if it were in a `CHAR(numCodepoints)` column in MySQL, meaning that the resulting + // weight string will be padded with the weight for the SPACE character until it becomes + // wide enough to fill the `CHAR` column. This is necessary to perform weight comparisons + // in fixed-`CHAR` columns. If `numCodepoints` is smaller than the actual amount of + // codepoints stored in `src`, the result is unspecified. + // + // - if `numCodepoints` is zero, this is equivalent to `numCodepoints = RuneCount(src)`, + // meaning that the resulting weight string will have no padding at the end: it'll only have + // the weight values for the exact amount of codepoints contained in `src`. This is the + // behavior required to sort `VARCHAR` columns. + // + // - if `numCodepoints` is the special constant PadToMax, then the `dst` slice must be + // pre-allocated to a zero-length slice with enough capacity to hold the complete weight + // string, and any remaining capacity in `dst` will be filled by the weights for the + // padding character, repeatedly. This is a special flag used by MySQL when performing + // filesorts, where all the sorting keys must have identical sizes, even for `VARCHAR` + // columns. + // + // - For collations that have NO PAD (this is, the newly introduced UCA v9.0.0 utf8mb4 collations + // in MySQL 8.0), `numCodepoints` can only have the special constant `PadToMax`, which will make + // the weight string padding equivalent to a PAD SPACE collation (as explained in the previous + // section). All other values for `numCodepoints` are ignored, because NO PAD collations always + // return the weights for the codepoints in their strings, with no further padding at the end. + // + // The resulting weight string is written to `dst`, which can be pre-allocated to + // WeightStringLen() bytes to prevent growing the slice. `dst` can also be nil, in which + // case it will grow dynamically. If `numCodepoints` has the special PadToMax value explained + // earlier, `dst` MUST be pre-allocated to the target size or the function will return an + // empty slice. + WeightString(dst, src []byte, numCodepoints int) []byte + + // WeightStringLen returns a size (in bytes) that would fit any weight strings for a string + // with `numCodepoints` using this collation. Note that this is a higher bound for the size + // of the string, and in practice weight strings can be significantly smaller than the + // returned value. + WeightStringLen(numCodepoints int) int + + // Hash returns a 32 or 64 bit identifier (depending on the platform) that uniquely identifies + // the given string based on this collation. It is functionally equivalent to calling WeightString + // and then hashing the result. + // + // Consequently, if the hashes for two strings are different, then the two strings are considered + // different according to this collation. If the hashes for two strings are equal, the two strings + // may or may not be considered equal according to this collation, because hashes can collide unlike + // weight strings. + // + // The numCodepoints argument has the same behavior as in WeightString: if this collation uses PAD SPACE, + // the hash will interpret the source string as if it were stored in a `CHAR(n)` column. If the value of + // numCodepoints is 0, this is equivalent to setting `numCodepoints = RuneCount(src)`. + // For collations with NO PAD, the numCodepoint argument is ignored. + Hash(hasher *vthash.Hasher, src []byte, numCodepoints int) + + // Wildcard returns a matcher for the given wildcard pattern. The matcher can be used to repeatedly + // test different strings to check if they match the pattern. The pattern must be a traditional wildcard + // pattern, which may contain the provided special characters for matching one character or several characters. + // The provided `escape` character will be used as an escape sequence in front of the other special characters. + // + // This method is fully collation aware; the matching will be performed according to the underlying collation. + // I.e. if this is a case-insensitive collation, matching will be case-insensitive. + // + // The returned WildcardPattern is always valid, but if the provided special characters do not exist in this + // collation's repertoire, the returned pattern will not match any strings. Likewise, if the provided pattern + // has invalid syntax, the returned pattern will not match any strings. + // + // If the provided special characters are 0, the defaults to parse an SQL 'LIKE' statement will be used. + // This is, '_' for matching one character, '%' for matching many and '\\' for escape. + // + // This method can also be used for Shell-like matching with '?', '*' and '\\' as their respective special + // characters. + Wildcard(pat []byte, matchOne, matchMany, escape rune) WildcardPattern + + // Charset returns the Charset with which this collation is encoded + Charset() Charset + + // IsBinary returns whether this collation is a binary collation + IsBinary() bool +} + +// WildcardPattern is a matcher for a wildcard pattern, constructed from a given collation +type WildcardPattern interface { + // Match returns whether the given string matches this pattern + Match(in []byte) bool +} + +const PadToMax = math.MaxInt32 + +// CaseAwareCollation implements lowercase and uppercase conventions for collations. +type CaseAwareCollation interface { + Collation + ToUpper(dst []byte, src []byte) []byte + ToLower(dst []byte, src []byte) []byte +} + +func Lookup(id collations.ID) Collation { + if int(id) >= len(collationsById) { + return nil + } + return collationsById[id] +} + +// All returns a slice with all known collations in Vitess. +func All(env *collations.Environment) []Collation { + allCols := env.AllCollationIDs() + all := make([]Collation, 0, len(allCols)) + for _, col := range allCols { + all = append(all, collationsById[col]) + } + return all +} + +func checkCompatibleCollations( + left Collation, leftCoercibility collations.Coercibility, leftRepertoire collations.Repertoire, + right Collation, rightCoercibility collations.Coercibility, rightRepertoire collations.Repertoire, +) bool { + leftCS := left.Charset() + rightCS := right.Charset() + + switch leftCS.(type) { + case charset.Charset_utf8mb4: + if leftCoercibility <= rightCoercibility { + return true + } + + case charset.Charset_utf32: + switch { + case leftCoercibility < rightCoercibility: + return true + case leftCoercibility == rightCoercibility: + if !charset.IsUnicode(rightCS) { + return true + } + if !left.IsBinary() { + return true + } + } + + case charset.Charset_utf8mb3, charset.Charset_ucs2, charset.Charset_utf16, charset.Charset_utf16le: + switch { + case leftCoercibility < rightCoercibility: + return true + case leftCoercibility == rightCoercibility: + if !charset.IsUnicode(rightCS) { + return true + } + } + } + + if rightRepertoire == collations.RepertoireASCII { + switch { + case leftCoercibility < rightCoercibility: + return true + case leftCoercibility == rightCoercibility: + if leftRepertoire == collations.RepertoireUnicode { + return true + } + } + } + + return false +} + +// CoercionOptions is used to configure how aggressive the algorithm can be +// when merging two different collations by transcoding them. +type CoercionOptions struct { + // ConvertToSuperset allows merging two different collations as long + // as the charset of one of them is a strict superset of the other. In + // order to operate on the two expressions, one of them will need to + // be transcoded. This transcoding will always be safe because the string + // with the smallest repertoire will be transcoded to its superset, which + // cannot fail. + ConvertToSuperset bool + + // ConvertWithCoercion allows merging two different collations by forcing + // a coercion as long as the coercibility of the two sides is lax enough. + // This will force a transcoding of one of the expressions even if their + // respective charsets are not a strict superset, so the resulting transcoding + // CAN fail depending on the content of their strings. + ConvertWithCoercion bool +} + +// Coercion is a function that will transform either the given argument +// arguments of the function into a specific character set. The `dst` argument +// will be used as the destination of the coerced argument, but it can be nil. +type Coercion func(dst, in []byte) ([]byte, error) + +// Merge returns a Coercion function for a pair of TypedCollation based +// on their coercibility. +// +// The function takes the typed collations for the two sides of a text operation +// (namely, a comparison or concatenation of two textual expressions). These typed +// collations includes the actual collation for the expression on each size, their +// coercibility values (see: Coercibility) and their respective repertoires, +// and returns the target collation (i.e. the collation into which the two expressions +// must be coerced, and a Coercion function. The Coercion function can be called repeatedly +// with the different values for the two expressions and will transcode either +// the left-hand or right-hand value to the appropriate charset so it can be +// collated against the other value. +// +// If the collations for both sides of the expressions are the same, the returned +// Coercion function will be a no-op. Likewise, if the two collations are not the same, +// but they are compatible and have the same charset, the Coercion function will also +// be a no-op. +// +// If the collations for both sides of the expression are not compatible, an error +// will be returned and the returned TypedCollation and Coercion will be nil. +func Merge(env *collations.Environment, left, right collations.TypedCollation, opt CoercionOptions) (collations.TypedCollation, Coercion, Coercion, error) { + leftColl := Lookup(left.Collation) + rightColl := Lookup(right.Collation) + if leftColl == nil || rightColl == nil { + return collations.TypedCollation{}, nil, nil, fmt.Errorf("unsupported TypeCollationID: %v / %v", left.Collation, right.Collation) + } + + leftCS := leftColl.Charset() + rightCS := rightColl.Charset() + + if left.Coercibility == collations.CoerceExplicit && right.Coercibility == collations.CoerceExplicit { + if left.Collation != right.Collation { + goto cannotCoerce + } + } + + if leftCS.Name() == rightCS.Name() { + switch { + case left.Coercibility < right.Coercibility: + left.Repertoire |= right.Repertoire + return left, nil, nil, nil + + case left.Coercibility > right.Coercibility: + right.Repertoire |= left.Repertoire + return right, nil, nil, nil + + case left.Collation == right.Collation: + left.Repertoire |= right.Repertoire + return left, nil, nil, nil + } + + if left.Coercibility == collations.CoerceExplicit { + goto cannotCoerce + } + + leftCsBin := leftColl.IsBinary() + rightCsBin := rightColl.IsBinary() + + switch { + case leftCsBin && rightCsBin: + left.Coercibility = collations.CoerceNone + return left, nil, nil, nil + + case leftCsBin: + return left, nil, nil, nil + + case rightCsBin: + return right, nil, nil, nil + } + + defaults := env.LookupByCharset(leftCS.Name()) + return collations.TypedCollation{ + Collation: defaults.Binary, + Coercibility: collations.CoerceNone, + Repertoire: left.Repertoire | right.Repertoire, + }, nil, nil, nil + } + + if _, leftIsBinary := leftColl.(*Collation_binary); leftIsBinary { + if left.Coercibility <= right.Coercibility { + return left, nil, nil, nil + } + goto coerceToRight + } + if _, rightIsBinary := rightColl.(*Collation_binary); rightIsBinary { + if left.Coercibility >= right.Coercibility { + return right, nil, nil, nil + } + goto coerceToLeft + } + + if opt.ConvertToSuperset { + if checkCompatibleCollations(leftColl, left.Coercibility, left.Repertoire, rightColl, right.Coercibility, right.Repertoire) { + goto coerceToLeft + } + if checkCompatibleCollations(rightColl, right.Coercibility, right.Repertoire, leftColl, left.Coercibility, left.Repertoire) { + goto coerceToRight + } + } + + if opt.ConvertWithCoercion { + if left.Coercibility < right.Coercibility && right.Coercibility > collations.CoerceImplicit { + goto coerceToLeft + } + if right.Coercibility < left.Coercibility && left.Coercibility > collations.CoerceImplicit { + goto coerceToRight + } + } + +cannotCoerce: + return collations.TypedCollation{}, nil, nil, fmt.Errorf("Illegal mix of collations (%s,%s) and (%s,%s)", + leftColl.Name(), left.Coercibility, rightColl.Name(), right.Coercibility) + +coerceToLeft: + return left, nil, + func(dst, in []byte) ([]byte, error) { + return charset.Convert(dst, leftCS, in, rightCS) + }, nil + +coerceToRight: + return right, + func(dst, in []byte) ([]byte, error) { + return charset.Convert(dst, rightCS, in, leftCS) + }, nil, nil +} diff --git a/go/mysql/collations/fuzz.go b/go/mysql/collations/colldata/fuzz.go similarity index 98% rename from go/mysql/collations/fuzz.go rename to go/mysql/collations/colldata/fuzz.go index e71eae3fbdc..c5ebf50698b 100644 --- a/go/mysql/collations/fuzz.go +++ b/go/mysql/collations/colldata/fuzz.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( fuzz "github.com/AdaLogics/go-fuzz-headers" diff --git a/go/mysql/collations/fuzz_test.go b/go/mysql/collations/colldata/fuzz_test.go similarity index 96% rename from go/mysql/collations/fuzz_test.go rename to go/mysql/collations/colldata/fuzz_test.go index 1f36fd34ff3..0c11116f580 100644 --- a/go/mysql/collations/fuzz_test.go +++ b/go/mysql/collations/colldata/fuzz_test.go @@ -18,9 +18,11 @@ limitations under the License. // The fuzzing tests for collations use the new Fuzz implementation in Go 1.18+ -package collations +package colldata -import "testing" +import ( + "testing" +) func FuzzUCACollate(f *testing.F) { for _, left := range AllTestStrings { diff --git a/go/mysql/collations/colldata/golden_test.go b/go/mysql/collations/colldata/golden_test.go new file mode 100644 index 00000000000..2b41ebcddc6 --- /dev/null +++ b/go/mysql/collations/colldata/golden_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package colldata + +import ( + "bytes" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/internal/testutil" +) + +func TestGoldenWeights(t *testing.T) { + gllGoldenTests, err := filepath.Glob("testdata/wiki_*.gob.gz") + if err != nil { + t.Fatal(err) + } + + for _, goldenPath := range gllGoldenTests { + golden := &testutil.GoldenTest{} + if err := golden.DecodeFromFile(goldenPath); err != nil { + t.Fatal(err) + } + + for _, goldenCase := range golden.Cases { + t.Run(fmt.Sprintf("%s (%s)", golden.Name, goldenCase.Lang), func(t *testing.T) { + for coll, expected := range goldenCase.Weights { + coll := testcollation(t, coll) + + input, err := charset.ConvertFromUTF8(nil, coll.Charset(), goldenCase.Text) + if err != nil { + t.Fatal(err) + } + + result := coll.WeightString(nil, input, 0) + assert.True(t, bytes.Equal(expected, result), "mismatch for collation=%s\noriginal: %s\ninput: %#v\nexpected: %v\nactual: %v", coll.Name(), string(goldenCase.Text), input, expected, result) + + } + }) + } + } +} + +func TestCollationsForLanguage(t *testing.T) { + allCollations := testall() + langCounts := make(map[testutil.Lang][]string) + + for lang := range testutil.KnownLanguages { + var matched []string + for _, coll := range allCollations { + name := coll.Name() + if lang.MatchesCollation(name) { + matched = append(matched, name) + } + } + langCounts[lang] = matched + } + + for lang := range testutil.KnownLanguages { + assert.NotEqual(t, 0, len(langCounts[lang]), "no collations found for %q", lang) + + t.Logf("%s: %v", lang, langCounts[lang]) + } +} diff --git a/go/mysql/collations/multibyte.go b/go/mysql/collations/colldata/multibyte.go similarity index 96% rename from go/mysql/collations/multibyte.go rename to go/mysql/collations/colldata/multibyte.go index fcbcb97dc4c..cc123a25a1a 100644 --- a/go/mysql/collations/multibyte.go +++ b/go/mysql/collations/colldata/multibyte.go @@ -14,23 +14,24 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "math" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/vt/vthash" ) type Collation_multibyte struct { - id ID + id collations.ID name string sort *[256]byte charset charset.Charset } -func (c *Collation_multibyte) ID() ID { +func (c *Collation_multibyte) ID() collations.ID { return c.id } diff --git a/go/mysql/collations/mysqldata.go b/go/mysql/collations/colldata/mysqldata.go similarity index 99% rename from go/mysql/collations/mysqldata.go rename to go/mysql/collations/colldata/mysqldata.go index 0b3d10372d0..f626028cb95 100644 --- a/go/mysql/collations/mysqldata.go +++ b/go/mysql/collations/colldata/mysqldata.go @@ -1,6 +1,22 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by makecolldata DO NOT EDIT -package collations +package colldata import ( charset "vitess.io/vitess/go/mysql/collations/charset" diff --git a/go/mysql/collations/mysqlucadata.bin b/go/mysql/collations/colldata/mysqlucadata.bin similarity index 100% rename from go/mysql/collations/mysqlucadata.bin rename to go/mysql/collations/colldata/mysqlucadata.bin diff --git a/go/mysql/collations/mysqlucadata.go b/go/mysql/collations/colldata/mysqlucadata.go similarity index 99% rename from go/mysql/collations/mysqlucadata.go rename to go/mysql/collations/colldata/mysqlucadata.go index ae8e2d48642..b047de17d2a 100644 --- a/go/mysql/collations/mysqlucadata.go +++ b/go/mysql/collations/colldata/mysqlucadata.go @@ -1,6 +1,22 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by makecolldata DO NOT EDIT -package collations +package colldata import ( _ "embed" diff --git a/go/mysql/collations/uca.go b/go/mysql/collations/colldata/uca.go similarity index 96% rename from go/mysql/collations/uca.go rename to go/mysql/collations/colldata/uca.go index 444fd3c295c..4b7272bfbc3 100644 --- a/go/mysql/collations/uca.go +++ b/go/mysql/collations/colldata/uca.go @@ -14,12 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "bytes" "math/bits" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/mysql/collations/internal/uca" "vitess.io/vitess/go/vt/vthash" @@ -27,7 +28,7 @@ import ( type Collation_utf8mb4_uca_0900 struct { name string - id ID + id collations.ID uca *uca.Collation900 } @@ -35,7 +36,7 @@ func (c *Collation_utf8mb4_uca_0900) Name() string { return c.name } -func (c *Collation_utf8mb4_uca_0900) ID() ID { +func (c *Collation_utf8mb4_uca_0900) ID() collations.ID { return c.id } @@ -213,7 +214,7 @@ func (c *Collation_utf8mb4_uca_0900) ToUpper(dst, src []byte) []byte { type Collation_utf8mb4_0900_bin struct{} -func (c *Collation_utf8mb4_0900_bin) ID() ID { +func (c *Collation_utf8mb4_0900_bin) ID() collations.ID { return 309 } @@ -271,11 +272,11 @@ func (c *Collation_utf8mb4_0900_bin) ToUpper(dst, src []byte) []byte { type Collation_uca_legacy struct { name string - id ID + id collations.ID uca *uca.CollationLegacy } -func (c *Collation_uca_legacy) ID() ID { +func (c *Collation_uca_legacy) ID() collations.ID { return c.id } diff --git a/go/mysql/collations/uca_contraction_test.go b/go/mysql/collations/colldata/uca_contraction_test.go similarity index 99% rename from go/mysql/collations/uca_contraction_test.go rename to go/mysql/collations/colldata/uca_contraction_test.go index 7d59b6fa4a8..d17ff21e255 100644 --- a/go/mysql/collations/uca_contraction_test.go +++ b/go/mysql/collations/colldata/uca_contraction_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "encoding/json" diff --git a/go/mysql/collations/uca_tables_test.go b/go/mysql/collations/colldata/uca_tables_test.go similarity index 99% rename from go/mysql/collations/uca_tables_test.go rename to go/mysql/collations/colldata/uca_tables_test.go index 011095e1cf6..8823d8f2731 100644 --- a/go/mysql/collations/uca_tables_test.go +++ b/go/mysql/collations/colldata/uca_tables_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "encoding/json" diff --git a/go/mysql/collations/uca_test.go b/go/mysql/collations/colldata/uca_test.go similarity index 99% rename from go/mysql/collations/uca_test.go rename to go/mysql/collations/colldata/uca_test.go index 1938d5b83d7..70c9312636e 100644 --- a/go/mysql/collations/uca_test.go +++ b/go/mysql/collations/colldata/uca_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "bytes" diff --git a/go/mysql/collations/unicase.go b/go/mysql/collations/colldata/unicase.go similarity index 99% rename from go/mysql/collations/unicase.go rename to go/mysql/collations/colldata/unicase.go index c669c2368ad..964d48d7107 100644 --- a/go/mysql/collations/unicase.go +++ b/go/mysql/collations/colldata/unicase.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "vitess.io/vitess/go/mysql/collations/charset" diff --git a/go/mysql/collations/unicode.go b/go/mysql/collations/colldata/unicode.go similarity index 97% rename from go/mysql/collations/unicode.go rename to go/mysql/collations/colldata/unicode.go index 7aed06cda4f..c0495b0474f 100644 --- a/go/mysql/collations/unicode.go +++ b/go/mysql/collations/colldata/unicode.go @@ -14,25 +14,26 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "bytes" "math" "math/bits" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/vt/vthash" ) type Collation_unicode_general_ci struct { - id ID + id collations.ID name string unicase *UnicaseInfo charset charset.Charset } -func (c *Collation_unicode_general_ci) ID() ID { +func (c *Collation_unicode_general_ci) ID() collations.ID { return c.id } @@ -164,12 +165,12 @@ func (c *Collation_unicode_general_ci) Wildcard(pat []byte, matchOne rune, match } type Collation_unicode_bin struct { - id ID + id collations.ID name string charset charset.Charset } -func (c *Collation_unicode_bin) ID() ID { +func (c *Collation_unicode_bin) ID() collations.ID { return c.id } diff --git a/go/mysql/collations/wildcard.go b/go/mysql/collations/colldata/wildcard.go similarity index 99% rename from go/mysql/collations/wildcard.go rename to go/mysql/collations/colldata/wildcard.go index 5d8fd012375..01f4807b7df 100644 --- a/go/mysql/collations/wildcard.go +++ b/go/mysql/collations/colldata/wildcard.go @@ -38,7 +38,7 @@ limitations under the License. // // Because of this, we intend to enable the recursive algorithm by default. -package collations +package colldata import ( "unicode/utf8" diff --git a/go/mysql/collations/wildcard_test.go b/go/mysql/collations/colldata/wildcard_test.go similarity index 99% rename from go/mysql/collations/wildcard_test.go rename to go/mysql/collations/colldata/wildcard_test.go index dc6a44c644c..fff08f35c22 100644 --- a/go/mysql/collations/wildcard_test.go +++ b/go/mysql/collations/colldata/wildcard_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package collations +package colldata import ( "testing" diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 9144d196665..91fc2a8bd8c 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -18,26 +18,29 @@ package collations import ( "fmt" + "slices" "strings" "sync" ) type colldefaults struct { - Default Collation - Binary Collation + Default ID + Binary ID } // Environment is a collation environment for a MySQL version, which contains // a database of collations and defaults for that specific version. type Environment struct { - version collver - byName map[string]Collation - byCharset map[string]*colldefaults - unsupported map[string]ID + version collver + byName map[string]ID + byCharset map[string]*colldefaults + byCharsetName map[ID]string + unsupported map[string]ID + byID map[ID]string } // LookupByName returns the collation with the given name. -func (env *Environment) LookupByName(name string) Collation { +func (env *Environment) LookupByName(name string) ID { return env.byName[name] } @@ -45,37 +48,34 @@ func (env *Environment) LookupByName(name string) Collation { // the collation is supported by this package. func (env *Environment) LookupID(name string) (ID, bool) { if supported, ok := env.byName[name]; ok { - return supported.ID(), true + return supported, true } - if unsupported, ok := env.unsupported[name]; ok { - return unsupported, false + if unsup, ok := env.unsupported[name]; ok { + return unsup, false } return Unknown, false } +// LookupName returns the collation name for the given ID and whether +// the collation is supported by this package. +func (env *Environment) LookupName(id ID) string { + return env.byID[id] +} + // DefaultCollationForCharset returns the default collation for a charset -func (env *Environment) DefaultCollationForCharset(charset string) Collation { +func (env *Environment) DefaultCollationForCharset(charset string) ID { if defaults, ok := env.byCharset[charset]; ok { return defaults.Default } - return nil + return Unknown } // BinaryCollationForCharset returns the default binary collation for a charset -func (env *Environment) BinaryCollationForCharset(charset string) Collation { +func (env *Environment) BinaryCollationForCharset(charset string) ID { if defaults, ok := env.byCharset[charset]; ok { return defaults.Binary } - return nil -} - -// AllCollations returns a slice with all known collations in Vitess. -func (env *Environment) AllCollations() (all []Collation) { - all = make([]Collation, 0, len(env.byName)) - for _, col := range env.byName { - all = append(all, col) - } - return + return Unknown } var globalEnvironments = make(map[collver]*Environment) @@ -109,7 +109,7 @@ func NewEnvironment(serverVersion string) *Environment { case strings.HasSuffix(serverVersion, "-ripple"): // the ripple binlog server can mask the actual version of mysqld; // assume we have the highest - version = collverMySQL80 + version = collverMySQL8 case strings.Contains(serverVersion, "mariadb"): switch { case strings.Contains(serverVersion, "10.0."): @@ -125,66 +125,62 @@ func NewEnvironment(serverVersion string) *Environment { version = collverMySQL56 case strings.HasPrefix(serverVersion, "5.7."): version = collverMySQL57 - case strings.HasPrefix(serverVersion, "8.0."): - version = collverMySQL80 + case strings.HasPrefix(serverVersion, "8."): + version = collverMySQL8 } return fetchCacheEnvironment(version) } func makeEnv(version collver) *Environment { env := &Environment{ - version: version, - byName: make(map[string]Collation), - byCharset: make(map[string]*colldefaults), - unsupported: make(map[string]ID), + version: version, + byName: make(map[string]ID), + byCharset: make(map[string]*colldefaults), + byCharsetName: make(map[ID]string), + byID: make(map[ID]string), + unsupported: make(map[string]ID), } for collid, vi := range globalVersionInfo { var ournames []string + var ourcharsets []string for _, alias := range vi.alias { if alias.mask&version != 0 { ournames = append(ournames, alias.name) + ourcharsets = append(ourcharsets, alias.charset) } } if len(ournames) == 0 { continue } - var collation Collation - if int(collid) < len(collationsById) { - collation = collationsById[collid] - } - if collation == nil { + if int(collid) >= len(supported) || supported[collid] == "" { for _, name := range ournames { env.unsupported[name] = collid } continue } - for _, name := range ournames { - env.byName[name] = collation - } - - csname := collation.Charset().Name() - if _, ok := env.byCharset[csname]; !ok { - env.byCharset[csname] = &colldefaults{} - } - defaults := env.byCharset[csname] - if vi.isdefault&version != 0 { - defaults.Default = collation - } - if collation.IsBinary() { - if defaults.Binary != nil && defaults.Binary.ID() > collation.ID() { - // If there's more than one binary collation, the one with the - // highest ID (i.e. the newest one) takes precedence. This applies - // to utf8mb4_bin vs utf8mb4_0900_bin - continue + for i, name := range ournames { + cs := ourcharsets[i] + env.byName[name] = collid + env.byID[collid] = name + env.byCharsetName[collid] = cs + defaults := env.byCharset[cs] + if defaults == nil { + defaults = &colldefaults{} + env.byCharset[cs] = defaults + } + if vi.isdefault&version != 0 { + defaults.Default = collid + } + if strings.HasSuffix(name, "_bin") && defaults.Binary < collid { + defaults.Binary = collid } - defaults.Binary = collation } } - for from, to := range version.charsetAliases() { + for from, to := range charsetAliases() { env.byCharset[from] = env.byCharset[to] } @@ -201,9 +197,6 @@ const ( CollationLatin1Swedish = 8 ) -// Binary is the default Binary collation -var Binary = ID(CollationBinaryID).Get() - // SystemCollation is the default collation for the system tables // such as the information schema. This is still utf8mb3 to match // MySQLs behavior. This means that you can't use utf8mb4 in table @@ -219,7 +212,7 @@ var SystemCollation = TypedCollation{ // this mapping will change, so it's important to use this helper so that // Vitess code has a consistent mapping for the active collations environment. func (env *Environment) CharsetAlias(charset string) (alias string, ok bool) { - alias, ok = env.version.charsetAliases()[charset] + alias, ok = charsetAliases()[charset] return } @@ -229,10 +222,10 @@ func (env *Environment) CharsetAlias(charset string) (alias string, ok bool) { // Vitess code has a consistent mapping for the active collations environment. func (env *Environment) CollationAlias(collation string) (string, bool) { col := env.LookupByName(collation) - if col == nil { + if col == Unknown { return collation, false } - allCols, ok := globalVersionInfo[col.ID()] + allCols, ok := globalVersionInfo[col] if !ok { return collation, false } @@ -240,7 +233,7 @@ func (env *Environment) CollationAlias(collation string) (string, bool) { return collation, false } for _, alias := range allCols.alias { - for source, dest := range env.version.charsetAliases() { + for source, dest := range charsetAliases() { if strings.HasPrefix(collation, fmt.Sprintf("%s_", source)) && strings.HasPrefix(alias.name, fmt.Sprintf("%s_", dest)) { return alias.name, true @@ -257,7 +250,7 @@ func (env *Environment) CollationAlias(collation string) (string, bool) { // For older MySQL environments, the default charset is `utf8mb4_general_ci`. func (env *Environment) DefaultConnectionCharset() uint8 { switch env.version { - case collverMySQL80: + case collverMySQL8: return uint8(CollationUtf8mb4ID) default: return 45 @@ -282,12 +275,29 @@ func (env *Environment) ParseConnectionCharset(csname string) (uint8, error) { var collid ID = 0 csname = strings.ToLower(csname) if defaults, ok := env.byCharset[csname]; ok { - collid = defaults.Default.ID() + collid = defaults.Default } else if coll, ok := env.byName[csname]; ok { - collid = coll.ID() + collid = coll } if collid == 0 || collid > 255 { return 0, fmt.Errorf("unsupported connection charset: %q", csname) } return uint8(collid), nil } + +func (env *Environment) AllCollationIDs() []ID { + all := make([]ID, 0, len(env.byID)) + for v := range env.byID { + all = append(all, v) + } + slices.Sort(all) + return all +} + +func (env *Environment) LookupByCharset(name string) *colldefaults { + return env.byCharset[name] +} + +func (env *Environment) LookupCharsetName(coll ID) string { + return env.byCharsetName[coll] +} diff --git a/go/mysql/collations/golden_test.go b/go/mysql/collations/golden_test.go index 32b9e90394f..099f77268b7 100644 --- a/go/mysql/collations/golden_test.go +++ b/go/mysql/collations/golden_test.go @@ -1,5 +1,5 @@ /* -Copyright 2021 The Vitess Authors. +Copyright 2023 The Vitess Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,71 +17,58 @@ limitations under the License. package collations import ( - "bytes" "fmt" "os" - "path/filepath" "sort" "strings" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/mysql/collations/charset" - "vitess.io/vitess/go/mysql/collations/internal/testutil" ) -func TestGoldenWeights(t *testing.T) { - gllGoldenTests, err := filepath.Glob("testdata/wiki_*.gob.gz") - if err != nil { - t.Fatal(err) +func TestAllCollationsByCharset(t *testing.T) { + var defaults1 = map[string][2]string{ + "utf8mb4": {"utf8mb4_general_ci", "utf8mb4_bin"}, } - - for _, goldenPath := range gllGoldenTests { - golden := &testutil.GoldenTest{} - if err := golden.DecodeFromFile(goldenPath); err != nil { - t.Fatal(err) - } - - for _, goldenCase := range golden.Cases { - t.Run(fmt.Sprintf("%s (%s)", golden.Name, goldenCase.Lang), func(t *testing.T) { - for coll, expected := range goldenCase.Weights { - coll := testcollation(t, coll) - - input, err := charset.ConvertFromUTF8(nil, coll.Charset(), goldenCase.Text) - if err != nil { - t.Fatal(err) - } - - result := coll.WeightString(nil, input, 0) - assert.True(t, bytes.Equal(expected, result), "mismatch for collation=%s\noriginal: %s\ninput: %#v\nexpected: %v\nactual: %v", coll.Name(), string(goldenCase.Text), input, expected, result) - - } - }) - } + var defaults2 = map[string][2]string{ + "utf8mb4": {"utf8mb4_0900_ai_ci", "utf8mb4_0900_bin"}, } -} -func TestCollationsForLanguage(t *testing.T) { - allCollations := testall() - langCounts := make(map[testutil.Lang][]string) + for _, tc := range []struct { + version collver + defaults map[string][2]string + }{ + {collverMariaDB100, defaults1}, + {collverMariaDB101, defaults1}, + {collverMariaDB102, defaults1}, + {collverMariaDB103, defaults1}, + {collverMySQL56, defaults1}, + {collverMySQL57, defaults1}, + {collverMySQL8, defaults2}, + } { + t.Run(tc.version.String(), func(t *testing.T) { + env := makeEnv(tc.version) + for csname, cset := range env.byCharset { + switch csname { + case "gb18030": + // this doesn't work yet + continue + } + require.NotNil(t, cset.Default, "charset %s has no default", csname) + require.NotNil(t, cset.Binary, "charset %s has no binary", csname) - for lang := range testutil.KnownLanguages { - var matched []string - for _, coll := range allCollations { - name := coll.Name() - if lang.MatchesCollation(name) { - matched = append(matched, name) } - } - langCounts[lang] = matched - } - for lang := range testutil.KnownLanguages { - assert.NotEqual(t, 0, len(langCounts[lang]), "no collations found for %q", lang) - - t.Logf("%s: %v", lang, langCounts[lang]) + for charset, expected := range tc.defaults { + expectedDefault, expectedBinary := expected[0], expected[1] + if def := env.DefaultCollationForCharset(charset); env.LookupName(def) != expectedDefault { + t.Fatalf("bad default for utf8mb4: %s (expected %s)", env.LookupName(def), expectedDefault) + } + if def := env.BinaryCollationForCharset(charset); env.LookupName(def) != expectedBinary { + t.Fatalf("bad binary for utf8mb4: %s (expected %s)", env.LookupName(def), expectedBinary) + } + } + }) } } @@ -89,7 +76,7 @@ func TestCollationsForLanguage(t *testing.T) { // table with Collation support information for the current build of Vitess. func XTestSupportTables(t *testing.T) { var versions = []collver{ - collverMySQL80, + collverMySQL8, collverMySQL57, collverMySQL56, collverMariaDB103, @@ -120,8 +107,8 @@ func XTestSupportTables(t *testing.T) { fmt.Fprintf(out, " |\n|%s\n", strings.Repeat("---|", len(envs)+2)) for _, id := range all { - coll := collationsById[id] - if coll == nil { + name := envs[0].LookupName(id) + if name == "" { vdata := globalVersionInfo[id] var collnames []string @@ -148,9 +135,9 @@ func XTestSupportTables(t *testing.T) { } } } else { - fmt.Fprintf(out, "| %s | %s", coll.Name(), coll.Charset().Name()) + fmt.Fprintf(out, "| %s | %s", name, envs[0].LookupCharsetName(id)) for _, env := range envs { - _, supported := env.byName[coll.Name()] + _, supported := env.LookupID(name) if supported { fmt.Fprintf(out, " | ✅") } else { @@ -162,49 +149,3 @@ func XTestSupportTables(t *testing.T) { fmt.Fprintf(out, " |\n") } } - -func TestAllCollationsByCharset(t *testing.T) { - var defaults1 = map[string][2]string{ - "utf8mb4": {"utf8mb4_general_ci", "utf8mb4_bin"}, - } - var defaults2 = map[string][2]string{ - "utf8mb4": {"utf8mb4_0900_ai_ci", "utf8mb4_0900_bin"}, - } - - for _, tc := range []struct { - version collver - defaults map[string][2]string - }{ - {collverMariaDB100, defaults1}, - {collverMariaDB101, defaults1}, - {collverMariaDB102, defaults1}, - {collverMariaDB103, defaults1}, - {collverMySQL56, defaults1}, - {collverMySQL57, defaults1}, - {collverMySQL80, defaults2}, - } { - t.Run(tc.version.String(), func(t *testing.T) { - env := makeEnv(tc.version) - for csname, cset := range env.byCharset { - switch csname { - case "gb18030": - // this doesn't work yet - continue - } - require.NotNil(t, cset.Default, "charset %s has no default", csname) - require.NotNil(t, cset.Binary, "charset %s has no binary", csname) - - } - - for charset, expected := range tc.defaults { - expectedDefault, expectedBinary := expected[0], expected[1] - if def := env.DefaultCollationForCharset(charset); def.Name() != expectedDefault { - t.Fatalf("bad default for utf8mb4: %s (expected %s)", def.Name(), expectedDefault) - } - if def := env.BinaryCollationForCharset(charset); def.Name() != expectedBinary { - t.Fatalf("bad binary for utf8mb4: %s (expected %s)", def.Name(), expectedBinary) - } - } - }) - } -} diff --git a/go/mysql/collations/integration/charset_test.go b/go/mysql/collations/integration/charset_test.go index 2705dc29f5d..8a4d12a0e4d 100644 --- a/go/mysql/collations/integration/charset_test.go +++ b/go/mysql/collations/integration/charset_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/mysql/collations/remote" @@ -45,7 +47,7 @@ func TestLocalEncodings(t *testing.T) { for _, tc := range cases { local := collations.Local().LookupByName(tc.collation) remote := remote.NewCollation(conn, tc.collation) - verifyTranscoding(t, local, remote, tc.input) + verifyTranscoding(t, colldata.Lookup(local), remote, tc.input) } } diff --git a/go/mysql/collations/integration/coercion_test.go b/go/mysql/collations/integration/coercion_test.go index 7ad31f78852..dad55bcafad 100644 --- a/go/mysql/collations/integration/coercion_test.go +++ b/go/mysql/collations/integration/coercion_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/remote" "vitess.io/vitess/go/sqltypes" @@ -33,18 +35,18 @@ import ( type TextWithCollation struct { Text []byte - Collation collations.Collation + Collation collations.ID } type RemoteCoercionResult struct { Expr sqltypes.Value - Collation collations.Collation + Collation collations.ID Coercibility collations.Coercibility } type RemoteCoercionTest interface { Expression() string - Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coerce1, coerce2 collations.Coercion) + Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coerce1, coerce2 colldata.Coercion) } type testConcat struct { @@ -52,15 +54,17 @@ type testConcat struct { } func (tc *testConcat) Expression() string { + env := collations.Local() return fmt.Sprintf("CONCAT((_%s X'%x' COLLATE %q), (_%s X'%x' COLLATE %q))", - tc.left.Collation.Charset().Name(), tc.left.Text, tc.left.Collation.Name(), - tc.right.Collation.Charset().Name(), tc.right.Text, tc.right.Collation.Name(), + colldata.Lookup(tc.left.Collation).Charset().Name(), tc.left.Text, env.LookupName(tc.left.Collation), + colldata.Lookup(tc.right.Collation).Charset().Name(), tc.right.Text, env.LookupName(tc.right.Collation), ) } -func (tc *testConcat) Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coercion1, coercion2 collations.Coercion) { - localCollation := local.Collation.Get() - assert.Equal(t, remote.Collation.Name(), localCollation.Name(), "bad collation resolved: local is %s, remote is %s", localCollation.Name(), remote.Collation.Name()) +func (tc *testConcat) Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coercion1, coercion2 colldata.Coercion) { + localCollation := colldata.Lookup(local.Collation) + remoteName := collations.Local().LookupName(remote.Collation) + assert.Equal(t, remoteName, localCollation.Name(), "bad collation resolved: local is %s, remote is %s", localCollation.Name(), remoteName) assert.Equal(t, remote.Coercibility, local.Coercibility, "bad coercibility resolved: local is %d, remote is %d", local.Coercibility, remote.Coercibility) leftText, err := coercion1(nil, tc.left.Text) @@ -81,8 +85,8 @@ func (tc *testConcat) Test(t *testing.T, remote *RemoteCoercionResult, local col rEBytes, err := remote.Expr.ToBytes() require.NoError(t, err) - assert.True(t, bytes.Equal(concat.Bytes(), rEBytes), "failed to concatenate text;\n\tCONCAT(%v COLLATE %s, %v COLLATE %s) = \n\tCONCAT(%v, %v) COLLATE %s = \n\t\t%v\n\n\texpected: %v", tc.left.Text, tc.left.Collation.Name(), - tc.right.Text, tc.right.Collation.Name(), leftText, rightText, localCollation.Name(), + assert.True(t, bytes.Equal(concat.Bytes(), rEBytes), "failed to concatenate text;\n\tCONCAT(%v COLLATE %s, %v COLLATE %s) = \n\tCONCAT(%v, %v) COLLATE %s = \n\t\t%v\n\n\texpected: %v", tc.left.Text, collations.Local().LookupName(tc.left.Collation), + tc.right.Text, collations.Local().LookupName(tc.right.Collation), leftText, rightText, localCollation.Name(), concat.Bytes(), rEBytes) } @@ -92,14 +96,15 @@ type testComparison struct { } func (tc *testComparison) Expression() string { + env := collations.Local() return fmt.Sprintf("(_%s X'%x' COLLATE %q) = (_%s X'%x' COLLATE %q)", - tc.left.Collation.Charset().Name(), tc.left.Text, tc.left.Collation.Name(), - tc.right.Collation.Charset().Name(), tc.right.Text, tc.right.Collation.Name(), + env.LookupCharsetName(tc.left.Collation), tc.left.Text, env.LookupName(tc.left.Collation), + env.LookupCharsetName(tc.right.Collation), tc.right.Text, env.LookupName(tc.right.Collation), ) } -func (tc *testComparison) Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coerce1, coerce2 collations.Coercion) { - localCollation := local.Collation.Get() +func (tc *testComparison) Test(t *testing.T, remote *RemoteCoercionResult, local collations.TypedCollation, coerce1, coerce2 colldata.Coercion) { + localCollation := colldata.Lookup(local.Collation) leftText, err := coerce1(nil, tc.left.Text) if err != nil { t.Errorf("failed to transcode left: %v", err) @@ -130,12 +135,12 @@ func TestComparisonSemantics(t *testing.T) { t.Skipf("The behavior of Coercion Semantics is not correct before 8.0.31") } - for _, coll := range collations.Local().AllCollations() { + for _, coll := range colldata.All(collations.Local()) { text := verifyTranscoding(t, coll, remote.NewCollation(conn, coll.Name()), []byte(BaseString)) - testInputs = append(testInputs, &TextWithCollation{Text: text, Collation: coll}) + testInputs = append(testInputs, &TextWithCollation{Text: text, Collation: coll.ID()}) } sort.Slice(testInputs, func(i, j int) bool { - return testInputs[i].Collation.ID() < testInputs[j].Collation.ID() + return testInputs[i].Collation < testInputs[j].Collation }) var testCases = []struct { @@ -161,17 +166,17 @@ func TestComparisonSemantics(t *testing.T) { for _, collA := range testInputs { for _, collB := range testInputs { left := collations.TypedCollation{ - Collation: collA.Collation.ID(), + Collation: collA.Collation, Coercibility: 0, Repertoire: collations.RepertoireASCII, } right := collations.TypedCollation{ - Collation: collB.Collation.ID(), + Collation: collB.Collation, Coercibility: 0, Repertoire: collations.RepertoireASCII, } - resultLocal, coercionLocal1, coercionLocal2, errLocal := collations.Local().MergeCollations(left, right, - collations.CoercionOptions{ + resultLocal, coercionLocal1, coercionLocal2, errLocal := colldata.Merge(collations.Local(), left, right, + colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -189,11 +194,12 @@ func TestComparisonSemantics(t *testing.T) { query := fmt.Sprintf("SELECT CAST((%s) AS BINARY), COLLATION(%s), COERCIBILITY(%s)", expr, expr, expr) resultRemote, errRemote := conn.ExecuteFetch(query, 1, false) + env := collations.Local() if errRemote != nil { require.True(t, strings.Contains(errRemote.Error(), "Illegal mix of collations"), "query %s failed: %v", query, errRemote) if errLocal == nil { - t.Errorf("expected %s vs %s to fail coercion: %v", collA.Collation.Name(), collB.Collation.Name(), errRemote) + t.Errorf("expected %s vs %s to fail coercion: %v", env.LookupName(collA.Collation), env.LookupName(collB.Collation), errRemote) continue } require.True(t, strings.HasPrefix(normalizeCollationInError(errRemote.Error()), normalizeCollationInError(errLocal.Error())), "bad error message: expected %q, got %q", errRemote, errLocal) @@ -202,7 +208,7 @@ func TestComparisonSemantics(t *testing.T) { } if errLocal != nil { - t.Errorf("expected %s vs %s to coerce, but they failed: %v", collA.Collation.Name(), collB.Collation.Name(), errLocal) + t.Errorf("expected %s vs %s to coerce, but they failed: %v", env.LookupName(collA.Collation), env.LookupName(collB.Collation), errLocal) continue } diff --git a/go/mysql/collations/integration/collations_test.go b/go/mysql/collations/integration/collations_test.go index 32ffb81a498..3b33e23e2d3 100644 --- a/go/mysql/collations/integration/collations_test.go +++ b/go/mysql/collations/integration/collations_test.go @@ -31,6 +31,8 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/text/encoding/unicode/utf32" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/remote" @@ -140,7 +142,7 @@ func (u *uca900CollationTest) Test(t *testing.T, result *sqltypes.Result) { continue } - weightString := coll.WeightString(make([]byte, 0, 128), utf8Input, 0) + weightString := colldata.Lookup(coll).WeightString(make([]byte, 0, 128), utf8Input, 0) if !bytes.Equal(weightString, expectedWeightString) { t.Errorf("[%s] mismatch for %s (%v): \n\twant: %v\n\tgot: %v", u.collation, row[2].ToString(), utf8Input, expectedWeightString, weightString) errors++ @@ -227,7 +229,7 @@ func TestCollationWithSpace(t *testing.T) { remote := remote.NewCollation(conn, collName) for _, size := range []int{0, codepoints, codepoints + 1, codepoints + 2, 20, 32} { - localWeight := local.WeightString(nil, []byte(ExampleString), size) + localWeight := colldata.Lookup(local).WeightString(nil, []byte(ExampleString), size) remoteWeight := remote.WeightString(nil, []byte(ExampleString), size) require.True(t, bytes.Equal(localWeight, remoteWeight), "mismatch at len=%d\ninput: %#v\nexpected: %#v\nactual: %#v", size, []byte(ExampleString), remoteWeight, localWeight) diff --git a/go/mysql/collations/integration/helpers_test.go b/go/mysql/collations/integration/helpers_test.go index 95410fbb74a..d436280f04b 100644 --- a/go/mysql/collations/integration/helpers_test.go +++ b/go/mysql/collations/integration/helpers_test.go @@ -27,6 +27,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" @@ -52,7 +54,7 @@ func testRemoteWeights(t *testing.T, golden io.Writer, cases []testweight) { t.Run(tc.collation, func(t *testing.T) { local := collations.Local().LookupByName(tc.collation) remote := remote.NewCollation(conn, tc.collation) - localResult := local.WeightString(nil, tc.input, 0) + localResult := colldata.Lookup(local).WeightString(nil, tc.input, 0) remoteResult := remote.WeightString(nil, tc.input, 0) if err := remote.LastError(); err != nil { @@ -85,7 +87,7 @@ func testRemoteComparison(t *testing.T, golden io.Writer, cases []testcmp) { t.Run(tc.collation, func(t *testing.T) { local := collations.Local().LookupByName(tc.collation) remote := remote.NewCollation(conn, tc.collation) - localResult := normalizecmp(local.Collate(tc.left, tc.right, false)) + localResult := normalizecmp(colldata.Lookup(local).Collate(tc.left, tc.right, false)) remoteResult := remote.Collate(tc.left, tc.right, false) if err := remote.LastError(); err != nil { @@ -101,7 +103,7 @@ func testRemoteComparison(t *testing.T, golden io.Writer, cases []testcmp) { } } -func verifyTranscoding(t *testing.T, local collations.Collation, remote *remote.Collation, text []byte) []byte { +func verifyTranscoding(t *testing.T, local colldata.Collation, remote *remote.Collation, text []byte) []byte { transRemote, err := charset.ConvertFromUTF8(nil, remote.Charset(), text) require.NoError(t, err, "remote transcoding failed: %v", err) @@ -112,7 +114,7 @@ func verifyTranscoding(t *testing.T, local collations.Collation, remote *remote. return transLocal } -func verifyWeightString(t *testing.T, local collations.Collation, remote *remote.Collation, text []byte) { +func verifyWeightString(t *testing.T, local colldata.Collation, remote *remote.Collation, text []byte) { localResult := local.WeightString(nil, text, 0) remoteResult := remote.WeightString(nil, text, 0) diff --git a/go/mysql/collations/integration/weight_string_test.go b/go/mysql/collations/integration/weight_string_test.go index c93a9ed586e..170da4f5987 100644 --- a/go/mysql/collations/integration/weight_string_test.go +++ b/go/mysql/collations/integration/weight_string_test.go @@ -25,6 +25,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/collations/internal/testutil" "vitess.io/vitess/go/mysql/collations/remote" ) @@ -46,7 +47,7 @@ func TestFastIterators(t *testing.T) { func TestWeightStringsComprehensive(t *testing.T) { type collationsForCharset struct { charset charset.Charset - locals []collations.Collation + locals []colldata.Collation remotes []*remote.Collation } var charsetMap = make(map[string]*collationsForCharset) @@ -59,7 +60,7 @@ func TestWeightStringsComprehensive(t *testing.T) { conn := mysqlconn(t) defer conn.Close() - allCollations := collations.Local().AllCollations() + allCollations := colldata.All(collations.Local()) sort.Slice(allCollations, func(i, j int) bool { return allCollations[i].ID() < allCollations[j].ID() }) @@ -103,16 +104,16 @@ func TestCJKWeightStrings(t *testing.T) { conn := mysqlconn(t) defer conn.Close() - allCollations := collations.Local().AllCollations() + allCollations := colldata.All(collations.Local()) testdata, _ := filepath.Glob("../internal/charset/testdata/*.txt") for _, testfile := range testdata { - charset := filepath.Base(testfile) - charset = strings.TrimSuffix(charset, ".txt") - charset = charset[strings.LastIndexByte(charset, '-')+1:] + cs := filepath.Base(testfile) + cs = strings.TrimSuffix(cs, ".txt") + cs = cs[strings.LastIndexByte(cs, '-')+1:] - var valid []collations.Collation + var valid []colldata.Collation for _, coll := range allCollations { - if coll.Charset().Name() == charset { + if coll.Charset().Name() == cs { valid = append(valid, coll) t.Logf("%s -> %s", testfile, coll.Name()) } diff --git a/go/mysql/collations/integration/wildcard_test.go b/go/mysql/collations/integration/wildcard_test.go index a848e5b7867..6475a35dd21 100644 --- a/go/mysql/collations/integration/wildcard_test.go +++ b/go/mysql/collations/integration/wildcard_test.go @@ -22,6 +22,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/collations/remote" ) @@ -78,7 +79,7 @@ func TestRemoteWildcardMatches(t *testing.T) { {"Ǎḅeçd", "a%bd"}, } - for _, local := range collations.Local().AllCollations() { + for _, local := range colldata.All(collations.Local()) { t.Run(local.Name(), func(t *testing.T) { var remote = remote.NewCollation(conn, local.Name()) var err error diff --git a/go/mysql/collations/internal/uca/fasttables.go b/go/mysql/collations/internal/uca/fasttables.go index 1995a78a664..40f3718babe 100644 --- a/go/mysql/collations/internal/uca/fasttables.go +++ b/go/mysql/collations/internal/uca/fasttables.go @@ -1,3 +1,19 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by makecolldata DO NOT EDIT package uca diff --git a/go/mysql/collations/mysqlversion.go b/go/mysql/collations/mysqlversion.go index 2a1409fbb7e..93d1add9b6a 100644 --- a/go/mysql/collations/mysqlversion.go +++ b/go/mysql/collations/mysqlversion.go @@ -1,11 +1,28 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + // Code generated by makecolldata DO NOT EDIT package collations type collver byte type collalias struct { - mask collver - name string + mask collver + name string + charset string } const ( @@ -16,7 +33,7 @@ const ( collverMariaDB103 collver = 1 << 3 collverMySQL56 collver = 1 << 4 collverMySQL57 collver = 1 << 5 - collverMySQL80 collver = 1 << 6 + collverMySQL8 collver = 1 << 6 ) func (v collver) String() string { @@ -35,405 +52,405 @@ func (v collver) String() string { return "MySQL 5.6" case collverMySQL57: return "MySQL 5.7" - case collverMySQL80: - return "MySQL 8.0" + case collverMySQL8: + return "MySQL 0.8" default: panic("invalid version identifier") } } -func (v collver) charsetAliases() map[string]string { return map[string]string{"utf8": "utf8mb3"} } +func charsetAliases() map[string]string { return map[string]string{"utf8": "utf8mb3"} } var globalVersionInfo = map[ID]struct { alias []collalias isdefault collver }{ - 1: {alias: []collalias{{0b01111111, "big5_chinese_ci"}}, isdefault: 0b01111111}, - 2: {alias: []collalias{{0b01111111, "latin2_czech_cs"}}, isdefault: 0b00000000}, - 3: {alias: []collalias{{0b01111111, "dec8_swedish_ci"}}, isdefault: 0b01111111}, - 4: {alias: []collalias{{0b01111111, "cp850_general_ci"}}, isdefault: 0b01111111}, - 5: {alias: []collalias{{0b01111111, "latin1_german1_ci"}}, isdefault: 0b00000000}, - 6: {alias: []collalias{{0b01111111, "hp8_english_ci"}}, isdefault: 0b01111111}, - 7: {alias: []collalias{{0b01111111, "koi8r_general_ci"}}, isdefault: 0b01111111}, - 8: {alias: []collalias{{0b01111111, "latin1_swedish_ci"}}, isdefault: 0b01111111}, - 9: {alias: []collalias{{0b01111111, "latin2_general_ci"}}, isdefault: 0b01111111}, - 10: {alias: []collalias{{0b01111111, "swe7_swedish_ci"}}, isdefault: 0b01111111}, - 11: {alias: []collalias{{0b01111111, "ascii_general_ci"}}, isdefault: 0b01111111}, - 12: {alias: []collalias{{0b01111111, "ujis_japanese_ci"}}, isdefault: 0b01111111}, - 13: {alias: []collalias{{0b01111111, "sjis_japanese_ci"}}, isdefault: 0b01111111}, - 14: {alias: []collalias{{0b01111111, "cp1251_bulgarian_ci"}}, isdefault: 0b00000000}, - 15: {alias: []collalias{{0b01111111, "latin1_danish_ci"}}, isdefault: 0b00000000}, - 16: {alias: []collalias{{0b01111111, "hebrew_general_ci"}}, isdefault: 0b01111111}, - 18: {alias: []collalias{{0b01111111, "tis620_thai_ci"}}, isdefault: 0b01111111}, - 19: {alias: []collalias{{0b01111111, "euckr_korean_ci"}}, isdefault: 0b01111111}, - 20: {alias: []collalias{{0b01111111, "latin7_estonian_cs"}}, isdefault: 0b00000000}, - 21: {alias: []collalias{{0b01111111, "latin2_hungarian_ci"}}, isdefault: 0b00000000}, - 22: {alias: []collalias{{0b01111111, "koi8u_general_ci"}}, isdefault: 0b01111111}, - 23: {alias: []collalias{{0b01111111, "cp1251_ukrainian_ci"}}, isdefault: 0b00000000}, - 24: {alias: []collalias{{0b01111111, "gb2312_chinese_ci"}}, isdefault: 0b01111111}, - 25: {alias: []collalias{{0b01111111, "greek_general_ci"}}, isdefault: 0b01111111}, - 26: {alias: []collalias{{0b01111111, "cp1250_general_ci"}}, isdefault: 0b01111111}, - 27: {alias: []collalias{{0b01111111, "latin2_croatian_ci"}}, isdefault: 0b00000000}, - 28: {alias: []collalias{{0b01111111, "gbk_chinese_ci"}}, isdefault: 0b01111111}, - 29: {alias: []collalias{{0b01111111, "cp1257_lithuanian_ci"}}, isdefault: 0b00000000}, - 30: {alias: []collalias{{0b01111111, "latin5_turkish_ci"}}, isdefault: 0b01111111}, - 31: {alias: []collalias{{0b01111111, "latin1_german2_ci"}}, isdefault: 0b00000000}, - 32: {alias: []collalias{{0b01111111, "armscii8_general_ci"}}, isdefault: 0b01111111}, - 33: {alias: []collalias{{0b01111111, "utf8_general_ci"}, {0b01111111, "utf8mb3_general_ci"}}, isdefault: 0b01111111}, - 34: {alias: []collalias{{0b01111111, "cp1250_czech_cs"}}, isdefault: 0b00000000}, - 35: {alias: []collalias{{0b01111111, "ucs2_general_ci"}}, isdefault: 0b01111111}, - 36: {alias: []collalias{{0b01111111, "cp866_general_ci"}}, isdefault: 0b01111111}, - 37: {alias: []collalias{{0b01111111, "keybcs2_general_ci"}}, isdefault: 0b01111111}, - 38: {alias: []collalias{{0b01111111, "macce_general_ci"}}, isdefault: 0b01111111}, - 39: {alias: []collalias{{0b01111111, "macroman_general_ci"}}, isdefault: 0b01111111}, - 40: {alias: []collalias{{0b01111111, "cp852_general_ci"}}, isdefault: 0b01111111}, - 41: {alias: []collalias{{0b01111111, "latin7_general_ci"}}, isdefault: 0b01111111}, - 42: {alias: []collalias{{0b01111111, "latin7_general_cs"}}, isdefault: 0b00000000}, - 43: {alias: []collalias{{0b01111111, "macce_bin"}}, isdefault: 0b00000000}, - 44: {alias: []collalias{{0b01111111, "cp1250_croatian_ci"}}, isdefault: 0b00000000}, - 45: {alias: []collalias{{0b01111111, "utf8mb4_general_ci"}}, isdefault: 0b00111111}, - 46: {alias: []collalias{{0b01111111, "utf8mb4_bin"}}, isdefault: 0b00000000}, - 47: {alias: []collalias{{0b01111111, "latin1_bin"}}, isdefault: 0b00000000}, - 48: {alias: []collalias{{0b01111111, "latin1_general_ci"}}, isdefault: 0b00000000}, - 49: {alias: []collalias{{0b01111111, "latin1_general_cs"}}, isdefault: 0b00000000}, - 50: {alias: []collalias{{0b01111111, "cp1251_bin"}}, isdefault: 0b00000000}, - 51: {alias: []collalias{{0b01111111, "cp1251_general_ci"}}, isdefault: 0b01111111}, - 52: {alias: []collalias{{0b01111111, "cp1251_general_cs"}}, isdefault: 0b00000000}, - 53: {alias: []collalias{{0b01111111, "macroman_bin"}}, isdefault: 0b00000000}, - 54: {alias: []collalias{{0b01111111, "utf16_general_ci"}}, isdefault: 0b01111111}, - 55: {alias: []collalias{{0b01111111, "utf16_bin"}}, isdefault: 0b00000000}, - 56: {alias: []collalias{{0b01111111, "utf16le_general_ci"}}, isdefault: 0b01111111}, - 57: {alias: []collalias{{0b01111111, "cp1256_general_ci"}}, isdefault: 0b01111111}, - 58: {alias: []collalias{{0b01111111, "cp1257_bin"}}, isdefault: 0b00000000}, - 59: {alias: []collalias{{0b01111111, "cp1257_general_ci"}}, isdefault: 0b01111111}, - 60: {alias: []collalias{{0b01111111, "utf32_general_ci"}}, isdefault: 0b01111111}, - 61: {alias: []collalias{{0b01111111, "utf32_bin"}}, isdefault: 0b00000000}, - 62: {alias: []collalias{{0b01111111, "utf16le_bin"}}, isdefault: 0b00000000}, - 63: {alias: []collalias{{0b01111111, "binary"}}, isdefault: 0b01111111}, - 64: {alias: []collalias{{0b01111111, "armscii8_bin"}}, isdefault: 0b00000000}, - 65: {alias: []collalias{{0b01111111, "ascii_bin"}}, isdefault: 0b00000000}, - 66: {alias: []collalias{{0b01111111, "cp1250_bin"}}, isdefault: 0b00000000}, - 67: {alias: []collalias{{0b01111111, "cp1256_bin"}}, isdefault: 0b00000000}, - 68: {alias: []collalias{{0b01111111, "cp866_bin"}}, isdefault: 0b00000000}, - 69: {alias: []collalias{{0b01111111, "dec8_bin"}}, isdefault: 0b00000000}, - 70: {alias: []collalias{{0b01111111, "greek_bin"}}, isdefault: 0b00000000}, - 71: {alias: []collalias{{0b01111111, "hebrew_bin"}}, isdefault: 0b00000000}, - 72: {alias: []collalias{{0b01111111, "hp8_bin"}}, isdefault: 0b00000000}, - 73: {alias: []collalias{{0b01111111, "keybcs2_bin"}}, isdefault: 0b00000000}, - 74: {alias: []collalias{{0b01111111, "koi8r_bin"}}, isdefault: 0b00000000}, - 75: {alias: []collalias{{0b01111111, "koi8u_bin"}}, isdefault: 0b00000000}, - 76: {alias: []collalias{{0b01000000, "utf8_tolower_ci"}, {0b01000000, "utf8mb3_tolower_ci"}}, isdefault: 0b00000000}, - 77: {alias: []collalias{{0b01111111, "latin2_bin"}}, isdefault: 0b00000000}, - 78: {alias: []collalias{{0b01111111, "latin5_bin"}}, isdefault: 0b00000000}, - 79: {alias: []collalias{{0b01111111, "latin7_bin"}}, isdefault: 0b00000000}, - 80: {alias: []collalias{{0b01111111, "cp850_bin"}}, isdefault: 0b00000000}, - 81: {alias: []collalias{{0b01111111, "cp852_bin"}}, isdefault: 0b00000000}, - 82: {alias: []collalias{{0b01111111, "swe7_bin"}}, isdefault: 0b00000000}, - 83: {alias: []collalias{{0b01111111, "utf8_bin"}, {0b01111111, "utf8mb3_bin"}}, isdefault: 0b00000000}, - 84: {alias: []collalias{{0b01111111, "big5_bin"}}, isdefault: 0b00000000}, - 85: {alias: []collalias{{0b01111111, "euckr_bin"}}, isdefault: 0b00000000}, - 86: {alias: []collalias{{0b01111111, "gb2312_bin"}}, isdefault: 0b00000000}, - 87: {alias: []collalias{{0b01111111, "gbk_bin"}}, isdefault: 0b00000000}, - 88: {alias: []collalias{{0b01111111, "sjis_bin"}}, isdefault: 0b00000000}, - 89: {alias: []collalias{{0b01111111, "tis620_bin"}}, isdefault: 0b00000000}, - 90: {alias: []collalias{{0b01111111, "ucs2_bin"}}, isdefault: 0b00000000}, - 91: {alias: []collalias{{0b01111111, "ujis_bin"}}, isdefault: 0b00000000}, - 92: {alias: []collalias{{0b01111111, "geostd8_general_ci"}}, isdefault: 0b01111111}, - 93: {alias: []collalias{{0b01111111, "geostd8_bin"}}, isdefault: 0b00000000}, - 94: {alias: []collalias{{0b01111111, "latin1_spanish_ci"}}, isdefault: 0b00000000}, - 95: {alias: []collalias{{0b01111111, "cp932_japanese_ci"}}, isdefault: 0b01111111}, - 96: {alias: []collalias{{0b01111111, "cp932_bin"}}, isdefault: 0b00000000}, - 97: {alias: []collalias{{0b01111111, "eucjpms_japanese_ci"}}, isdefault: 0b01111111}, - 98: {alias: []collalias{{0b01111111, "eucjpms_bin"}}, isdefault: 0b00000000}, - 99: {alias: []collalias{{0b01111111, "cp1250_polish_ci"}}, isdefault: 0b00000000}, - 101: {alias: []collalias{{0b01111111, "utf16_unicode_ci"}}, isdefault: 0b00000000}, - 102: {alias: []collalias{{0b01111111, "utf16_icelandic_ci"}}, isdefault: 0b00000000}, - 103: {alias: []collalias{{0b01111111, "utf16_latvian_ci"}}, isdefault: 0b00000000}, - 104: {alias: []collalias{{0b01111111, "utf16_romanian_ci"}}, isdefault: 0b00000000}, - 105: {alias: []collalias{{0b01111111, "utf16_slovenian_ci"}}, isdefault: 0b00000000}, - 106: {alias: []collalias{{0b01111111, "utf16_polish_ci"}}, isdefault: 0b00000000}, - 107: {alias: []collalias{{0b01111111, "utf16_estonian_ci"}}, isdefault: 0b00000000}, - 108: {alias: []collalias{{0b01111111, "utf16_spanish_ci"}}, isdefault: 0b00000000}, - 109: {alias: []collalias{{0b01111111, "utf16_swedish_ci"}}, isdefault: 0b00000000}, - 110: {alias: []collalias{{0b01111111, "utf16_turkish_ci"}}, isdefault: 0b00000000}, - 111: {alias: []collalias{{0b01111111, "utf16_czech_ci"}}, isdefault: 0b00000000}, - 112: {alias: []collalias{{0b01111111, "utf16_danish_ci"}}, isdefault: 0b00000000}, - 113: {alias: []collalias{{0b01111111, "utf16_lithuanian_ci"}}, isdefault: 0b00000000}, - 114: {alias: []collalias{{0b01111111, "utf16_slovak_ci"}}, isdefault: 0b00000000}, - 115: {alias: []collalias{{0b01111111, "utf16_spanish2_ci"}}, isdefault: 0b00000000}, - 116: {alias: []collalias{{0b01111111, "utf16_roman_ci"}}, isdefault: 0b00000000}, - 117: {alias: []collalias{{0b01111111, "utf16_persian_ci"}}, isdefault: 0b00000000}, - 118: {alias: []collalias{{0b01111111, "utf16_esperanto_ci"}}, isdefault: 0b00000000}, - 119: {alias: []collalias{{0b01111111, "utf16_hungarian_ci"}}, isdefault: 0b00000000}, - 120: {alias: []collalias{{0b01111111, "utf16_sinhala_ci"}}, isdefault: 0b00000000}, - 121: {alias: []collalias{{0b01111111, "utf16_german2_ci"}}, isdefault: 0b00000000}, - 122: {alias: []collalias{{0b01110000, "utf16_croatian_ci"}, {0b00001111, "utf16_croatian_mysql561_ci"}}, isdefault: 0b00000000}, - 123: {alias: []collalias{{0b01111111, "utf16_unicode_520_ci"}}, isdefault: 0b00000000}, - 124: {alias: []collalias{{0b01111111, "utf16_vietnamese_ci"}}, isdefault: 0b00000000}, - 128: {alias: []collalias{{0b01111111, "ucs2_unicode_ci"}}, isdefault: 0b00000000}, - 129: {alias: []collalias{{0b01111111, "ucs2_icelandic_ci"}}, isdefault: 0b00000000}, - 130: {alias: []collalias{{0b01111111, "ucs2_latvian_ci"}}, isdefault: 0b00000000}, - 131: {alias: []collalias{{0b01111111, "ucs2_romanian_ci"}}, isdefault: 0b00000000}, - 132: {alias: []collalias{{0b01111111, "ucs2_slovenian_ci"}}, isdefault: 0b00000000}, - 133: {alias: []collalias{{0b01111111, "ucs2_polish_ci"}}, isdefault: 0b00000000}, - 134: {alias: []collalias{{0b01111111, "ucs2_estonian_ci"}}, isdefault: 0b00000000}, - 135: {alias: []collalias{{0b01111111, "ucs2_spanish_ci"}}, isdefault: 0b00000000}, - 136: {alias: []collalias{{0b01111111, "ucs2_swedish_ci"}}, isdefault: 0b00000000}, - 137: {alias: []collalias{{0b01111111, "ucs2_turkish_ci"}}, isdefault: 0b00000000}, - 138: {alias: []collalias{{0b01111111, "ucs2_czech_ci"}}, isdefault: 0b00000000}, - 139: {alias: []collalias{{0b01111111, "ucs2_danish_ci"}}, isdefault: 0b00000000}, - 140: {alias: []collalias{{0b01111111, "ucs2_lithuanian_ci"}}, isdefault: 0b00000000}, - 141: {alias: []collalias{{0b01111111, "ucs2_slovak_ci"}}, isdefault: 0b00000000}, - 142: {alias: []collalias{{0b01111111, "ucs2_spanish2_ci"}}, isdefault: 0b00000000}, - 143: {alias: []collalias{{0b01111111, "ucs2_roman_ci"}}, isdefault: 0b00000000}, - 144: {alias: []collalias{{0b01111111, "ucs2_persian_ci"}}, isdefault: 0b00000000}, - 145: {alias: []collalias{{0b01111111, "ucs2_esperanto_ci"}}, isdefault: 0b00000000}, - 146: {alias: []collalias{{0b01111111, "ucs2_hungarian_ci"}}, isdefault: 0b00000000}, - 147: {alias: []collalias{{0b01111111, "ucs2_sinhala_ci"}}, isdefault: 0b00000000}, - 148: {alias: []collalias{{0b01111111, "ucs2_german2_ci"}}, isdefault: 0b00000000}, - 149: {alias: []collalias{{0b01110000, "ucs2_croatian_ci"}, {0b00001111, "ucs2_croatian_mysql561_ci"}}, isdefault: 0b00000000}, - 150: {alias: []collalias{{0b01111111, "ucs2_unicode_520_ci"}}, isdefault: 0b00000000}, - 151: {alias: []collalias{{0b01111111, "ucs2_vietnamese_ci"}}, isdefault: 0b00000000}, - 159: {alias: []collalias{{0b01111111, "ucs2_general_mysql500_ci"}}, isdefault: 0b00000000}, - 160: {alias: []collalias{{0b01111111, "utf32_unicode_ci"}}, isdefault: 0b00000000}, - 161: {alias: []collalias{{0b01111111, "utf32_icelandic_ci"}}, isdefault: 0b00000000}, - 162: {alias: []collalias{{0b01111111, "utf32_latvian_ci"}}, isdefault: 0b00000000}, - 163: {alias: []collalias{{0b01111111, "utf32_romanian_ci"}}, isdefault: 0b00000000}, - 164: {alias: []collalias{{0b01111111, "utf32_slovenian_ci"}}, isdefault: 0b00000000}, - 165: {alias: []collalias{{0b01111111, "utf32_polish_ci"}}, isdefault: 0b00000000}, - 166: {alias: []collalias{{0b01111111, "utf32_estonian_ci"}}, isdefault: 0b00000000}, - 167: {alias: []collalias{{0b01111111, "utf32_spanish_ci"}}, isdefault: 0b00000000}, - 168: {alias: []collalias{{0b01111111, "utf32_swedish_ci"}}, isdefault: 0b00000000}, - 169: {alias: []collalias{{0b01111111, "utf32_turkish_ci"}}, isdefault: 0b00000000}, - 170: {alias: []collalias{{0b01111111, "utf32_czech_ci"}}, isdefault: 0b00000000}, - 171: {alias: []collalias{{0b01111111, "utf32_danish_ci"}}, isdefault: 0b00000000}, - 172: {alias: []collalias{{0b01111111, "utf32_lithuanian_ci"}}, isdefault: 0b00000000}, - 173: {alias: []collalias{{0b01111111, "utf32_slovak_ci"}}, isdefault: 0b00000000}, - 174: {alias: []collalias{{0b01111111, "utf32_spanish2_ci"}}, isdefault: 0b00000000}, - 175: {alias: []collalias{{0b01111111, "utf32_roman_ci"}}, isdefault: 0b00000000}, - 176: {alias: []collalias{{0b01111111, "utf32_persian_ci"}}, isdefault: 0b00000000}, - 177: {alias: []collalias{{0b01111111, "utf32_esperanto_ci"}}, isdefault: 0b00000000}, - 178: {alias: []collalias{{0b01111111, "utf32_hungarian_ci"}}, isdefault: 0b00000000}, - 179: {alias: []collalias{{0b01111111, "utf32_sinhala_ci"}}, isdefault: 0b00000000}, - 180: {alias: []collalias{{0b01111111, "utf32_german2_ci"}}, isdefault: 0b00000000}, - 181: {alias: []collalias{{0b01110000, "utf32_croatian_ci"}, {0b00001111, "utf32_croatian_mysql561_ci"}}, isdefault: 0b00000000}, - 182: {alias: []collalias{{0b01111111, "utf32_unicode_520_ci"}}, isdefault: 0b00000000}, - 183: {alias: []collalias{{0b01111111, "utf32_vietnamese_ci"}}, isdefault: 0b00000000}, - 192: {alias: []collalias{{0b01111111, "utf8_unicode_ci"}, {0b01111111, "utf8mb3_unicode_ci"}}, isdefault: 0b00000000}, - 193: {alias: []collalias{{0b01111111, "utf8_icelandic_ci"}, {0b01111111, "utf8mb3_icelandic_ci"}}, isdefault: 0b00000000}, - 194: {alias: []collalias{{0b01111111, "utf8_latvian_ci"}, {0b01111111, "utf8mb3_latvian_ci"}}, isdefault: 0b00000000}, - 195: {alias: []collalias{{0b01111111, "utf8_romanian_ci"}, {0b01111111, "utf8mb3_romanian_ci"}}, isdefault: 0b00000000}, - 196: {alias: []collalias{{0b01111111, "utf8_slovenian_ci"}, {0b01111111, "utf8mb3_slovenian_ci"}}, isdefault: 0b00000000}, - 197: {alias: []collalias{{0b01111111, "utf8_polish_ci"}, {0b01111111, "utf8mb3_polish_ci"}}, isdefault: 0b00000000}, - 198: {alias: []collalias{{0b01111111, "utf8_estonian_ci"}, {0b01111111, "utf8mb3_estonian_ci"}}, isdefault: 0b00000000}, - 199: {alias: []collalias{{0b01111111, "utf8_spanish_ci"}, {0b01111111, "utf8mb3_spanish_ci"}}, isdefault: 0b00000000}, - 200: {alias: []collalias{{0b01111111, "utf8_swedish_ci"}, {0b01111111, "utf8mb3_swedish_ci"}}, isdefault: 0b00000000}, - 201: {alias: []collalias{{0b01111111, "utf8_turkish_ci"}, {0b01111111, "utf8mb3_turkish_ci"}}, isdefault: 0b00000000}, - 202: {alias: []collalias{{0b01111111, "utf8_czech_ci"}, {0b01111111, "utf8mb3_czech_ci"}}, isdefault: 0b00000000}, - 203: {alias: []collalias{{0b01111111, "utf8_danish_ci"}, {0b01111111, "utf8mb3_danish_ci"}}, isdefault: 0b00000000}, - 204: {alias: []collalias{{0b01111111, "utf8_lithuanian_ci"}, {0b01111111, "utf8mb3_lithuanian_ci"}}, isdefault: 0b00000000}, - 205: {alias: []collalias{{0b01111111, "utf8_slovak_ci"}, {0b01111111, "utf8mb3_slovak_ci"}}, isdefault: 0b00000000}, - 206: {alias: []collalias{{0b01111111, "utf8_spanish2_ci"}, {0b01111111, "utf8mb3_spanish2_ci"}}, isdefault: 0b00000000}, - 207: {alias: []collalias{{0b01111111, "utf8_roman_ci"}, {0b01111111, "utf8mb3_roman_ci"}}, isdefault: 0b00000000}, - 208: {alias: []collalias{{0b01111111, "utf8_persian_ci"}, {0b01111111, "utf8mb3_persian_ci"}}, isdefault: 0b00000000}, - 209: {alias: []collalias{{0b01111111, "utf8_esperanto_ci"}, {0b01111111, "utf8mb3_esperanto_ci"}}, isdefault: 0b00000000}, - 210: {alias: []collalias{{0b01111111, "utf8_hungarian_ci"}, {0b01111111, "utf8mb3_hungarian_ci"}}, isdefault: 0b00000000}, - 211: {alias: []collalias{{0b01111111, "utf8_sinhala_ci"}, {0b01111111, "utf8mb3_sinhala_ci"}}, isdefault: 0b00000000}, - 212: {alias: []collalias{{0b01111111, "utf8_german2_ci"}, {0b01111111, "utf8mb3_german2_ci"}}, isdefault: 0b00000000}, - 213: {alias: []collalias{{0b01110000, "utf8_croatian_ci"}, {0b00001111, "utf8_croatian_mysql561_ci"}, {0b01110000, "utf8mb3_croatian_ci"}, {0b00001111, "utf8mb3_croatian_mysql561_ci"}}, isdefault: 0b00000000}, - 214: {alias: []collalias{{0b01111111, "utf8_unicode_520_ci"}, {0b01111111, "utf8mb3_unicode_520_ci"}}, isdefault: 0b00000000}, - 215: {alias: []collalias{{0b01111111, "utf8_vietnamese_ci"}, {0b01111111, "utf8mb3_vietnamese_ci"}}, isdefault: 0b00000000}, - 223: {alias: []collalias{{0b01111111, "utf8_general_mysql500_ci"}, {0b01111111, "utf8mb3_general_mysql500_ci"}}, isdefault: 0b00000000}, - 224: {alias: []collalias{{0b01111111, "utf8mb4_unicode_ci"}}, isdefault: 0b00000000}, - 225: {alias: []collalias{{0b01111111, "utf8mb4_icelandic_ci"}}, isdefault: 0b00000000}, - 226: {alias: []collalias{{0b01111111, "utf8mb4_latvian_ci"}}, isdefault: 0b00000000}, - 227: {alias: []collalias{{0b01111111, "utf8mb4_romanian_ci"}}, isdefault: 0b00000000}, - 228: {alias: []collalias{{0b01111111, "utf8mb4_slovenian_ci"}}, isdefault: 0b00000000}, - 229: {alias: []collalias{{0b01111111, "utf8mb4_polish_ci"}}, isdefault: 0b00000000}, - 230: {alias: []collalias{{0b01111111, "utf8mb4_estonian_ci"}}, isdefault: 0b00000000}, - 231: {alias: []collalias{{0b01111111, "utf8mb4_spanish_ci"}}, isdefault: 0b00000000}, - 232: {alias: []collalias{{0b01111111, "utf8mb4_swedish_ci"}}, isdefault: 0b00000000}, - 233: {alias: []collalias{{0b01111111, "utf8mb4_turkish_ci"}}, isdefault: 0b00000000}, - 234: {alias: []collalias{{0b01111111, "utf8mb4_czech_ci"}}, isdefault: 0b00000000}, - 235: {alias: []collalias{{0b01111111, "utf8mb4_danish_ci"}}, isdefault: 0b00000000}, - 236: {alias: []collalias{{0b01111111, "utf8mb4_lithuanian_ci"}}, isdefault: 0b00000000}, - 237: {alias: []collalias{{0b01111111, "utf8mb4_slovak_ci"}}, isdefault: 0b00000000}, - 238: {alias: []collalias{{0b01111111, "utf8mb4_spanish2_ci"}}, isdefault: 0b00000000}, - 239: {alias: []collalias{{0b01111111, "utf8mb4_roman_ci"}}, isdefault: 0b00000000}, - 240: {alias: []collalias{{0b01111111, "utf8mb4_persian_ci"}}, isdefault: 0b00000000}, - 241: {alias: []collalias{{0b01111111, "utf8mb4_esperanto_ci"}}, isdefault: 0b00000000}, - 242: {alias: []collalias{{0b01111111, "utf8mb4_hungarian_ci"}}, isdefault: 0b00000000}, - 243: {alias: []collalias{{0b01111111, "utf8mb4_sinhala_ci"}}, isdefault: 0b00000000}, - 244: {alias: []collalias{{0b01111111, "utf8mb4_german2_ci"}}, isdefault: 0b00000000}, - 245: {alias: []collalias{{0b01110000, "utf8mb4_croatian_ci"}, {0b00001111, "utf8mb4_croatian_mysql561_ci"}}, isdefault: 0b00000000}, - 246: {alias: []collalias{{0b01111111, "utf8mb4_unicode_520_ci"}}, isdefault: 0b00000000}, - 247: {alias: []collalias{{0b01111111, "utf8mb4_vietnamese_ci"}}, isdefault: 0b00000000}, - 248: {alias: []collalias{{0b01100000, "gb18030_chinese_ci"}}, isdefault: 0b01100000}, - 249: {alias: []collalias{{0b01100000, "gb18030_bin"}}, isdefault: 0b00000000}, - 250: {alias: []collalias{{0b01100000, "gb18030_unicode_520_ci"}}, isdefault: 0b00000000}, - 255: {alias: []collalias{{0b01000000, "utf8mb4_0900_ai_ci"}}, isdefault: 0b01000000}, - 256: {alias: []collalias{{0b01000000, "utf8mb4_de_pb_0900_ai_ci"}}, isdefault: 0b00000000}, - 257: {alias: []collalias{{0b01000000, "utf8mb4_is_0900_ai_ci"}}, isdefault: 0b00000000}, - 258: {alias: []collalias{{0b01000000, "utf8mb4_lv_0900_ai_ci"}}, isdefault: 0b00000000}, - 259: {alias: []collalias{{0b01000000, "utf8mb4_ro_0900_ai_ci"}}, isdefault: 0b00000000}, - 260: {alias: []collalias{{0b01000000, "utf8mb4_sl_0900_ai_ci"}}, isdefault: 0b00000000}, - 261: {alias: []collalias{{0b01000000, "utf8mb4_pl_0900_ai_ci"}}, isdefault: 0b00000000}, - 262: {alias: []collalias{{0b01000000, "utf8mb4_et_0900_ai_ci"}}, isdefault: 0b00000000}, - 263: {alias: []collalias{{0b01000000, "utf8mb4_es_0900_ai_ci"}}, isdefault: 0b00000000}, - 264: {alias: []collalias{{0b01000000, "utf8mb4_sv_0900_ai_ci"}}, isdefault: 0b00000000}, - 265: {alias: []collalias{{0b01000000, "utf8mb4_tr_0900_ai_ci"}}, isdefault: 0b00000000}, - 266: {alias: []collalias{{0b01000000, "utf8mb4_cs_0900_ai_ci"}}, isdefault: 0b00000000}, - 267: {alias: []collalias{{0b01000000, "utf8mb4_da_0900_ai_ci"}}, isdefault: 0b00000000}, - 268: {alias: []collalias{{0b01000000, "utf8mb4_lt_0900_ai_ci"}}, isdefault: 0b00000000}, - 269: {alias: []collalias{{0b01000000, "utf8mb4_sk_0900_ai_ci"}}, isdefault: 0b00000000}, - 270: {alias: []collalias{{0b01000000, "utf8mb4_es_trad_0900_ai_ci"}}, isdefault: 0b00000000}, - 271: {alias: []collalias{{0b01000000, "utf8mb4_la_0900_ai_ci"}}, isdefault: 0b00000000}, - 273: {alias: []collalias{{0b01000000, "utf8mb4_eo_0900_ai_ci"}}, isdefault: 0b00000000}, - 274: {alias: []collalias{{0b01000000, "utf8mb4_hu_0900_ai_ci"}}, isdefault: 0b00000000}, - 275: {alias: []collalias{{0b01000000, "utf8mb4_hr_0900_ai_ci"}}, isdefault: 0b00000000}, - 277: {alias: []collalias{{0b01000000, "utf8mb4_vi_0900_ai_ci"}}, isdefault: 0b00000000}, - 278: {alias: []collalias{{0b01000000, "utf8mb4_0900_as_cs"}}, isdefault: 0b00000000}, - 279: {alias: []collalias{{0b01000000, "utf8mb4_de_pb_0900_as_cs"}}, isdefault: 0b00000000}, - 280: {alias: []collalias{{0b01000000, "utf8mb4_is_0900_as_cs"}}, isdefault: 0b00000000}, - 281: {alias: []collalias{{0b01000000, "utf8mb4_lv_0900_as_cs"}}, isdefault: 0b00000000}, - 282: {alias: []collalias{{0b01000000, "utf8mb4_ro_0900_as_cs"}}, isdefault: 0b00000000}, - 283: {alias: []collalias{{0b01000000, "utf8mb4_sl_0900_as_cs"}}, isdefault: 0b00000000}, - 284: {alias: []collalias{{0b01000000, "utf8mb4_pl_0900_as_cs"}}, isdefault: 0b00000000}, - 285: {alias: []collalias{{0b01000000, "utf8mb4_et_0900_as_cs"}}, isdefault: 0b00000000}, - 286: {alias: []collalias{{0b01000000, "utf8mb4_es_0900_as_cs"}}, isdefault: 0b00000000}, - 287: {alias: []collalias{{0b01000000, "utf8mb4_sv_0900_as_cs"}}, isdefault: 0b00000000}, - 288: {alias: []collalias{{0b01000000, "utf8mb4_tr_0900_as_cs"}}, isdefault: 0b00000000}, - 289: {alias: []collalias{{0b01000000, "utf8mb4_cs_0900_as_cs"}}, isdefault: 0b00000000}, - 290: {alias: []collalias{{0b01000000, "utf8mb4_da_0900_as_cs"}}, isdefault: 0b00000000}, - 291: {alias: []collalias{{0b01000000, "utf8mb4_lt_0900_as_cs"}}, isdefault: 0b00000000}, - 292: {alias: []collalias{{0b01000000, "utf8mb4_sk_0900_as_cs"}}, isdefault: 0b00000000}, - 293: {alias: []collalias{{0b01000000, "utf8mb4_es_trad_0900_as_cs"}}, isdefault: 0b00000000}, - 294: {alias: []collalias{{0b01000000, "utf8mb4_la_0900_as_cs"}}, isdefault: 0b00000000}, - 296: {alias: []collalias{{0b01000000, "utf8mb4_eo_0900_as_cs"}}, isdefault: 0b00000000}, - 297: {alias: []collalias{{0b01000000, "utf8mb4_hu_0900_as_cs"}}, isdefault: 0b00000000}, - 298: {alias: []collalias{{0b01000000, "utf8mb4_hr_0900_as_cs"}}, isdefault: 0b00000000}, - 300: {alias: []collalias{{0b01000000, "utf8mb4_vi_0900_as_cs"}}, isdefault: 0b00000000}, - 303: {alias: []collalias{{0b01000000, "utf8mb4_ja_0900_as_cs"}}, isdefault: 0b00000000}, - 304: {alias: []collalias{{0b01000000, "utf8mb4_ja_0900_as_cs_ks"}}, isdefault: 0b00000000}, - 305: {alias: []collalias{{0b01000000, "utf8mb4_0900_as_ci"}}, isdefault: 0b00000000}, - 306: {alias: []collalias{{0b01000000, "utf8mb4_ru_0900_ai_ci"}}, isdefault: 0b00000000}, - 307: {alias: []collalias{{0b01000000, "utf8mb4_ru_0900_as_cs"}}, isdefault: 0b00000000}, - 308: {alias: []collalias{{0b01000000, "utf8mb4_zh_0900_as_cs"}}, isdefault: 0b00000000}, - 309: {alias: []collalias{{0b01000000, "utf8mb4_0900_bin"}}, isdefault: 0b00000000}, - 310: {alias: []collalias{{0b01000000, "utf8mb4_nb_0900_ai_ci"}}, isdefault: 0b00000000}, - 311: {alias: []collalias{{0b01000000, "utf8mb4_nb_0900_as_cs"}}, isdefault: 0b00000000}, - 312: {alias: []collalias{{0b01000000, "utf8mb4_nn_0900_ai_ci"}}, isdefault: 0b00000000}, - 313: {alias: []collalias{{0b01000000, "utf8mb4_nn_0900_as_cs"}}, isdefault: 0b00000000}, - 314: {alias: []collalias{{0b01000000, "utf8mb4_sr_latn_0900_ai_ci"}}, isdefault: 0b00000000}, - 315: {alias: []collalias{{0b01000000, "utf8mb4_sr_latn_0900_as_cs"}}, isdefault: 0b00000000}, - 316: {alias: []collalias{{0b01000000, "utf8mb4_bs_0900_ai_ci"}}, isdefault: 0b00000000}, - 317: {alias: []collalias{{0b01000000, "utf8mb4_bs_0900_as_cs"}}, isdefault: 0b00000000}, - 318: {alias: []collalias{{0b01000000, "utf8mb4_bg_0900_ai_ci"}}, isdefault: 0b00000000}, - 319: {alias: []collalias{{0b01000000, "utf8mb4_bg_0900_as_cs"}}, isdefault: 0b00000000}, - 320: {alias: []collalias{{0b01000000, "utf8mb4_gl_0900_ai_ci"}}, isdefault: 0b00000000}, - 321: {alias: []collalias{{0b01000000, "utf8mb4_gl_0900_as_cs"}}, isdefault: 0b00000000}, - 322: {alias: []collalias{{0b01000000, "utf8mb4_mn_cyrl_0900_ai_ci"}}, isdefault: 0b00000000}, - 323: {alias: []collalias{{0b01000000, "utf8mb4_mn_cyrl_0900_as_cs"}}, isdefault: 0b00000000}, - 576: {alias: []collalias{{0b00001111, "utf8_croatian_ci"}, {0b00001111, "utf8mb3_croatian_ci"}}, isdefault: 0b00000000}, - 577: {alias: []collalias{{0b00001111, "utf8_myanmar_ci"}, {0b00001111, "utf8mb3_myanmar_ci"}}, isdefault: 0b00000000}, - 578: {alias: []collalias{{0b00001110, "utf8_thai_520_w2"}, {0b00001110, "utf8mb3_thai_520_w2"}}, isdefault: 0b00000000}, - 608: {alias: []collalias{{0b00001111, "utf8mb4_croatian_ci"}}, isdefault: 0b00000000}, - 609: {alias: []collalias{{0b00001111, "utf8mb4_myanmar_ci"}}, isdefault: 0b00000000}, - 610: {alias: []collalias{{0b00001110, "utf8mb4_thai_520_w2"}}, isdefault: 0b00000000}, - 640: {alias: []collalias{{0b00001111, "ucs2_croatian_ci"}}, isdefault: 0b00000000}, - 641: {alias: []collalias{{0b00001111, "ucs2_myanmar_ci"}}, isdefault: 0b00000000}, - 642: {alias: []collalias{{0b00001110, "ucs2_thai_520_w2"}}, isdefault: 0b00000000}, - 672: {alias: []collalias{{0b00001111, "utf16_croatian_ci"}}, isdefault: 0b00000000}, - 673: {alias: []collalias{{0b00001111, "utf16_myanmar_ci"}}, isdefault: 0b00000000}, - 674: {alias: []collalias{{0b00001110, "utf16_thai_520_w2"}}, isdefault: 0b00000000}, - 736: {alias: []collalias{{0b00001111, "utf32_croatian_ci"}}, isdefault: 0b00000000}, - 737: {alias: []collalias{{0b00001111, "utf32_myanmar_ci"}}, isdefault: 0b00000000}, - 738: {alias: []collalias{{0b00001110, "utf32_thai_520_w2"}}, isdefault: 0b00000000}, - 1025: {alias: []collalias{{0b00001100, "big5_chinese_nopad_ci"}}, isdefault: 0b00000000}, - 1027: {alias: []collalias{{0b00001100, "dec8_swedish_nopad_ci"}}, isdefault: 0b00000000}, - 1028: {alias: []collalias{{0b00001100, "cp850_general_nopad_ci"}}, isdefault: 0b00000000}, - 1030: {alias: []collalias{{0b00001100, "hp8_english_nopad_ci"}}, isdefault: 0b00000000}, - 1031: {alias: []collalias{{0b00001100, "koi8r_general_nopad_ci"}}, isdefault: 0b00000000}, - 1032: {alias: []collalias{{0b00001100, "latin1_swedish_nopad_ci"}}, isdefault: 0b00000000}, - 1033: {alias: []collalias{{0b00001100, "latin2_general_nopad_ci"}}, isdefault: 0b00000000}, - 1034: {alias: []collalias{{0b00001100, "swe7_swedish_nopad_ci"}}, isdefault: 0b00000000}, - 1035: {alias: []collalias{{0b00001100, "ascii_general_nopad_ci"}}, isdefault: 0b00000000}, - 1036: {alias: []collalias{{0b00001100, "ujis_japanese_nopad_ci"}}, isdefault: 0b00000000}, - 1037: {alias: []collalias{{0b00001100, "sjis_japanese_nopad_ci"}}, isdefault: 0b00000000}, - 1040: {alias: []collalias{{0b00001100, "hebrew_general_nopad_ci"}}, isdefault: 0b00000000}, - 1042: {alias: []collalias{{0b00001100, "tis620_thai_nopad_ci"}}, isdefault: 0b00000000}, - 1043: {alias: []collalias{{0b00001100, "euckr_korean_nopad_ci"}}, isdefault: 0b00000000}, - 1046: {alias: []collalias{{0b00001100, "koi8u_general_nopad_ci"}}, isdefault: 0b00000000}, - 1048: {alias: []collalias{{0b00001100, "gb2312_chinese_nopad_ci"}}, isdefault: 0b00000000}, - 1049: {alias: []collalias{{0b00001100, "greek_general_nopad_ci"}}, isdefault: 0b00000000}, - 1050: {alias: []collalias{{0b00001100, "cp1250_general_nopad_ci"}}, isdefault: 0b00000000}, - 1052: {alias: []collalias{{0b00001100, "gbk_chinese_nopad_ci"}}, isdefault: 0b00000000}, - 1054: {alias: []collalias{{0b00001100, "latin5_turkish_nopad_ci"}}, isdefault: 0b00000000}, - 1056: {alias: []collalias{{0b00001100, "armscii8_general_nopad_ci"}}, isdefault: 0b00000000}, - 1057: {alias: []collalias{{0b00001100, "utf8_general_nopad_ci"}, {0b00001100, "utf8mb3_general_nopad_ci"}}, isdefault: 0b00000000}, - 1059: {alias: []collalias{{0b00001100, "ucs2_general_nopad_ci"}}, isdefault: 0b00000000}, - 1060: {alias: []collalias{{0b00001100, "cp866_general_nopad_ci"}}, isdefault: 0b00000000}, - 1061: {alias: []collalias{{0b00001100, "keybcs2_general_nopad_ci"}}, isdefault: 0b00000000}, - 1062: {alias: []collalias{{0b00001100, "macce_general_nopad_ci"}}, isdefault: 0b00000000}, - 1063: {alias: []collalias{{0b00001100, "macroman_general_nopad_ci"}}, isdefault: 0b00000000}, - 1064: {alias: []collalias{{0b00001100, "cp852_general_nopad_ci"}}, isdefault: 0b00000000}, - 1065: {alias: []collalias{{0b00001100, "latin7_general_nopad_ci"}}, isdefault: 0b00000000}, - 1067: {alias: []collalias{{0b00001100, "macce_nopad_bin"}}, isdefault: 0b00000000}, - 1069: {alias: []collalias{{0b00001100, "utf8mb4_general_nopad_ci"}}, isdefault: 0b00000000}, - 1070: {alias: []collalias{{0b00001100, "utf8mb4_nopad_bin"}}, isdefault: 0b00000000}, - 1071: {alias: []collalias{{0b00001100, "latin1_nopad_bin"}}, isdefault: 0b00000000}, - 1074: {alias: []collalias{{0b00001100, "cp1251_nopad_bin"}}, isdefault: 0b00000000}, - 1075: {alias: []collalias{{0b00001100, "cp1251_general_nopad_ci"}}, isdefault: 0b00000000}, - 1077: {alias: []collalias{{0b00001100, "macroman_nopad_bin"}}, isdefault: 0b00000000}, - 1078: {alias: []collalias{{0b00001100, "utf16_general_nopad_ci"}}, isdefault: 0b00000000}, - 1079: {alias: []collalias{{0b00001100, "utf16_nopad_bin"}}, isdefault: 0b00000000}, - 1080: {alias: []collalias{{0b00001100, "utf16le_general_nopad_ci"}}, isdefault: 0b00000000}, - 1081: {alias: []collalias{{0b00001100, "cp1256_general_nopad_ci"}}, isdefault: 0b00000000}, - 1082: {alias: []collalias{{0b00001100, "cp1257_nopad_bin"}}, isdefault: 0b00000000}, - 1083: {alias: []collalias{{0b00001100, "cp1257_general_nopad_ci"}}, isdefault: 0b00000000}, - 1084: {alias: []collalias{{0b00001100, "utf32_general_nopad_ci"}}, isdefault: 0b00000000}, - 1085: {alias: []collalias{{0b00001100, "utf32_nopad_bin"}}, isdefault: 0b00000000}, - 1086: {alias: []collalias{{0b00001100, "utf16le_nopad_bin"}}, isdefault: 0b00000000}, - 1088: {alias: []collalias{{0b00001100, "armscii8_nopad_bin"}}, isdefault: 0b00000000}, - 1089: {alias: []collalias{{0b00001100, "ascii_nopad_bin"}}, isdefault: 0b00000000}, - 1090: {alias: []collalias{{0b00001100, "cp1250_nopad_bin"}}, isdefault: 0b00000000}, - 1091: {alias: []collalias{{0b00001100, "cp1256_nopad_bin"}}, isdefault: 0b00000000}, - 1092: {alias: []collalias{{0b00001100, "cp866_nopad_bin"}}, isdefault: 0b00000000}, - 1093: {alias: []collalias{{0b00001100, "dec8_nopad_bin"}}, isdefault: 0b00000000}, - 1094: {alias: []collalias{{0b00001100, "greek_nopad_bin"}}, isdefault: 0b00000000}, - 1095: {alias: []collalias{{0b00001100, "hebrew_nopad_bin"}}, isdefault: 0b00000000}, - 1096: {alias: []collalias{{0b00001100, "hp8_nopad_bin"}}, isdefault: 0b00000000}, - 1097: {alias: []collalias{{0b00001100, "keybcs2_nopad_bin"}}, isdefault: 0b00000000}, - 1098: {alias: []collalias{{0b00001100, "koi8r_nopad_bin"}}, isdefault: 0b00000000}, - 1099: {alias: []collalias{{0b00001100, "koi8u_nopad_bin"}}, isdefault: 0b00000000}, - 1101: {alias: []collalias{{0b00001100, "latin2_nopad_bin"}}, isdefault: 0b00000000}, - 1102: {alias: []collalias{{0b00001100, "latin5_nopad_bin"}}, isdefault: 0b00000000}, - 1103: {alias: []collalias{{0b00001100, "latin7_nopad_bin"}}, isdefault: 0b00000000}, - 1104: {alias: []collalias{{0b00001100, "cp850_nopad_bin"}}, isdefault: 0b00000000}, - 1105: {alias: []collalias{{0b00001100, "cp852_nopad_bin"}}, isdefault: 0b00000000}, - 1106: {alias: []collalias{{0b00001100, "swe7_nopad_bin"}}, isdefault: 0b00000000}, - 1107: {alias: []collalias{{0b00001100, "utf8_nopad_bin"}, {0b00001100, "utf8mb3_nopad_bin"}}, isdefault: 0b00000000}, - 1108: {alias: []collalias{{0b00001100, "big5_nopad_bin"}}, isdefault: 0b00000000}, - 1109: {alias: []collalias{{0b00001100, "euckr_nopad_bin"}}, isdefault: 0b00000000}, - 1110: {alias: []collalias{{0b00001100, "gb2312_nopad_bin"}}, isdefault: 0b00000000}, - 1111: {alias: []collalias{{0b00001100, "gbk_nopad_bin"}}, isdefault: 0b00000000}, - 1112: {alias: []collalias{{0b00001100, "sjis_nopad_bin"}}, isdefault: 0b00000000}, - 1113: {alias: []collalias{{0b00001100, "tis620_nopad_bin"}}, isdefault: 0b00000000}, - 1114: {alias: []collalias{{0b00001100, "ucs2_nopad_bin"}}, isdefault: 0b00000000}, - 1115: {alias: []collalias{{0b00001100, "ujis_nopad_bin"}}, isdefault: 0b00000000}, - 1116: {alias: []collalias{{0b00001100, "geostd8_general_nopad_ci"}}, isdefault: 0b00000000}, - 1117: {alias: []collalias{{0b00001100, "geostd8_nopad_bin"}}, isdefault: 0b00000000}, - 1119: {alias: []collalias{{0b00001100, "cp932_japanese_nopad_ci"}}, isdefault: 0b00000000}, - 1120: {alias: []collalias{{0b00001100, "cp932_nopad_bin"}}, isdefault: 0b00000000}, - 1121: {alias: []collalias{{0b00001100, "eucjpms_japanese_nopad_ci"}}, isdefault: 0b00000000}, - 1122: {alias: []collalias{{0b00001100, "eucjpms_nopad_bin"}}, isdefault: 0b00000000}, - 1125: {alias: []collalias{{0b00001100, "utf16_unicode_nopad_ci"}}, isdefault: 0b00000000}, - 1147: {alias: []collalias{{0b00001100, "utf16_unicode_520_nopad_ci"}}, isdefault: 0b00000000}, - 1152: {alias: []collalias{{0b00001100, "ucs2_unicode_nopad_ci"}}, isdefault: 0b00000000}, - 1174: {alias: []collalias{{0b00001100, "ucs2_unicode_520_nopad_ci"}}, isdefault: 0b00000000}, - 1184: {alias: []collalias{{0b00001100, "utf32_unicode_nopad_ci"}}, isdefault: 0b00000000}, - 1206: {alias: []collalias{{0b00001100, "utf32_unicode_520_nopad_ci"}}, isdefault: 0b00000000}, - 1216: {alias: []collalias{{0b00001100, "utf8_unicode_nopad_ci"}, {0b00001100, "utf8mb3_unicode_nopad_ci"}}, isdefault: 0b00000000}, - 1238: {alias: []collalias{{0b00001100, "utf8_unicode_520_nopad_ci"}, {0b00001100, "utf8mb3_unicode_520_nopad_ci"}}, isdefault: 0b00000000}, - 1248: {alias: []collalias{{0b00001100, "utf8mb4_unicode_nopad_ci"}}, isdefault: 0b00000000}, - 1270: {alias: []collalias{{0b00001100, "utf8mb4_unicode_520_nopad_ci"}}, isdefault: 0b00000000}, + 1: {alias: []collalias{{0b01111111, "big5_chinese_ci", "big5"}}, isdefault: 0b01111111}, + 2: {alias: []collalias{{0b01111111, "latin2_czech_cs", "latin2"}}, isdefault: 0b00000000}, + 3: {alias: []collalias{{0b01111111, "dec8_swedish_ci", "dec8"}}, isdefault: 0b01111111}, + 4: {alias: []collalias{{0b01111111, "cp850_general_ci", "cp850"}}, isdefault: 0b01111111}, + 5: {alias: []collalias{{0b01111111, "latin1_german1_ci", "latin1"}}, isdefault: 0b00000000}, + 6: {alias: []collalias{{0b01111111, "hp8_english_ci", "hp8"}}, isdefault: 0b01111111}, + 7: {alias: []collalias{{0b01111111, "koi8r_general_ci", "koi8r"}}, isdefault: 0b01111111}, + 8: {alias: []collalias{{0b01111111, "latin1_swedish_ci", "latin1"}}, isdefault: 0b01111111}, + 9: {alias: []collalias{{0b01111111, "latin2_general_ci", "latin2"}}, isdefault: 0b01111111}, + 10: {alias: []collalias{{0b01111111, "swe7_swedish_ci", "swe7"}}, isdefault: 0b01111111}, + 11: {alias: []collalias{{0b01111111, "ascii_general_ci", "ascii"}}, isdefault: 0b01111111}, + 12: {alias: []collalias{{0b01111111, "ujis_japanese_ci", "ujis"}}, isdefault: 0b01111111}, + 13: {alias: []collalias{{0b01111111, "sjis_japanese_ci", "sjis"}}, isdefault: 0b01111111}, + 14: {alias: []collalias{{0b01111111, "cp1251_bulgarian_ci", "cp1251"}}, isdefault: 0b00000000}, + 15: {alias: []collalias{{0b01111111, "latin1_danish_ci", "latin1"}}, isdefault: 0b00000000}, + 16: {alias: []collalias{{0b01111111, "hebrew_general_ci", "hebrew"}}, isdefault: 0b01111111}, + 18: {alias: []collalias{{0b01111111, "tis620_thai_ci", "tis620"}}, isdefault: 0b01111111}, + 19: {alias: []collalias{{0b01111111, "euckr_korean_ci", "euckr"}}, isdefault: 0b01111111}, + 20: {alias: []collalias{{0b01111111, "latin7_estonian_cs", "latin7"}}, isdefault: 0b00000000}, + 21: {alias: []collalias{{0b01111111, "latin2_hungarian_ci", "latin2"}}, isdefault: 0b00000000}, + 22: {alias: []collalias{{0b01111111, "koi8u_general_ci", "koi8u"}}, isdefault: 0b01111111}, + 23: {alias: []collalias{{0b01111111, "cp1251_ukrainian_ci", "cp1251"}}, isdefault: 0b00000000}, + 24: {alias: []collalias{{0b01111111, "gb2312_chinese_ci", "gb2312"}}, isdefault: 0b01111111}, + 25: {alias: []collalias{{0b01111111, "greek_general_ci", "greek"}}, isdefault: 0b01111111}, + 26: {alias: []collalias{{0b01111111, "cp1250_general_ci", "cp1250"}}, isdefault: 0b01111111}, + 27: {alias: []collalias{{0b01111111, "latin2_croatian_ci", "latin2"}}, isdefault: 0b00000000}, + 28: {alias: []collalias{{0b01111111, "gbk_chinese_ci", "gbk"}}, isdefault: 0b01111111}, + 29: {alias: []collalias{{0b01111111, "cp1257_lithuanian_ci", "cp1257"}}, isdefault: 0b00000000}, + 30: {alias: []collalias{{0b01111111, "latin5_turkish_ci", "latin5"}}, isdefault: 0b01111111}, + 31: {alias: []collalias{{0b01111111, "latin1_german2_ci", "latin1"}}, isdefault: 0b00000000}, + 32: {alias: []collalias{{0b01111111, "armscii8_general_ci", "armscii8"}}, isdefault: 0b01111111}, + 33: {alias: []collalias{{0b01111111, "utf8_general_ci", "utf8"}, {0b01111111, "utf8mb3_general_ci", "utf8mb3"}}, isdefault: 0b01111111}, + 34: {alias: []collalias{{0b01111111, "cp1250_czech_cs", "cp1250"}}, isdefault: 0b00000000}, + 35: {alias: []collalias{{0b01111111, "ucs2_general_ci", "ucs2"}}, isdefault: 0b01111111}, + 36: {alias: []collalias{{0b01111111, "cp866_general_ci", "cp866"}}, isdefault: 0b01111111}, + 37: {alias: []collalias{{0b01111111, "keybcs2_general_ci", "keybcs2"}}, isdefault: 0b01111111}, + 38: {alias: []collalias{{0b01111111, "macce_general_ci", "macce"}}, isdefault: 0b01111111}, + 39: {alias: []collalias{{0b01111111, "macroman_general_ci", "macroman"}}, isdefault: 0b01111111}, + 40: {alias: []collalias{{0b01111111, "cp852_general_ci", "cp852"}}, isdefault: 0b01111111}, + 41: {alias: []collalias{{0b01111111, "latin7_general_ci", "latin7"}}, isdefault: 0b01111111}, + 42: {alias: []collalias{{0b01111111, "latin7_general_cs", "latin7"}}, isdefault: 0b00000000}, + 43: {alias: []collalias{{0b01111111, "macce_bin", "macce"}}, isdefault: 0b00000000}, + 44: {alias: []collalias{{0b01111111, "cp1250_croatian_ci", "cp1250"}}, isdefault: 0b00000000}, + 45: {alias: []collalias{{0b01111111, "utf8mb4_general_ci", "utf8mb4"}}, isdefault: 0b00111111}, + 46: {alias: []collalias{{0b01111111, "utf8mb4_bin", "utf8mb4"}}, isdefault: 0b00000000}, + 47: {alias: []collalias{{0b01111111, "latin1_bin", "latin1"}}, isdefault: 0b00000000}, + 48: {alias: []collalias{{0b01111111, "latin1_general_ci", "latin1"}}, isdefault: 0b00000000}, + 49: {alias: []collalias{{0b01111111, "latin1_general_cs", "latin1"}}, isdefault: 0b00000000}, + 50: {alias: []collalias{{0b01111111, "cp1251_bin", "cp1251"}}, isdefault: 0b00000000}, + 51: {alias: []collalias{{0b01111111, "cp1251_general_ci", "cp1251"}}, isdefault: 0b01111111}, + 52: {alias: []collalias{{0b01111111, "cp1251_general_cs", "cp1251"}}, isdefault: 0b00000000}, + 53: {alias: []collalias{{0b01111111, "macroman_bin", "macroman"}}, isdefault: 0b00000000}, + 54: {alias: []collalias{{0b01111111, "utf16_general_ci", "utf16"}}, isdefault: 0b01111111}, + 55: {alias: []collalias{{0b01111111, "utf16_bin", "utf16"}}, isdefault: 0b00000000}, + 56: {alias: []collalias{{0b01111111, "utf16le_general_ci", "utf16le"}}, isdefault: 0b01111111}, + 57: {alias: []collalias{{0b01111111, "cp1256_general_ci", "cp1256"}}, isdefault: 0b01111111}, + 58: {alias: []collalias{{0b01111111, "cp1257_bin", "cp1257"}}, isdefault: 0b00000000}, + 59: {alias: []collalias{{0b01111111, "cp1257_general_ci", "cp1257"}}, isdefault: 0b01111111}, + 60: {alias: []collalias{{0b01111111, "utf32_general_ci", "utf32"}}, isdefault: 0b01111111}, + 61: {alias: []collalias{{0b01111111, "utf32_bin", "utf32"}}, isdefault: 0b00000000}, + 62: {alias: []collalias{{0b01111111, "utf16le_bin", "utf16le"}}, isdefault: 0b00000000}, + 63: {alias: []collalias{{0b01111111, "binary", "binary"}}, isdefault: 0b01111111}, + 64: {alias: []collalias{{0b01111111, "armscii8_bin", "armscii8"}}, isdefault: 0b00000000}, + 65: {alias: []collalias{{0b01111111, "ascii_bin", "ascii"}}, isdefault: 0b00000000}, + 66: {alias: []collalias{{0b01111111, "cp1250_bin", "cp1250"}}, isdefault: 0b00000000}, + 67: {alias: []collalias{{0b01111111, "cp1256_bin", "cp1256"}}, isdefault: 0b00000000}, + 68: {alias: []collalias{{0b01111111, "cp866_bin", "cp866"}}, isdefault: 0b00000000}, + 69: {alias: []collalias{{0b01111111, "dec8_bin", "dec8"}}, isdefault: 0b00000000}, + 70: {alias: []collalias{{0b01111111, "greek_bin", "greek"}}, isdefault: 0b00000000}, + 71: {alias: []collalias{{0b01111111, "hebrew_bin", "hebrew"}}, isdefault: 0b00000000}, + 72: {alias: []collalias{{0b01111111, "hp8_bin", "hp8"}}, isdefault: 0b00000000}, + 73: {alias: []collalias{{0b01111111, "keybcs2_bin", "keybcs2"}}, isdefault: 0b00000000}, + 74: {alias: []collalias{{0b01111111, "koi8r_bin", "koi8r"}}, isdefault: 0b00000000}, + 75: {alias: []collalias{{0b01111111, "koi8u_bin", "koi8u"}}, isdefault: 0b00000000}, + 76: {alias: []collalias{{0b01000000, "utf8_tolower_ci", "utf8"}, {0b01000000, "utf8mb3_tolower_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 77: {alias: []collalias{{0b01111111, "latin2_bin", "latin2"}}, isdefault: 0b00000000}, + 78: {alias: []collalias{{0b01111111, "latin5_bin", "latin5"}}, isdefault: 0b00000000}, + 79: {alias: []collalias{{0b01111111, "latin7_bin", "latin7"}}, isdefault: 0b00000000}, + 80: {alias: []collalias{{0b01111111, "cp850_bin", "cp850"}}, isdefault: 0b00000000}, + 81: {alias: []collalias{{0b01111111, "cp852_bin", "cp852"}}, isdefault: 0b00000000}, + 82: {alias: []collalias{{0b01111111, "swe7_bin", "swe7"}}, isdefault: 0b00000000}, + 83: {alias: []collalias{{0b01111111, "utf8_bin", "utf8"}, {0b01111111, "utf8mb3_bin", "utf8mb3"}}, isdefault: 0b00000000}, + 84: {alias: []collalias{{0b01111111, "big5_bin", "big5"}}, isdefault: 0b00000000}, + 85: {alias: []collalias{{0b01111111, "euckr_bin", "euckr"}}, isdefault: 0b00000000}, + 86: {alias: []collalias{{0b01111111, "gb2312_bin", "gb2312"}}, isdefault: 0b00000000}, + 87: {alias: []collalias{{0b01111111, "gbk_bin", "gbk"}}, isdefault: 0b00000000}, + 88: {alias: []collalias{{0b01111111, "sjis_bin", "sjis"}}, isdefault: 0b00000000}, + 89: {alias: []collalias{{0b01111111, "tis620_bin", "tis620"}}, isdefault: 0b00000000}, + 90: {alias: []collalias{{0b01111111, "ucs2_bin", "ucs2"}}, isdefault: 0b00000000}, + 91: {alias: []collalias{{0b01111111, "ujis_bin", "ujis"}}, isdefault: 0b00000000}, + 92: {alias: []collalias{{0b01111111, "geostd8_general_ci", "geostd8"}}, isdefault: 0b01111111}, + 93: {alias: []collalias{{0b01111111, "geostd8_bin", "geostd8"}}, isdefault: 0b00000000}, + 94: {alias: []collalias{{0b01111111, "latin1_spanish_ci", "latin1"}}, isdefault: 0b00000000}, + 95: {alias: []collalias{{0b01111111, "cp932_japanese_ci", "cp932"}}, isdefault: 0b01111111}, + 96: {alias: []collalias{{0b01111111, "cp932_bin", "cp932"}}, isdefault: 0b00000000}, + 97: {alias: []collalias{{0b01111111, "eucjpms_japanese_ci", "eucjpms"}}, isdefault: 0b01111111}, + 98: {alias: []collalias{{0b01111111, "eucjpms_bin", "eucjpms"}}, isdefault: 0b00000000}, + 99: {alias: []collalias{{0b01111111, "cp1250_polish_ci", "cp1250"}}, isdefault: 0b00000000}, + 101: {alias: []collalias{{0b01111111, "utf16_unicode_ci", "utf16"}}, isdefault: 0b00000000}, + 102: {alias: []collalias{{0b01111111, "utf16_icelandic_ci", "utf16"}}, isdefault: 0b00000000}, + 103: {alias: []collalias{{0b01111111, "utf16_latvian_ci", "utf16"}}, isdefault: 0b00000000}, + 104: {alias: []collalias{{0b01111111, "utf16_romanian_ci", "utf16"}}, isdefault: 0b00000000}, + 105: {alias: []collalias{{0b01111111, "utf16_slovenian_ci", "utf16"}}, isdefault: 0b00000000}, + 106: {alias: []collalias{{0b01111111, "utf16_polish_ci", "utf16"}}, isdefault: 0b00000000}, + 107: {alias: []collalias{{0b01111111, "utf16_estonian_ci", "utf16"}}, isdefault: 0b00000000}, + 108: {alias: []collalias{{0b01111111, "utf16_spanish_ci", "utf16"}}, isdefault: 0b00000000}, + 109: {alias: []collalias{{0b01111111, "utf16_swedish_ci", "utf16"}}, isdefault: 0b00000000}, + 110: {alias: []collalias{{0b01111111, "utf16_turkish_ci", "utf16"}}, isdefault: 0b00000000}, + 111: {alias: []collalias{{0b01111111, "utf16_czech_ci", "utf16"}}, isdefault: 0b00000000}, + 112: {alias: []collalias{{0b01111111, "utf16_danish_ci", "utf16"}}, isdefault: 0b00000000}, + 113: {alias: []collalias{{0b01111111, "utf16_lithuanian_ci", "utf16"}}, isdefault: 0b00000000}, + 114: {alias: []collalias{{0b01111111, "utf16_slovak_ci", "utf16"}}, isdefault: 0b00000000}, + 115: {alias: []collalias{{0b01111111, "utf16_spanish2_ci", "utf16"}}, isdefault: 0b00000000}, + 116: {alias: []collalias{{0b01111111, "utf16_roman_ci", "utf16"}}, isdefault: 0b00000000}, + 117: {alias: []collalias{{0b01111111, "utf16_persian_ci", "utf16"}}, isdefault: 0b00000000}, + 118: {alias: []collalias{{0b01111111, "utf16_esperanto_ci", "utf16"}}, isdefault: 0b00000000}, + 119: {alias: []collalias{{0b01111111, "utf16_hungarian_ci", "utf16"}}, isdefault: 0b00000000}, + 120: {alias: []collalias{{0b01111111, "utf16_sinhala_ci", "utf16"}}, isdefault: 0b00000000}, + 121: {alias: []collalias{{0b01111111, "utf16_german2_ci", "utf16"}}, isdefault: 0b00000000}, + 122: {alias: []collalias{{0b01110000, "utf16_croatian_ci", "utf16"}, {0b00001111, "utf16_croatian_mysql561_ci", "utf16"}}, isdefault: 0b00000000}, + 123: {alias: []collalias{{0b01111111, "utf16_unicode_520_ci", "utf16"}}, isdefault: 0b00000000}, + 124: {alias: []collalias{{0b01111111, "utf16_vietnamese_ci", "utf16"}}, isdefault: 0b00000000}, + 128: {alias: []collalias{{0b01111111, "ucs2_unicode_ci", "ucs2"}}, isdefault: 0b00000000}, + 129: {alias: []collalias{{0b01111111, "ucs2_icelandic_ci", "ucs2"}}, isdefault: 0b00000000}, + 130: {alias: []collalias{{0b01111111, "ucs2_latvian_ci", "ucs2"}}, isdefault: 0b00000000}, + 131: {alias: []collalias{{0b01111111, "ucs2_romanian_ci", "ucs2"}}, isdefault: 0b00000000}, + 132: {alias: []collalias{{0b01111111, "ucs2_slovenian_ci", "ucs2"}}, isdefault: 0b00000000}, + 133: {alias: []collalias{{0b01111111, "ucs2_polish_ci", "ucs2"}}, isdefault: 0b00000000}, + 134: {alias: []collalias{{0b01111111, "ucs2_estonian_ci", "ucs2"}}, isdefault: 0b00000000}, + 135: {alias: []collalias{{0b01111111, "ucs2_spanish_ci", "ucs2"}}, isdefault: 0b00000000}, + 136: {alias: []collalias{{0b01111111, "ucs2_swedish_ci", "ucs2"}}, isdefault: 0b00000000}, + 137: {alias: []collalias{{0b01111111, "ucs2_turkish_ci", "ucs2"}}, isdefault: 0b00000000}, + 138: {alias: []collalias{{0b01111111, "ucs2_czech_ci", "ucs2"}}, isdefault: 0b00000000}, + 139: {alias: []collalias{{0b01111111, "ucs2_danish_ci", "ucs2"}}, isdefault: 0b00000000}, + 140: {alias: []collalias{{0b01111111, "ucs2_lithuanian_ci", "ucs2"}}, isdefault: 0b00000000}, + 141: {alias: []collalias{{0b01111111, "ucs2_slovak_ci", "ucs2"}}, isdefault: 0b00000000}, + 142: {alias: []collalias{{0b01111111, "ucs2_spanish2_ci", "ucs2"}}, isdefault: 0b00000000}, + 143: {alias: []collalias{{0b01111111, "ucs2_roman_ci", "ucs2"}}, isdefault: 0b00000000}, + 144: {alias: []collalias{{0b01111111, "ucs2_persian_ci", "ucs2"}}, isdefault: 0b00000000}, + 145: {alias: []collalias{{0b01111111, "ucs2_esperanto_ci", "ucs2"}}, isdefault: 0b00000000}, + 146: {alias: []collalias{{0b01111111, "ucs2_hungarian_ci", "ucs2"}}, isdefault: 0b00000000}, + 147: {alias: []collalias{{0b01111111, "ucs2_sinhala_ci", "ucs2"}}, isdefault: 0b00000000}, + 148: {alias: []collalias{{0b01111111, "ucs2_german2_ci", "ucs2"}}, isdefault: 0b00000000}, + 149: {alias: []collalias{{0b01110000, "ucs2_croatian_ci", "ucs2"}, {0b00001111, "ucs2_croatian_mysql561_ci", "ucs2"}}, isdefault: 0b00000000}, + 150: {alias: []collalias{{0b01111111, "ucs2_unicode_520_ci", "ucs2"}}, isdefault: 0b00000000}, + 151: {alias: []collalias{{0b01111111, "ucs2_vietnamese_ci", "ucs2"}}, isdefault: 0b00000000}, + 159: {alias: []collalias{{0b01111111, "ucs2_general_mysql500_ci", "ucs2"}}, isdefault: 0b00000000}, + 160: {alias: []collalias{{0b01111111, "utf32_unicode_ci", "utf32"}}, isdefault: 0b00000000}, + 161: {alias: []collalias{{0b01111111, "utf32_icelandic_ci", "utf32"}}, isdefault: 0b00000000}, + 162: {alias: []collalias{{0b01111111, "utf32_latvian_ci", "utf32"}}, isdefault: 0b00000000}, + 163: {alias: []collalias{{0b01111111, "utf32_romanian_ci", "utf32"}}, isdefault: 0b00000000}, + 164: {alias: []collalias{{0b01111111, "utf32_slovenian_ci", "utf32"}}, isdefault: 0b00000000}, + 165: {alias: []collalias{{0b01111111, "utf32_polish_ci", "utf32"}}, isdefault: 0b00000000}, + 166: {alias: []collalias{{0b01111111, "utf32_estonian_ci", "utf32"}}, isdefault: 0b00000000}, + 167: {alias: []collalias{{0b01111111, "utf32_spanish_ci", "utf32"}}, isdefault: 0b00000000}, + 168: {alias: []collalias{{0b01111111, "utf32_swedish_ci", "utf32"}}, isdefault: 0b00000000}, + 169: {alias: []collalias{{0b01111111, "utf32_turkish_ci", "utf32"}}, isdefault: 0b00000000}, + 170: {alias: []collalias{{0b01111111, "utf32_czech_ci", "utf32"}}, isdefault: 0b00000000}, + 171: {alias: []collalias{{0b01111111, "utf32_danish_ci", "utf32"}}, isdefault: 0b00000000}, + 172: {alias: []collalias{{0b01111111, "utf32_lithuanian_ci", "utf32"}}, isdefault: 0b00000000}, + 173: {alias: []collalias{{0b01111111, "utf32_slovak_ci", "utf32"}}, isdefault: 0b00000000}, + 174: {alias: []collalias{{0b01111111, "utf32_spanish2_ci", "utf32"}}, isdefault: 0b00000000}, + 175: {alias: []collalias{{0b01111111, "utf32_roman_ci", "utf32"}}, isdefault: 0b00000000}, + 176: {alias: []collalias{{0b01111111, "utf32_persian_ci", "utf32"}}, isdefault: 0b00000000}, + 177: {alias: []collalias{{0b01111111, "utf32_esperanto_ci", "utf32"}}, isdefault: 0b00000000}, + 178: {alias: []collalias{{0b01111111, "utf32_hungarian_ci", "utf32"}}, isdefault: 0b00000000}, + 179: {alias: []collalias{{0b01111111, "utf32_sinhala_ci", "utf32"}}, isdefault: 0b00000000}, + 180: {alias: []collalias{{0b01111111, "utf32_german2_ci", "utf32"}}, isdefault: 0b00000000}, + 181: {alias: []collalias{{0b01110000, "utf32_croatian_ci", "utf32"}, {0b00001111, "utf32_croatian_mysql561_ci", "utf32"}}, isdefault: 0b00000000}, + 182: {alias: []collalias{{0b01111111, "utf32_unicode_520_ci", "utf32"}}, isdefault: 0b00000000}, + 183: {alias: []collalias{{0b01111111, "utf32_vietnamese_ci", "utf32"}}, isdefault: 0b00000000}, + 192: {alias: []collalias{{0b01111111, "utf8_unicode_ci", "utf8"}, {0b01111111, "utf8mb3_unicode_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 193: {alias: []collalias{{0b01111111, "utf8_icelandic_ci", "utf8"}, {0b01111111, "utf8mb3_icelandic_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 194: {alias: []collalias{{0b01111111, "utf8_latvian_ci", "utf8"}, {0b01111111, "utf8mb3_latvian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 195: {alias: []collalias{{0b01111111, "utf8_romanian_ci", "utf8"}, {0b01111111, "utf8mb3_romanian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 196: {alias: []collalias{{0b01111111, "utf8_slovenian_ci", "utf8"}, {0b01111111, "utf8mb3_slovenian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 197: {alias: []collalias{{0b01111111, "utf8_polish_ci", "utf8"}, {0b01111111, "utf8mb3_polish_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 198: {alias: []collalias{{0b01111111, "utf8_estonian_ci", "utf8"}, {0b01111111, "utf8mb3_estonian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 199: {alias: []collalias{{0b01111111, "utf8_spanish_ci", "utf8"}, {0b01111111, "utf8mb3_spanish_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 200: {alias: []collalias{{0b01111111, "utf8_swedish_ci", "utf8"}, {0b01111111, "utf8mb3_swedish_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 201: {alias: []collalias{{0b01111111, "utf8_turkish_ci", "utf8"}, {0b01111111, "utf8mb3_turkish_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 202: {alias: []collalias{{0b01111111, "utf8_czech_ci", "utf8"}, {0b01111111, "utf8mb3_czech_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 203: {alias: []collalias{{0b01111111, "utf8_danish_ci", "utf8"}, {0b01111111, "utf8mb3_danish_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 204: {alias: []collalias{{0b01111111, "utf8_lithuanian_ci", "utf8"}, {0b01111111, "utf8mb3_lithuanian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 205: {alias: []collalias{{0b01111111, "utf8_slovak_ci", "utf8"}, {0b01111111, "utf8mb3_slovak_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 206: {alias: []collalias{{0b01111111, "utf8_spanish2_ci", "utf8"}, {0b01111111, "utf8mb3_spanish2_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 207: {alias: []collalias{{0b01111111, "utf8_roman_ci", "utf8"}, {0b01111111, "utf8mb3_roman_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 208: {alias: []collalias{{0b01111111, "utf8_persian_ci", "utf8"}, {0b01111111, "utf8mb3_persian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 209: {alias: []collalias{{0b01111111, "utf8_esperanto_ci", "utf8"}, {0b01111111, "utf8mb3_esperanto_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 210: {alias: []collalias{{0b01111111, "utf8_hungarian_ci", "utf8"}, {0b01111111, "utf8mb3_hungarian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 211: {alias: []collalias{{0b01111111, "utf8_sinhala_ci", "utf8"}, {0b01111111, "utf8mb3_sinhala_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 212: {alias: []collalias{{0b01111111, "utf8_german2_ci", "utf8"}, {0b01111111, "utf8mb3_german2_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 213: {alias: []collalias{{0b01110000, "utf8_croatian_ci", "utf8"}, {0b00001111, "utf8_croatian_mysql561_ci", "utf8"}, {0b01110000, "utf8mb3_croatian_ci", "utf8mb3"}, {0b00001111, "utf8mb3_croatian_mysql561_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 214: {alias: []collalias{{0b01111111, "utf8_unicode_520_ci", "utf8"}, {0b01111111, "utf8mb3_unicode_520_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 215: {alias: []collalias{{0b01111111, "utf8_vietnamese_ci", "utf8"}, {0b01111111, "utf8mb3_vietnamese_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 223: {alias: []collalias{{0b01111111, "utf8_general_mysql500_ci", "utf8"}, {0b01111111, "utf8mb3_general_mysql500_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 224: {alias: []collalias{{0b01111111, "utf8mb4_unicode_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 225: {alias: []collalias{{0b01111111, "utf8mb4_icelandic_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 226: {alias: []collalias{{0b01111111, "utf8mb4_latvian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 227: {alias: []collalias{{0b01111111, "utf8mb4_romanian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 228: {alias: []collalias{{0b01111111, "utf8mb4_slovenian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 229: {alias: []collalias{{0b01111111, "utf8mb4_polish_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 230: {alias: []collalias{{0b01111111, "utf8mb4_estonian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 231: {alias: []collalias{{0b01111111, "utf8mb4_spanish_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 232: {alias: []collalias{{0b01111111, "utf8mb4_swedish_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 233: {alias: []collalias{{0b01111111, "utf8mb4_turkish_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 234: {alias: []collalias{{0b01111111, "utf8mb4_czech_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 235: {alias: []collalias{{0b01111111, "utf8mb4_danish_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 236: {alias: []collalias{{0b01111111, "utf8mb4_lithuanian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 237: {alias: []collalias{{0b01111111, "utf8mb4_slovak_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 238: {alias: []collalias{{0b01111111, "utf8mb4_spanish2_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 239: {alias: []collalias{{0b01111111, "utf8mb4_roman_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 240: {alias: []collalias{{0b01111111, "utf8mb4_persian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 241: {alias: []collalias{{0b01111111, "utf8mb4_esperanto_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 242: {alias: []collalias{{0b01111111, "utf8mb4_hungarian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 243: {alias: []collalias{{0b01111111, "utf8mb4_sinhala_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 244: {alias: []collalias{{0b01111111, "utf8mb4_german2_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 245: {alias: []collalias{{0b01110000, "utf8mb4_croatian_ci", "utf8mb4"}, {0b00001111, "utf8mb4_croatian_mysql561_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 246: {alias: []collalias{{0b01111111, "utf8mb4_unicode_520_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 247: {alias: []collalias{{0b01111111, "utf8mb4_vietnamese_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 248: {alias: []collalias{{0b01100000, "gb18030_chinese_ci", "gb18030"}}, isdefault: 0b01100000}, + 249: {alias: []collalias{{0b01100000, "gb18030_bin", "gb18030"}}, isdefault: 0b00000000}, + 250: {alias: []collalias{{0b01100000, "gb18030_unicode_520_ci", "gb18030"}}, isdefault: 0b00000000}, + 255: {alias: []collalias{{0b01000000, "utf8mb4_0900_ai_ci", "utf8mb4"}}, isdefault: 0b01000000}, + 256: {alias: []collalias{{0b01000000, "utf8mb4_de_pb_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 257: {alias: []collalias{{0b01000000, "utf8mb4_is_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 258: {alias: []collalias{{0b01000000, "utf8mb4_lv_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 259: {alias: []collalias{{0b01000000, "utf8mb4_ro_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 260: {alias: []collalias{{0b01000000, "utf8mb4_sl_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 261: {alias: []collalias{{0b01000000, "utf8mb4_pl_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 262: {alias: []collalias{{0b01000000, "utf8mb4_et_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 263: {alias: []collalias{{0b01000000, "utf8mb4_es_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 264: {alias: []collalias{{0b01000000, "utf8mb4_sv_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 265: {alias: []collalias{{0b01000000, "utf8mb4_tr_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 266: {alias: []collalias{{0b01000000, "utf8mb4_cs_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 267: {alias: []collalias{{0b01000000, "utf8mb4_da_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 268: {alias: []collalias{{0b01000000, "utf8mb4_lt_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 269: {alias: []collalias{{0b01000000, "utf8mb4_sk_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 270: {alias: []collalias{{0b01000000, "utf8mb4_es_trad_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 271: {alias: []collalias{{0b01000000, "utf8mb4_la_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 273: {alias: []collalias{{0b01000000, "utf8mb4_eo_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 274: {alias: []collalias{{0b01000000, "utf8mb4_hu_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 275: {alias: []collalias{{0b01000000, "utf8mb4_hr_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 277: {alias: []collalias{{0b01000000, "utf8mb4_vi_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 278: {alias: []collalias{{0b01000000, "utf8mb4_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 279: {alias: []collalias{{0b01000000, "utf8mb4_de_pb_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 280: {alias: []collalias{{0b01000000, "utf8mb4_is_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 281: {alias: []collalias{{0b01000000, "utf8mb4_lv_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 282: {alias: []collalias{{0b01000000, "utf8mb4_ro_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 283: {alias: []collalias{{0b01000000, "utf8mb4_sl_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 284: {alias: []collalias{{0b01000000, "utf8mb4_pl_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 285: {alias: []collalias{{0b01000000, "utf8mb4_et_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 286: {alias: []collalias{{0b01000000, "utf8mb4_es_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 287: {alias: []collalias{{0b01000000, "utf8mb4_sv_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 288: {alias: []collalias{{0b01000000, "utf8mb4_tr_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 289: {alias: []collalias{{0b01000000, "utf8mb4_cs_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 290: {alias: []collalias{{0b01000000, "utf8mb4_da_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 291: {alias: []collalias{{0b01000000, "utf8mb4_lt_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 292: {alias: []collalias{{0b01000000, "utf8mb4_sk_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 293: {alias: []collalias{{0b01000000, "utf8mb4_es_trad_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 294: {alias: []collalias{{0b01000000, "utf8mb4_la_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 296: {alias: []collalias{{0b01000000, "utf8mb4_eo_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 297: {alias: []collalias{{0b01000000, "utf8mb4_hu_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 298: {alias: []collalias{{0b01000000, "utf8mb4_hr_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 300: {alias: []collalias{{0b01000000, "utf8mb4_vi_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 303: {alias: []collalias{{0b01000000, "utf8mb4_ja_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 304: {alias: []collalias{{0b01000000, "utf8mb4_ja_0900_as_cs_ks", "utf8mb4"}}, isdefault: 0b00000000}, + 305: {alias: []collalias{{0b01000000, "utf8mb4_0900_as_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 306: {alias: []collalias{{0b01000000, "utf8mb4_ru_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 307: {alias: []collalias{{0b01000000, "utf8mb4_ru_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 308: {alias: []collalias{{0b01000000, "utf8mb4_zh_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 309: {alias: []collalias{{0b01000000, "utf8mb4_0900_bin", "utf8mb4"}}, isdefault: 0b00000000}, + 310: {alias: []collalias{{0b01000000, "utf8mb4_nb_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 311: {alias: []collalias{{0b01000000, "utf8mb4_nb_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 312: {alias: []collalias{{0b01000000, "utf8mb4_nn_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 313: {alias: []collalias{{0b01000000, "utf8mb4_nn_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 314: {alias: []collalias{{0b01000000, "utf8mb4_sr_latn_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 315: {alias: []collalias{{0b01000000, "utf8mb4_sr_latn_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 316: {alias: []collalias{{0b01000000, "utf8mb4_bs_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 317: {alias: []collalias{{0b01000000, "utf8mb4_bs_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 318: {alias: []collalias{{0b01000000, "utf8mb4_bg_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 319: {alias: []collalias{{0b01000000, "utf8mb4_bg_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 320: {alias: []collalias{{0b01000000, "utf8mb4_gl_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 321: {alias: []collalias{{0b01000000, "utf8mb4_gl_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 322: {alias: []collalias{{0b01000000, "utf8mb4_mn_cyrl_0900_ai_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 323: {alias: []collalias{{0b01000000, "utf8mb4_mn_cyrl_0900_as_cs", "utf8mb4"}}, isdefault: 0b00000000}, + 576: {alias: []collalias{{0b00001111, "utf8_croatian_ci", "utf8"}, {0b00001111, "utf8mb3_croatian_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 577: {alias: []collalias{{0b00001111, "utf8_myanmar_ci", "utf8"}, {0b00001111, "utf8mb3_myanmar_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 578: {alias: []collalias{{0b00001110, "utf8_thai_520_w2", "utf8"}, {0b00001110, "utf8mb3_thai_520_w2", "utf8mb3"}}, isdefault: 0b00000000}, + 608: {alias: []collalias{{0b00001111, "utf8mb4_croatian_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 609: {alias: []collalias{{0b00001111, "utf8mb4_myanmar_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 610: {alias: []collalias{{0b00001110, "utf8mb4_thai_520_w2", "utf8mb4"}}, isdefault: 0b00000000}, + 640: {alias: []collalias{{0b00001111, "ucs2_croatian_ci", "ucs2"}}, isdefault: 0b00000000}, + 641: {alias: []collalias{{0b00001111, "ucs2_myanmar_ci", "ucs2"}}, isdefault: 0b00000000}, + 642: {alias: []collalias{{0b00001110, "ucs2_thai_520_w2", "ucs2"}}, isdefault: 0b00000000}, + 672: {alias: []collalias{{0b00001111, "utf16_croatian_ci", "utf16"}}, isdefault: 0b00000000}, + 673: {alias: []collalias{{0b00001111, "utf16_myanmar_ci", "utf16"}}, isdefault: 0b00000000}, + 674: {alias: []collalias{{0b00001110, "utf16_thai_520_w2", "utf16"}}, isdefault: 0b00000000}, + 736: {alias: []collalias{{0b00001111, "utf32_croatian_ci", "utf32"}}, isdefault: 0b00000000}, + 737: {alias: []collalias{{0b00001111, "utf32_myanmar_ci", "utf32"}}, isdefault: 0b00000000}, + 738: {alias: []collalias{{0b00001110, "utf32_thai_520_w2", "utf32"}}, isdefault: 0b00000000}, + 1025: {alias: []collalias{{0b00001100, "big5_chinese_nopad_ci", "big5"}}, isdefault: 0b00000000}, + 1027: {alias: []collalias{{0b00001100, "dec8_swedish_nopad_ci", "dec8"}}, isdefault: 0b00000000}, + 1028: {alias: []collalias{{0b00001100, "cp850_general_nopad_ci", "cp850"}}, isdefault: 0b00000000}, + 1030: {alias: []collalias{{0b00001100, "hp8_english_nopad_ci", "hp8"}}, isdefault: 0b00000000}, + 1031: {alias: []collalias{{0b00001100, "koi8r_general_nopad_ci", "koi8r"}}, isdefault: 0b00000000}, + 1032: {alias: []collalias{{0b00001100, "latin1_swedish_nopad_ci", "latin1"}}, isdefault: 0b00000000}, + 1033: {alias: []collalias{{0b00001100, "latin2_general_nopad_ci", "latin2"}}, isdefault: 0b00000000}, + 1034: {alias: []collalias{{0b00001100, "swe7_swedish_nopad_ci", "swe7"}}, isdefault: 0b00000000}, + 1035: {alias: []collalias{{0b00001100, "ascii_general_nopad_ci", "ascii"}}, isdefault: 0b00000000}, + 1036: {alias: []collalias{{0b00001100, "ujis_japanese_nopad_ci", "ujis"}}, isdefault: 0b00000000}, + 1037: {alias: []collalias{{0b00001100, "sjis_japanese_nopad_ci", "sjis"}}, isdefault: 0b00000000}, + 1040: {alias: []collalias{{0b00001100, "hebrew_general_nopad_ci", "hebrew"}}, isdefault: 0b00000000}, + 1042: {alias: []collalias{{0b00001100, "tis620_thai_nopad_ci", "tis620"}}, isdefault: 0b00000000}, + 1043: {alias: []collalias{{0b00001100, "euckr_korean_nopad_ci", "euckr"}}, isdefault: 0b00000000}, + 1046: {alias: []collalias{{0b00001100, "koi8u_general_nopad_ci", "koi8u"}}, isdefault: 0b00000000}, + 1048: {alias: []collalias{{0b00001100, "gb2312_chinese_nopad_ci", "gb2312"}}, isdefault: 0b00000000}, + 1049: {alias: []collalias{{0b00001100, "greek_general_nopad_ci", "greek"}}, isdefault: 0b00000000}, + 1050: {alias: []collalias{{0b00001100, "cp1250_general_nopad_ci", "cp1250"}}, isdefault: 0b00000000}, + 1052: {alias: []collalias{{0b00001100, "gbk_chinese_nopad_ci", "gbk"}}, isdefault: 0b00000000}, + 1054: {alias: []collalias{{0b00001100, "latin5_turkish_nopad_ci", "latin5"}}, isdefault: 0b00000000}, + 1056: {alias: []collalias{{0b00001100, "armscii8_general_nopad_ci", "armscii8"}}, isdefault: 0b00000000}, + 1057: {alias: []collalias{{0b00001100, "utf8_general_nopad_ci", "utf8"}, {0b00001100, "utf8mb3_general_nopad_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 1059: {alias: []collalias{{0b00001100, "ucs2_general_nopad_ci", "ucs2"}}, isdefault: 0b00000000}, + 1060: {alias: []collalias{{0b00001100, "cp866_general_nopad_ci", "cp866"}}, isdefault: 0b00000000}, + 1061: {alias: []collalias{{0b00001100, "keybcs2_general_nopad_ci", "keybcs2"}}, isdefault: 0b00000000}, + 1062: {alias: []collalias{{0b00001100, "macce_general_nopad_ci", "macce"}}, isdefault: 0b00000000}, + 1063: {alias: []collalias{{0b00001100, "macroman_general_nopad_ci", "macroman"}}, isdefault: 0b00000000}, + 1064: {alias: []collalias{{0b00001100, "cp852_general_nopad_ci", "cp852"}}, isdefault: 0b00000000}, + 1065: {alias: []collalias{{0b00001100, "latin7_general_nopad_ci", "latin7"}}, isdefault: 0b00000000}, + 1067: {alias: []collalias{{0b00001100, "macce_nopad_bin", "macce"}}, isdefault: 0b00000000}, + 1069: {alias: []collalias{{0b00001100, "utf8mb4_general_nopad_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 1070: {alias: []collalias{{0b00001100, "utf8mb4_nopad_bin", "utf8mb4"}}, isdefault: 0b00000000}, + 1071: {alias: []collalias{{0b00001100, "latin1_nopad_bin", "latin1"}}, isdefault: 0b00000000}, + 1074: {alias: []collalias{{0b00001100, "cp1251_nopad_bin", "cp1251"}}, isdefault: 0b00000000}, + 1075: {alias: []collalias{{0b00001100, "cp1251_general_nopad_ci", "cp1251"}}, isdefault: 0b00000000}, + 1077: {alias: []collalias{{0b00001100, "macroman_nopad_bin", "macroman"}}, isdefault: 0b00000000}, + 1078: {alias: []collalias{{0b00001100, "utf16_general_nopad_ci", "utf16"}}, isdefault: 0b00000000}, + 1079: {alias: []collalias{{0b00001100, "utf16_nopad_bin", "utf16"}}, isdefault: 0b00000000}, + 1080: {alias: []collalias{{0b00001100, "utf16le_general_nopad_ci", "utf16le"}}, isdefault: 0b00000000}, + 1081: {alias: []collalias{{0b00001100, "cp1256_general_nopad_ci", "cp1256"}}, isdefault: 0b00000000}, + 1082: {alias: []collalias{{0b00001100, "cp1257_nopad_bin", "cp1257"}}, isdefault: 0b00000000}, + 1083: {alias: []collalias{{0b00001100, "cp1257_general_nopad_ci", "cp1257"}}, isdefault: 0b00000000}, + 1084: {alias: []collalias{{0b00001100, "utf32_general_nopad_ci", "utf32"}}, isdefault: 0b00000000}, + 1085: {alias: []collalias{{0b00001100, "utf32_nopad_bin", "utf32"}}, isdefault: 0b00000000}, + 1086: {alias: []collalias{{0b00001100, "utf16le_nopad_bin", "utf16le"}}, isdefault: 0b00000000}, + 1088: {alias: []collalias{{0b00001100, "armscii8_nopad_bin", "armscii8"}}, isdefault: 0b00000000}, + 1089: {alias: []collalias{{0b00001100, "ascii_nopad_bin", "ascii"}}, isdefault: 0b00000000}, + 1090: {alias: []collalias{{0b00001100, "cp1250_nopad_bin", "cp1250"}}, isdefault: 0b00000000}, + 1091: {alias: []collalias{{0b00001100, "cp1256_nopad_bin", "cp1256"}}, isdefault: 0b00000000}, + 1092: {alias: []collalias{{0b00001100, "cp866_nopad_bin", "cp866"}}, isdefault: 0b00000000}, + 1093: {alias: []collalias{{0b00001100, "dec8_nopad_bin", "dec8"}}, isdefault: 0b00000000}, + 1094: {alias: []collalias{{0b00001100, "greek_nopad_bin", "greek"}}, isdefault: 0b00000000}, + 1095: {alias: []collalias{{0b00001100, "hebrew_nopad_bin", "hebrew"}}, isdefault: 0b00000000}, + 1096: {alias: []collalias{{0b00001100, "hp8_nopad_bin", "hp8"}}, isdefault: 0b00000000}, + 1097: {alias: []collalias{{0b00001100, "keybcs2_nopad_bin", "keybcs2"}}, isdefault: 0b00000000}, + 1098: {alias: []collalias{{0b00001100, "koi8r_nopad_bin", "koi8r"}}, isdefault: 0b00000000}, + 1099: {alias: []collalias{{0b00001100, "koi8u_nopad_bin", "koi8u"}}, isdefault: 0b00000000}, + 1101: {alias: []collalias{{0b00001100, "latin2_nopad_bin", "latin2"}}, isdefault: 0b00000000}, + 1102: {alias: []collalias{{0b00001100, "latin5_nopad_bin", "latin5"}}, isdefault: 0b00000000}, + 1103: {alias: []collalias{{0b00001100, "latin7_nopad_bin", "latin7"}}, isdefault: 0b00000000}, + 1104: {alias: []collalias{{0b00001100, "cp850_nopad_bin", "cp850"}}, isdefault: 0b00000000}, + 1105: {alias: []collalias{{0b00001100, "cp852_nopad_bin", "cp852"}}, isdefault: 0b00000000}, + 1106: {alias: []collalias{{0b00001100, "swe7_nopad_bin", "swe7"}}, isdefault: 0b00000000}, + 1107: {alias: []collalias{{0b00001100, "utf8_nopad_bin", "utf8"}, {0b00001100, "utf8mb3_nopad_bin", "utf8mb3"}}, isdefault: 0b00000000}, + 1108: {alias: []collalias{{0b00001100, "big5_nopad_bin", "big5"}}, isdefault: 0b00000000}, + 1109: {alias: []collalias{{0b00001100, "euckr_nopad_bin", "euckr"}}, isdefault: 0b00000000}, + 1110: {alias: []collalias{{0b00001100, "gb2312_nopad_bin", "gb2312"}}, isdefault: 0b00000000}, + 1111: {alias: []collalias{{0b00001100, "gbk_nopad_bin", "gbk"}}, isdefault: 0b00000000}, + 1112: {alias: []collalias{{0b00001100, "sjis_nopad_bin", "sjis"}}, isdefault: 0b00000000}, + 1113: {alias: []collalias{{0b00001100, "tis620_nopad_bin", "tis620"}}, isdefault: 0b00000000}, + 1114: {alias: []collalias{{0b00001100, "ucs2_nopad_bin", "ucs2"}}, isdefault: 0b00000000}, + 1115: {alias: []collalias{{0b00001100, "ujis_nopad_bin", "ujis"}}, isdefault: 0b00000000}, + 1116: {alias: []collalias{{0b00001100, "geostd8_general_nopad_ci", "geostd8"}}, isdefault: 0b00000000}, + 1117: {alias: []collalias{{0b00001100, "geostd8_nopad_bin", "geostd8"}}, isdefault: 0b00000000}, + 1119: {alias: []collalias{{0b00001100, "cp932_japanese_nopad_ci", "cp932"}}, isdefault: 0b00000000}, + 1120: {alias: []collalias{{0b00001100, "cp932_nopad_bin", "cp932"}}, isdefault: 0b00000000}, + 1121: {alias: []collalias{{0b00001100, "eucjpms_japanese_nopad_ci", "eucjpms"}}, isdefault: 0b00000000}, + 1122: {alias: []collalias{{0b00001100, "eucjpms_nopad_bin", "eucjpms"}}, isdefault: 0b00000000}, + 1125: {alias: []collalias{{0b00001100, "utf16_unicode_nopad_ci", "utf16"}}, isdefault: 0b00000000}, + 1147: {alias: []collalias{{0b00001100, "utf16_unicode_520_nopad_ci", "utf16"}}, isdefault: 0b00000000}, + 1152: {alias: []collalias{{0b00001100, "ucs2_unicode_nopad_ci", "ucs2"}}, isdefault: 0b00000000}, + 1174: {alias: []collalias{{0b00001100, "ucs2_unicode_520_nopad_ci", "ucs2"}}, isdefault: 0b00000000}, + 1184: {alias: []collalias{{0b00001100, "utf32_unicode_nopad_ci", "utf32"}}, isdefault: 0b00000000}, + 1206: {alias: []collalias{{0b00001100, "utf32_unicode_520_nopad_ci", "utf32"}}, isdefault: 0b00000000}, + 1216: {alias: []collalias{{0b00001100, "utf8_unicode_nopad_ci", "utf8"}, {0b00001100, "utf8mb3_unicode_nopad_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 1238: {alias: []collalias{{0b00001100, "utf8_unicode_520_nopad_ci", "utf8"}, {0b00001100, "utf8mb3_unicode_520_nopad_ci", "utf8mb3"}}, isdefault: 0b00000000}, + 1248: {alias: []collalias{{0b00001100, "utf8mb4_unicode_nopad_ci", "utf8mb4"}}, isdefault: 0b00000000}, + 1270: {alias: []collalias{{0b00001100, "utf8mb4_unicode_520_nopad_ci", "utf8mb4"}}, isdefault: 0b00000000}, } diff --git a/go/mysql/collations/remote/collation.go b/go/mysql/collations/remote/collation.go index 1e81c429794..dcc2acfee61 100644 --- a/go/mysql/collations/remote/collation.go +++ b/go/mysql/collations/remote/collation.go @@ -28,6 +28,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vthash" ) @@ -53,22 +54,22 @@ type Collation struct { err error } -var _ collations.Collation = (*Collation)(nil) +var _ colldata.Collation = (*Collation)(nil) func makeRemoteCollation(conn *mysql.Conn, collid collations.ID, collname string) *Collation { - charset := collname + cs := collname if idx := strings.IndexByte(collname, '_'); idx >= 0 { - charset = collname[:idx] + cs = collname[:idx] } coll := &Collation{ name: collname, id: collid, conn: conn, - charset: charset, + charset: cs, } - coll.prefix = fmt.Sprintf("_%s X'", charset) + coll.prefix = fmt.Sprintf("_%s X'", cs) coll.suffix = fmt.Sprintf("' COLLATE %q", collname) coll.hex = hex.NewEncoder(&coll.sql) return coll @@ -204,7 +205,7 @@ func (rp *remotePattern) Match(in []byte) bool { return match } -func (c *Collation) Wildcard(pat []byte, _ rune, _ rune, escape rune) collations.WildcardPattern { +func (c *Collation) Wildcard(pat []byte, _ rune, _ rune, escape rune) colldata.WildcardPattern { return &remotePattern{ pattern: fmt.Sprintf("_%s X'%x'", c.charset, pat), remote: c, diff --git a/go/mysql/collations/supported.go b/go/mysql/collations/supported.go new file mode 100644 index 00000000000..4404af2d4fb --- /dev/null +++ b/go/mysql/collations/supported.go @@ -0,0 +1,294 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by makecolldata DO NOT EDIT + +package collations + +var supported = [...]string{ + 0x3: "dec8_swedish_ci", + 0x4: "cp850_general_ci", + 0x5: "latin1_german1_ci", + 0x6: "hp8_english_ci", + 0x7: "koi8r_general_ci", + 0x8: "latin1_swedish_ci", + 0x9: "latin2_general_ci", + 0xa: "swe7_swedish_ci", + 0xb: "ascii_general_ci", + 0xc: "ujis_japanese_ci", + 0xd: "sjis_japanese_ci", + 0xe: "cp1251_bulgarian_ci", + 0xf: "latin1_danish_ci", + 0x10: "hebrew_general_ci", + 0x13: "euckr_korean_ci", + 0x14: "latin7_estonian_cs", + 0x15: "latin2_hungarian_ci", + 0x16: "koi8u_general_ci", + 0x17: "cp1251_ukrainian_ci", + 0x18: "gb2312_chinese_ci", + 0x19: "greek_general_ci", + 0x1a: "cp1250_general_ci", + 0x1b: "latin2_croatian_ci", + 0x1d: "cp1257_lithuanian_ci", + 0x1e: "latin5_turkish_ci", + 0x20: "armscii8_general_ci", + 0x21: "utf8mb3_general_ci", + 0x23: "ucs2_general_ci", + 0x24: "cp866_general_ci", + 0x25: "keybcs2_general_ci", + 0x26: "macce_general_ci", + 0x27: "macroman_general_ci", + 0x28: "cp852_general_ci", + 0x29: "latin7_general_ci", + 0x2a: "latin7_general_cs", + 0x2b: "macce_bin", + 0x2c: "cp1250_croatian_ci", + 0x2d: "utf8mb4_general_ci", + 0x2e: "utf8mb4_bin", + 0x2f: "latin1_bin", + 0x30: "latin1_general_ci", + 0x31: "latin1_general_cs", + 0x32: "cp1251_bin", + 0x33: "cp1251_general_ci", + 0x34: "cp1251_general_cs", + 0x35: "macroman_bin", + 0x36: "utf16_general_ci", + 0x37: "utf16_bin", + 0x38: "utf16le_general_ci", + 0x39: "cp1256_general_ci", + 0x3a: "cp1257_bin", + 0x3b: "cp1257_general_ci", + 0x3c: "utf32_general_ci", + 0x3d: "utf32_bin", + 0x3e: "utf16le_bin", + 0x3f: "binary", + 0x40: "armscii8_bin", + 0x41: "ascii_bin", + 0x42: "cp1250_bin", + 0x43: "cp1256_bin", + 0x44: "cp866_bin", + 0x45: "dec8_bin", + 0x46: "greek_bin", + 0x47: "hebrew_bin", + 0x48: "hp8_bin", + 0x49: "keybcs2_bin", + 0x4a: "koi8r_bin", + 0x4b: "koi8u_bin", + 0x4d: "latin2_bin", + 0x4e: "latin5_bin", + 0x4f: "latin7_bin", + 0x50: "cp850_bin", + 0x51: "cp852_bin", + 0x52: "swe7_bin", + 0x53: "utf8mb3_bin", + 0x55: "euckr_bin", + 0x56: "gb2312_bin", + 0x58: "sjis_bin", + 0x5a: "ucs2_bin", + 0x5b: "ujis_bin", + 0x5c: "geostd8_general_ci", + 0x5d: "geostd8_bin", + 0x5e: "latin1_spanish_ci", + 0x5f: "cp932_japanese_ci", + 0x60: "cp932_bin", + 0x61: "eucjpms_japanese_ci", + 0x62: "eucjpms_bin", + 0x63: "cp1250_polish_ci", + 0x65: "utf16_unicode_ci", + 0x66: "utf16_icelandic_ci", + 0x67: "utf16_latvian_ci", + 0x68: "utf16_romanian_ci", + 0x69: "utf16_slovenian_ci", + 0x6a: "utf16_polish_ci", + 0x6b: "utf16_estonian_ci", + 0x6c: "utf16_spanish_ci", + 0x6d: "utf16_swedish_ci", + 0x6e: "utf16_turkish_ci", + 0x6f: "utf16_czech_ci", + 0x70: "utf16_danish_ci", + 0x71: "utf16_lithuanian_ci", + 0x72: "utf16_slovak_ci", + 0x73: "utf16_spanish2_ci", + 0x74: "utf16_roman_ci", + 0x75: "utf16_persian_ci", + 0x76: "utf16_esperanto_ci", + 0x77: "utf16_hungarian_ci", + 0x78: "utf16_sinhala_ci", + 0x79: "utf16_german2_ci", + 0x7a: "utf16_croatian_ci", + 0x7b: "utf16_unicode_520_ci", + 0x7c: "utf16_vietnamese_ci", + 0x80: "ucs2_unicode_ci", + 0x81: "ucs2_icelandic_ci", + 0x82: "ucs2_latvian_ci", + 0x83: "ucs2_romanian_ci", + 0x84: "ucs2_slovenian_ci", + 0x85: "ucs2_polish_ci", + 0x86: "ucs2_estonian_ci", + 0x87: "ucs2_spanish_ci", + 0x88: "ucs2_swedish_ci", + 0x89: "ucs2_turkish_ci", + 0x8a: "ucs2_czech_ci", + 0x8b: "ucs2_danish_ci", + 0x8c: "ucs2_lithuanian_ci", + 0x8d: "ucs2_slovak_ci", + 0x8e: "ucs2_spanish2_ci", + 0x8f: "ucs2_roman_ci", + 0x90: "ucs2_persian_ci", + 0x91: "ucs2_esperanto_ci", + 0x92: "ucs2_hungarian_ci", + 0x93: "ucs2_sinhala_ci", + 0x94: "ucs2_german2_ci", + 0x95: "ucs2_croatian_ci", + 0x96: "ucs2_unicode_520_ci", + 0x97: "ucs2_vietnamese_ci", + 0xa0: "utf32_unicode_ci", + 0xa1: "utf32_icelandic_ci", + 0xa2: "utf32_latvian_ci", + 0xa3: "utf32_romanian_ci", + 0xa4: "utf32_slovenian_ci", + 0xa5: "utf32_polish_ci", + 0xa6: "utf32_estonian_ci", + 0xa7: "utf32_spanish_ci", + 0xa8: "utf32_swedish_ci", + 0xa9: "utf32_turkish_ci", + 0xaa: "utf32_czech_ci", + 0xab: "utf32_danish_ci", + 0xac: "utf32_lithuanian_ci", + 0xad: "utf32_slovak_ci", + 0xae: "utf32_spanish2_ci", + 0xaf: "utf32_roman_ci", + 0xb0: "utf32_persian_ci", + 0xb1: "utf32_esperanto_ci", + 0xb2: "utf32_hungarian_ci", + 0xb3: "utf32_sinhala_ci", + 0xb4: "utf32_german2_ci", + 0xb5: "utf32_croatian_ci", + 0xb6: "utf32_unicode_520_ci", + 0xb7: "utf32_vietnamese_ci", + 0xc0: "utf8mb3_unicode_ci", + 0xc1: "utf8mb3_icelandic_ci", + 0xc2: "utf8mb3_latvian_ci", + 0xc3: "utf8mb3_romanian_ci", + 0xc4: "utf8mb3_slovenian_ci", + 0xc5: "utf8mb3_polish_ci", + 0xc6: "utf8mb3_estonian_ci", + 0xc7: "utf8mb3_spanish_ci", + 0xc8: "utf8mb3_swedish_ci", + 0xc9: "utf8mb3_turkish_ci", + 0xca: "utf8mb3_czech_ci", + 0xcb: "utf8mb3_danish_ci", + 0xcc: "utf8mb3_lithuanian_ci", + 0xcd: "utf8mb3_slovak_ci", + 0xce: "utf8mb3_spanish2_ci", + 0xcf: "utf8mb3_roman_ci", + 0xd0: "utf8mb3_persian_ci", + 0xd1: "utf8mb3_esperanto_ci", + 0xd2: "utf8mb3_hungarian_ci", + 0xd3: "utf8mb3_sinhala_ci", + 0xd4: "utf8mb3_german2_ci", + 0xd5: "utf8mb3_croatian_ci", + 0xd6: "utf8mb3_unicode_520_ci", + 0xd7: "utf8mb3_vietnamese_ci", + 0xe0: "utf8mb4_unicode_ci", + 0xe1: "utf8mb4_icelandic_ci", + 0xe2: "utf8mb4_latvian_ci", + 0xe3: "utf8mb4_romanian_ci", + 0xe4: "utf8mb4_slovenian_ci", + 0xe5: "utf8mb4_polish_ci", + 0xe6: "utf8mb4_estonian_ci", + 0xe7: "utf8mb4_spanish_ci", + 0xe8: "utf8mb4_swedish_ci", + 0xe9: "utf8mb4_turkish_ci", + 0xea: "utf8mb4_czech_ci", + 0xeb: "utf8mb4_danish_ci", + 0xec: "utf8mb4_lithuanian_ci", + 0xed: "utf8mb4_slovak_ci", + 0xee: "utf8mb4_spanish2_ci", + 0xef: "utf8mb4_roman_ci", + 0xf0: "utf8mb4_persian_ci", + 0xf1: "utf8mb4_esperanto_ci", + 0xf2: "utf8mb4_hungarian_ci", + 0xf3: "utf8mb4_sinhala_ci", + 0xf4: "utf8mb4_german2_ci", + 0xf5: "utf8mb4_croatian_ci", + 0xf6: "utf8mb4_unicode_520_ci", + 0xf7: "utf8mb4_vietnamese_ci", + 0xfa: "gb18030_unicode_520_ci", + 0xff: "utf8mb4_0900_ai_ci", + 0x100: "utf8mb4_de_pb_0900_ai_ci", + 0x101: "utf8mb4_is_0900_ai_ci", + 0x102: "utf8mb4_lv_0900_ai_ci", + 0x103: "utf8mb4_ro_0900_ai_ci", + 0x104: "utf8mb4_sl_0900_ai_ci", + 0x105: "utf8mb4_pl_0900_ai_ci", + 0x106: "utf8mb4_et_0900_ai_ci", + 0x107: "utf8mb4_es_0900_ai_ci", + 0x108: "utf8mb4_sv_0900_ai_ci", + 0x109: "utf8mb4_tr_0900_ai_ci", + 0x10a: "utf8mb4_cs_0900_ai_ci", + 0x10b: "utf8mb4_da_0900_ai_ci", + 0x10c: "utf8mb4_lt_0900_ai_ci", + 0x10d: "utf8mb4_sk_0900_ai_ci", + 0x10e: "utf8mb4_es_trad_0900_ai_ci", + 0x10f: "utf8mb4_la_0900_ai_ci", + 0x111: "utf8mb4_eo_0900_ai_ci", + 0x112: "utf8mb4_hu_0900_ai_ci", + 0x113: "utf8mb4_hr_0900_ai_ci", + 0x115: "utf8mb4_vi_0900_ai_ci", + 0x116: "utf8mb4_0900_as_cs", + 0x117: "utf8mb4_de_pb_0900_as_cs", + 0x118: "utf8mb4_is_0900_as_cs", + 0x119: "utf8mb4_lv_0900_as_cs", + 0x11a: "utf8mb4_ro_0900_as_cs", + 0x11b: "utf8mb4_sl_0900_as_cs", + 0x11c: "utf8mb4_pl_0900_as_cs", + 0x11d: "utf8mb4_et_0900_as_cs", + 0x11e: "utf8mb4_es_0900_as_cs", + 0x11f: "utf8mb4_sv_0900_as_cs", + 0x120: "utf8mb4_tr_0900_as_cs", + 0x121: "utf8mb4_cs_0900_as_cs", + 0x122: "utf8mb4_da_0900_as_cs", + 0x123: "utf8mb4_lt_0900_as_cs", + 0x124: "utf8mb4_sk_0900_as_cs", + 0x125: "utf8mb4_es_trad_0900_as_cs", + 0x126: "utf8mb4_la_0900_as_cs", + 0x128: "utf8mb4_eo_0900_as_cs", + 0x129: "utf8mb4_hu_0900_as_cs", + 0x12a: "utf8mb4_hr_0900_as_cs", + 0x12c: "utf8mb4_vi_0900_as_cs", + 0x12f: "utf8mb4_ja_0900_as_cs", + 0x130: "utf8mb4_ja_0900_as_cs_ks", + 0x131: "utf8mb4_0900_as_ci", + 0x132: "utf8mb4_ru_0900_ai_ci", + 0x133: "utf8mb4_ru_0900_as_cs", + 0x134: "utf8mb4_zh_0900_as_cs", + 0x135: "utf8mb4_0900_bin", + 0x136: "utf8mb4_nb_0900_ai_ci", + 0x137: "utf8mb4_nb_0900_as_cs", + 0x138: "utf8mb4_nn_0900_ai_ci", + 0x139: "utf8mb4_nn_0900_as_cs", + 0x13a: "utf8mb4_sr_latn_0900_ai_ci", + 0x13b: "utf8mb4_sr_latn_0900_as_cs", + 0x13c: "utf8mb4_bs_0900_ai_ci", + 0x13d: "utf8mb4_bs_0900_as_cs", + 0x13e: "utf8mb4_bg_0900_ai_ci", + 0x13f: "utf8mb4_bg_0900_as_cs", + 0x140: "utf8mb4_gl_0900_ai_ci", + 0x141: "utf8mb4_gl_0900_as_cs", + 0x142: "utf8mb4_mn_cyrl_0900_ai_ci", + 0x143: "utf8mb4_mn_cyrl_0900_as_cs", +} diff --git a/go/mysql/collations/testdata/versions/collations_MySQL80.csv b/go/mysql/collations/testdata/versions/collations_MySQL8.csv similarity index 100% rename from go/mysql/collations/testdata/versions/collations_MySQL80.csv rename to go/mysql/collations/testdata/versions/collations_MySQL8.csv diff --git a/go/mysql/collations/tools/makecolldata/codegen/codegen.go b/go/mysql/collations/tools/makecolldata/codegen/codegen.go index cc2d5ad3a90..4fa98f2afd1 100644 --- a/go/mysql/collations/tools/makecolldata/codegen/codegen.go +++ b/go/mysql/collations/tools/makecolldata/codegen/codegen.go @@ -24,6 +24,7 @@ import ( "path" "reflect" "sort" + "time" "vitess.io/vitess/go/tools/codegen" ) @@ -64,10 +65,29 @@ func Merge(gens ...*Generator) *Generator { return result } +const licenseFileHeader = `/* +Copyright %d The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +` + func (g *Generator) WriteToFile(out string) { var file, fmtfile bytes.Buffer file.Grow(g.Buffer.Len() + 1024) + fmt.Fprintf(&file, licenseFileHeader, time.Now().Year()) fmt.Fprintf(&file, "// Code generated by %s DO NOT EDIT\n\n", path.Base(os.Args[0])) fmt.Fprintf(&file, "package %s\n\n", g.local.Name()) fmt.Fprintf(&file, "import (\n") diff --git a/go/mysql/collations/tools/makecolldata/main.go b/go/mysql/collations/tools/makecolldata/main.go index 0bcbd1ecb2b..ee559a886b5 100644 --- a/go/mysql/collations/tools/makecolldata/main.go +++ b/go/mysql/collations/tools/makecolldata/main.go @@ -106,7 +106,7 @@ func (all AllMetadata) get(name string) *CollationMetadata { return nil } -const PkgCollations codegen.Package = "vitess.io/vitess/go/mysql/collations" +const PkgCollationsData codegen.Package = "vitess.io/vitess/go/mysql/collations/colldata" const PkgCharset codegen.Package = "vitess.io/vitess/go/mysql/collations/charset" func main() { @@ -114,5 +114,5 @@ func main() { metadata := loadMysqlMetadata() maketables(*Embed, ".", metadata) makeversions(".") - makemysqldata(".", metadata) + makemysqldata("colldata", ".", metadata) } diff --git a/go/mysql/collations/tools/makecolldata/maketables.go b/go/mysql/collations/tools/makecolldata/maketables.go index 8ac2f9049ce..055162401bb 100644 --- a/go/mysql/collations/tools/makecolldata/maketables.go +++ b/go/mysql/collations/tools/makecolldata/maketables.go @@ -39,7 +39,7 @@ func maketable(g *codegen.Generator, table string, collation *CollationMetadata, func maketables(embed bool, output string, metadata AllMetadata) { var pages = codegen.NewPageGenerator(embed) - var g = codegen.NewGenerator("vitess.io/vitess/go/mysql/collations") + var g = codegen.NewGenerator("vitess.io/vitess/go/mysql/collations/colldata") var fastg = codegen.NewGenerator("vitess.io/vitess/go/mysql/collations/internal/uca") tablegen := maketable(g, "uca900", metadata.get("utf8mb4_0900_ai_ci"), pages, uca.Layout_uca900{}) @@ -53,9 +53,9 @@ func maketables(embed bool, output string, metadata AllMetadata) { if pages, ok := pages.(*codegen.EmbedPageGenerator); ok { pages.WriteTrailer(g, "mysqlucadata.bin") - pages.WriteToFile(path.Join(output, "mysqlucadata.bin")) + pages.WriteToFile(path.Join(output, "colldata/mysqlucadata.bin")) } - g.WriteToFile(path.Join(output, "mysqlucadata.go")) + g.WriteToFile(path.Join(output, "colldata/mysqlucadata.go")) fastg.WriteToFile(path.Join(output, "internal/uca/fasttables.go")) } diff --git a/go/mysql/collations/tools/makecolldata/mysqldata.go b/go/mysql/collations/tools/makecolldata/mysqldata.go index 351e578d2af..567f04362de 100644 --- a/go/mysql/collations/tools/makecolldata/mysqldata.go +++ b/go/mysql/collations/tools/makecolldata/mysqldata.go @@ -353,12 +353,12 @@ func (g *Generator) printCollationMultibyte(meta *CollationMetadata) { g.P("},") } -func makemysqldata(output string, metadata AllMetadata) { +func makemysqldata(output string, supportedOutput string, metadata AllMetadata) { var unsupportedByCharset = make(map[string][]string) var g = Generator{ - Generator: codegen.NewGenerator(PkgCollations), + Generator: codegen.NewGenerator(PkgCollationsData), Tables: TableGenerator{ - Generator: codegen.NewGenerator(PkgCollations), + Generator: codegen.NewGenerator(PkgCollationsData), dedup: make(map[string]string), baseWeightsUca400: metadata.get("utf8mb4_unicode_ci").Weights, baseWeightsUca520: metadata.get("utf8mb4_unicode_520_ci").Weights, @@ -366,15 +366,22 @@ func makemysqldata(output string, metadata AllMetadata) { }, } + var h = Generator{ + Generator: codegen.NewGenerator("vitess.io/vitess/go/mysql/collations"), + } + g.P("var collationsById = [...]Collation{") + h.P("var supported = [...]string{") for _, meta := range metadata { switch { case meta.Name == "utf8mb4_0900_bin": g.P(uint(309), ": &Collation_utf8mb4_0900_bin{},") + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case meta.Name == "binary": g.P(uint(63), ": &Collation_binary{},") + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case meta.Name == "tis620_bin": // explicitly unsupported for now because of not accurate results @@ -384,24 +391,31 @@ func makemysqldata(output string, metadata AllMetadata) { meta.CollationImpl == "utf32_uca" || meta.CollationImpl == "ucs2_uca": g.printCollationUcaLegacy(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case meta.CollationImpl == "uca_900": g.printCollationUca900(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case meta.CollationImpl == "8bit_bin" || meta.CollationImpl == "8bit_simple_ci": g.printCollation8bit(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case meta.Name == "gb18030_unicode_520_ci": g.printCollationUcaLegacy(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case charset.IsMultibyteByName(meta.Charset): g.printCollationMultibyte(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case strings.HasSuffix(meta.Name, "_bin") && charset.IsUnicodeByName(meta.Charset): g.printCollationUnicode(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") case strings.HasSuffix(meta.Name, "_general_ci"): g.printCollationUnicode(meta) + h.P(meta.Number, ": ", codegen.Quote(meta.Name), ",") default: unsupportedByCharset[meta.Charset] = append(unsupportedByCharset[meta.Charset], meta.Name) @@ -409,7 +423,9 @@ func makemysqldata(output string, metadata AllMetadata) { } g.P("}") + h.P("}") codegen.Merge(g.Tables.Generator, g.Generator).WriteToFile(path.Join(output, "mysqldata.go")) + h.WriteToFile(path.Join(supportedOutput, "supported.go")) var unhandledCount int for impl, collations := range unsupportedByCharset { diff --git a/go/mysql/collations/tools/makecolldata/mysqlversions.go b/go/mysql/collations/tools/makecolldata/mysqlversions.go index 5bdd3165e53..f0578ecd95b 100644 --- a/go/mysql/collations/tools/makecolldata/mysqlversions.go +++ b/go/mysql/collations/tools/makecolldata/mysqlversions.go @@ -60,6 +60,7 @@ func makeversions(output string) { } sort.Strings(versionfiles) + charsets := make(map[string]string) versioninfo := make(map[uint]*versionInfo) for v, versionCsv := range versionfiles { f, err := os.Open(versionCsv) @@ -89,14 +90,17 @@ func makeversions(output string) { collname := cols[0] vi.alias[collname] |= 1 << v + charsets[collname] = cols[1] for from, to := range CharsetAliases { if strings.HasPrefix(collname, from+"_") { aliased := strings.Replace(collname, from+"_", to+"_", 1) + charsets[aliased] = to vi.alias[aliased] |= 1 << v } if strings.HasPrefix(collname, to+"_") { aliased := strings.Replace(collname, to+"_", from+"_", 1) + charsets[aliased] = from vi.alias[aliased] |= 1 << v } } @@ -123,7 +127,7 @@ func makeversions(output string) { var g = codegen.NewGenerator("vitess.io/vitess/go/mysql/collations") g.P("type collver byte") - g.P("type collalias struct { mask collver; name string }") + g.P("type collalias struct { mask collver; name string; charset string }") g.P("const (") g.P("collverInvalid collver = 0") for n, version := range versions { @@ -150,7 +154,7 @@ func makeversions(output string) { // all MySQL versions, but this is implemented as a method on `collver` so when // MySQL maps utf8 to utfmb4, we can perform the mapping only for the specific // MySQL version onwards. - g.P("func (v collver) charsetAliases() map[string]string { return ", fmt.Sprintf("%#v", CharsetAliases), "}") + g.P("func charsetAliases() map[string]string { return ", fmt.Sprintf("%#v", CharsetAliases), "}") g.P() g.P("var globalVersionInfo = map[ID]struct{alias []collalias; isdefault collver}{") @@ -164,14 +168,14 @@ func makeversions(output string) { for _, vi := range sorted { var reverse []alias for a, m := range vi.alias { - reverse = append(reverse, alias{m, a}) + reverse = append(reverse, alias{mask: m, name: a}) } sort.Slice(reverse, func(i, j int) bool { return reverse[i].name < reverse[j].name }) fmt.Fprintf(g, "%d: {alias: []collalias{", vi.id) for _, a := range reverse { - fmt.Fprintf(g, "{0b%08b, %q},", a.mask, a.name) + fmt.Fprintf(g, "{0b%08b, %q, %q},", a.mask, a.name, charsets[a.name]) } fmt.Fprintf(g, "}, isdefault: 0b%08b},\n", vi.isdefault) } diff --git a/go/mysql/collations/tools/maketestdata/maketestdata.go b/go/mysql/collations/tools/maketestdata/maketestdata.go index e8cb0daee5d..67d5a4739f6 100644 --- a/go/mysql/collations/tools/maketestdata/maketestdata.go +++ b/go/mysql/collations/tools/maketestdata/maketestdata.go @@ -30,6 +30,8 @@ import ( "github.com/spf13/pflag" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/internal/flag" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" @@ -166,17 +168,17 @@ func main() { flag.Parse(fs) var defaults = collations.Local() - var collationsForLanguage = make(map[testutil.Lang][]collations.Collation) - var allcollations = defaults.AllCollations() + var collationsForLanguage = make(map[testutil.Lang][]collations.ID) + var allcollations = colldata.All(defaults) for lang := range testutil.KnownLanguages { for _, coll := range allcollations { if lang.MatchesCollation(coll.Name()) { - collationsForLanguage[lang] = append(collationsForLanguage[lang], coll) + collationsForLanguage[lang] = append(collationsForLanguage[lang], coll.ID()) } } } - var rootCollations = []collations.Collation{ + var rootCollations = []collations.ID{ defaults.LookupByName("utf8mb4_0900_as_cs"), defaults.LookupByName("utf8mb4_0900_as_ci"), defaults.LookupByName("utf8mb4_0900_ai_ci"), @@ -211,21 +213,22 @@ func main() { var total int var collationNames []string - var interestingCollations []collations.Collation + var interestingCollations []collations.ID interestingCollations = append(interestingCollations, rootCollations...) interestingCollations = append(interestingCollations, collationsForLanguage[lang]...) for _, collation := range interestingCollations { - transcoded, err := charset.ConvertFromUTF8(nil, collation.Charset(), []byte(snippet)) + transcoded, err := charset.ConvertFromUTF8(nil, colldata.Lookup(collation).Charset(), []byte(snippet)) if err != nil { - log.Printf("[%s] skip collation %s", lang, collation.Name()) + log.Printf("[%s] skip collation %s", lang, defaults.LookupName(collation)) continue } - weights := colldump(collation.Name(), transcoded) - gcase.Weights[collation.Name()] = weights + colName := defaults.LookupName(collation) + weights := colldump(colName, transcoded) + gcase.Weights[colName] = weights total += len(weights) - collationNames = append(collationNames, collation.Name()) + collationNames = append(collationNames, colName) } log.Printf("[%s] written samples for %d collations (%.02fkb): %s", diff --git a/go/mysql/endtoend/query_test.go b/go/mysql/endtoend/query_test.go index 7565c2913e9..576960f2acb 100644 --- a/go/mysql/endtoend/query_test.go +++ b/go/mysql/endtoend/query_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" @@ -39,7 +41,7 @@ const ( func columnSize(cs collations.ID, size uint32) uint32 { // utf8_general_ci results in smaller max column sizes because MySQL 5.7 is silly - if cs.Get().Charset().Name() == "utf8mb3" { + if colldata.Lookup(cs).Charset().Name() == "utf8mb3" { return size * 3 / 4 } return size @@ -321,6 +323,5 @@ func TestSysInfo(t *testing.T) { func getDefaultCollationID() collations.ID { collationHandler := collations.Local() - collation := collationHandler.DefaultCollationForCharset(charsetName) - return collation.ID() + return collationHandler.DefaultCollationForCharset(charsetName) } diff --git a/go/vt/dbconfigs/dbconfigs.go b/go/vt/dbconfigs/dbconfigs.go index c904c273632..fe3a228835c 100644 --- a/go/vt/dbconfigs/dbconfigs.go +++ b/go/vt/dbconfigs/dbconfigs.go @@ -26,8 +26,6 @@ import ( "github.com/spf13/pflag" - "vitess.io/vitess/go/mysql/collations" - "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/vttls" @@ -418,6 +416,6 @@ func NewTestDBConfigs(genParams, appDebugParams mysql.ConnParams, dbname string) replParams: genParams, externalReplParams: genParams, DBName: dbname, - Charset: collations.Default().Get().Name(), + Charset: "", } } diff --git a/go/vt/schemadiff/table.go b/go/vt/schemadiff/table.go index 2db3f47969f..dbc01ec315c 100644 --- a/go/vt/schemadiff/table.go +++ b/go/vt/schemadiff/table.go @@ -25,6 +25,8 @@ import ( golcs "github.com/yudai/golcs" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" ) @@ -392,7 +394,7 @@ const mysqlCollationVersion = "8.0.0" var collationEnv = collations.NewEnvironment(mysqlCollationVersion) func defaultCharset() string { - collation := collations.ID(collationEnv.DefaultConnectionCharset()).Get() + collation := colldata.Lookup(collations.ID(collationEnv.DefaultConnectionCharset())) if collation == nil { return "" } @@ -401,10 +403,10 @@ func defaultCharset() string { func defaultCharsetCollation(charset string) string { collation := collationEnv.DefaultCollationForCharset(charset) - if collation == nil { + if collation == collations.Unknown { return "" } - return collation.Name() + return collationEnv.LookupName(collation) } func (c *CreateTableEntity) normalizeColumnOptions() { diff --git a/go/vt/vtgate/engine/aggregations.go b/go/vt/vtgate/engine/aggregations.go index e34618744c6..d8c648a4ce4 100644 --- a/go/vt/vtgate/engine/aggregations.go +++ b/go/vt/vtgate/engine/aggregations.go @@ -78,7 +78,7 @@ func (ap *AggregateParams) String() string { keyCol = fmt.Sprintf("%s|%d", keyCol, ap.WCol) } if sqltypes.IsText(ap.Type) && ap.CollationID != collations.Unknown { - keyCol += " COLLATE " + ap.CollationID.Get().Name() + keyCol += " COLLATE " + collations.Local().LookupName(ap.CollationID) } dispOrigOp := "" if ap.OrigOpcode != AggregateUnassigned && ap.OrigOpcode != ap.Opcode { diff --git a/go/vt/vtgate/engine/distinct.go b/go/vt/vtgate/engine/distinct.go index 0998bfa0e62..1d057462ec9 100644 --- a/go/vt/vtgate/engine/distinct.go +++ b/go/vt/vtgate/engine/distinct.go @@ -283,7 +283,7 @@ func (cc CheckCol) SwitchToWeightString() CheckCol { func (cc CheckCol) String() string { var collation string if sqltypes.IsText(cc.Type) && cc.Collation != collations.Unknown { - collation = ": " + cc.Collation.Get().Name() + collation = ": " + collations.Local().LookupName(cc.Collation) } var column string diff --git a/go/vt/vtgate/engine/filter_test.go b/go/vt/vtgate/engine/filter_test.go index 01ba7175db4..9a8335e4d7e 100644 --- a/go/vt/vtgate/engine/filter_test.go +++ b/go/vt/vtgate/engine/filter_test.go @@ -29,7 +29,7 @@ import ( ) func TestFilterPass(t *testing.T) { - utf8mb4Bin := collationEnv.LookupByName("utf8mb4_bin").ID() + utf8mb4Bin := collationEnv.LookupByName("utf8mb4_bin") predicate := &sqlparser.ComparisonExpr{ Operator: sqlparser.GreaterThanOp, Left: sqlparser.NewColName("left"), diff --git a/go/vt/vtgate/engine/hash_join.go b/go/vt/vtgate/engine/hash_join.go index 89f552f98c1..3fcda9cf308 100644 --- a/go/vt/vtgate/engine/hash_join.go +++ b/go/vt/vtgate/engine/hash_join.go @@ -246,9 +246,9 @@ func (hj *HashJoin) description() PrimitiveDescription { "Predicate": sqlparser.String(hj.ASTPred), "ComparisonType": hj.ComparisonType.String(), } - coll := hj.Collation.Get() - if coll != nil { - other["Collation"] = coll.Name() + coll := hj.Collation + if coll != collations.Unknown { + other["Collation"] = collations.Local().LookupName(coll) } return PrimitiveDescription{ OperatorType: "Join", diff --git a/go/vt/vtgate/engine/ordered_aggregate.go b/go/vt/vtgate/engine/ordered_aggregate.go index 8427ebb10ce..e0859efe180 100644 --- a/go/vt/vtgate/engine/ordered_aggregate.go +++ b/go/vt/vtgate/engine/ordered_aggregate.go @@ -80,8 +80,7 @@ func (gbp GroupByParams) String() string { } if sqltypes.IsText(gbp.Type) && gbp.CollationID != collations.Unknown { - collation := gbp.CollationID.Get() - out += " COLLATE " + collation.Name() + out += " COLLATE " + collations.Local().LookupName(gbp.CollationID) } return out diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 70c5b7b4380..80c4f181830 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -132,8 +132,7 @@ func (obp OrderByParams) String() string { } if sqltypes.IsText(obp.Type) && obp.CollationID != collations.Unknown { - collation := obp.CollationID.Get() - val += " COLLATE " + collation.Name() + val += " COLLATE " + collations.Local().LookupName(obp.CollationID) } return val } diff --git a/go/vt/vtgate/evalengine/api_compare.go b/go/vt/vtgate/evalengine/api_compare.go index 7289af093f2..e5a4e7335f2 100644 --- a/go/vt/vtgate/evalengine/api_compare.go +++ b/go/vt/vtgate/evalengine/api_compare.go @@ -21,6 +21,7 @@ import ( "fmt" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -155,7 +156,7 @@ func NullsafeCompare(v1, v2 sqltypes.Value, collationID collations.ID) (int, err if collationID == collations.CollationBinaryID { return bytes.Compare(v1.Raw(), v2.Raw()), nil } - coll := collationID.Get() + coll := colldata.Lookup(collationID) if coll == nil { return 0, UnsupportedCollationError{ID: collationID} } diff --git a/go/vt/vtgate/evalengine/api_compare_test.go b/go/vt/vtgate/evalengine/api_compare_test.go index e4fb5d38470..bd87363b7e8 100644 --- a/go/vt/vtgate/evalengine/api_compare_test.go +++ b/go/vt/vtgate/evalengine/api_compare_test.go @@ -61,7 +61,7 @@ func init() { func defaultCollation() collations.TypedCollation { return collations.TypedCollation{ - Collation: collationEnv.LookupByName("utf8mb4_bin").ID(), + Collation: collationEnv.LookupByName("utf8mb4_bin"), Coercibility: collations.CoerceImplicit, Repertoire: collations.RepertoireASCII, } @@ -1107,7 +1107,7 @@ func TestNullComparisons(t *testing.T) { } func TestNullsafeCompare(t *testing.T) { - collation := collationEnv.LookupByName("utf8mb4_general_ci").ID() + collation := collationEnv.LookupByName("utf8mb4_general_ci") tcases := []struct { v1, v2 sqltypes.Value out int diff --git a/go/vt/vtgate/evalengine/api_hash.go b/go/vt/vtgate/evalengine/api_hash.go index 5d5aac98457..209f766840d 100644 --- a/go/vt/vtgate/evalengine/api_hash.go +++ b/go/vt/vtgate/evalengine/api_hash.go @@ -20,6 +20,7 @@ import ( "math" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/decimal" "vitess.io/vitess/go/mysql/fastparse" "vitess.io/vitess/go/sqltypes" @@ -45,7 +46,7 @@ func NullsafeHashcode(v sqltypes.Value, collation collations.ID, coerceType sqlt h := vthash.New() switch e := e.(type) { case *evalBytes: - if !collation.Valid() { + if collation == collations.Unknown { return 0, UnsupportedCollationHashError } e.col.Collation = collation @@ -193,10 +194,10 @@ func NullsafeHashcode128(hash *vthash.Hasher, v sqltypes.Value, collation collat case sqltypes.IsBinary(coerceTo): hash.Write16(hashPrefixBytes) - collations.Binary.Hash(hash, v.Raw(), 0) + colldata.Lookup(collations.CollationBinaryID).Hash(hash, v.Raw(), 0) case sqltypes.IsText(coerceTo): - coll := collation.Get() + coll := colldata.Lookup(collation) if coll == nil { panic("cannot hash unsupported collation") } diff --git a/go/vt/vtgate/evalengine/api_hash_test.go b/go/vt/vtgate/evalengine/api_hash_test.go index 55e39dad77f..f27eb2c1854 100644 --- a/go/vt/vtgate/evalengine/api_hash_test.go +++ b/go/vt/vtgate/evalengine/api_hash_test.go @@ -75,7 +75,7 @@ func TestHashCodes(t *testing.T) { func TestHashCodesRandom(t *testing.T) { tested := 0 equal := 0 - collation := collations.Local().LookupByName("utf8mb4_general_ci").ID() + collation := collations.Local().LookupByName("utf8mb4_general_ci") endTime := time.Now().Add(1 * time.Second) for time.Now().Before(endTime) { tested++ @@ -164,7 +164,7 @@ func TestHashCodes128(t *testing.T) { func TestHashCodesRandom128(t *testing.T) { tested := 0 equal := 0 - collation := collations.Local().LookupByName("utf8mb4_general_ci").ID() + collation := collations.Local().LookupByName("utf8mb4_general_ci") endTime := time.Now().Add(1 * time.Second) for time.Now().Before(endTime) { tested++ diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index de45e6ccc84..69c39249fb9 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -279,7 +279,7 @@ func (cached *LikeExpr) CachedSize(alloc bool) int64 { } // field BinaryExpr vitess.io/vitess/go/vt/vtgate/evalengine.BinaryExpr size += cached.BinaryExpr.CachedSize(false) - // field Match vitess.io/vitess/go/mysql/collations.WildcardPattern + // field Match vitess.io/vitess/go/mysql/collations/colldata.WildcardPattern if cc, ok := cached.Match.(cachedObject); ok { size += cc.CachedSize(true) } diff --git a/go/vt/vtgate/evalengine/compare.go b/go/vt/vtgate/evalengine/compare.go index ef0cafb6127..aa452c61729 100644 --- a/go/vt/vtgate/evalengine/compare.go +++ b/go/vt/vtgate/evalengine/compare.go @@ -19,6 +19,7 @@ package evalengine import ( "bytes" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/decimal" "vitess.io/vitess/go/mysql/json" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -137,7 +138,7 @@ func compareStrings(l, r eval) (int, error) { if err != nil { return 0, err } - collation := col.Collation.Get() + collation := colldata.Lookup(col.Collation) if collation == nil { return 0, vterrors.Errorf(vtrpcpb.Code_UNKNOWN, "cannot compare strings, collation is unknown or unsupported (collation ID: %d)", col.Collation) } @@ -180,7 +181,7 @@ func compareJSONValue(lj, rj *json.Value) (int, error) { } return ld.Cmp(rd), nil case json.TypeString: - return collationJSON.Collation.Get().Collate(lj.ToRawBytes(), rj.ToRawBytes(), false), nil + return colldata.Lookup(collationJSON.Collation).Collate(lj.ToRawBytes(), rj.ToRawBytes(), false), nil case json.TypeBlob, json.TypeBit, json.TypeOpaque: return bytes.Compare(lj.ToUnencodedBytes(), rj.ToUnencodedBytes()), nil case json.TypeBoolean: diff --git a/go/vt/vtgate/evalengine/compiler.go b/go/vt/vtgate/evalengine/compiler.go index 58fb02dae9f..23f7a9f10aa 100644 --- a/go/vt/vtgate/evalengine/compiler.go +++ b/go/vt/vtgate/evalengine/compiler.go @@ -19,6 +19,7 @@ package evalengine import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -38,9 +39,9 @@ type CompilerLog interface { } type compiledCoercion struct { - col collations.Collation - left collations.Coercion - right collations.Coercion + col colldata.Collation + left colldata.Coercion + right colldata.Coercion } type ctype struct { @@ -349,7 +350,7 @@ func (c *compiler) compareAsStrings(lt ctype, rt ctype) error { return err } if coerceLeft == nil && coerceRight == nil { - c.asm.CmpString_collate(merged.Collation.Get()) + c.asm.CmpString_collate(colldata.Lookup(merged.Collation)) } else { if coerceLeft == nil { coerceLeft = func(dst, in []byte) ([]byte, error) { return in, nil } @@ -358,7 +359,7 @@ func (c *compiler) compareAsStrings(lt ctype, rt ctype) error { coerceRight = func(dst, in []byte) ([]byte, error) { return in, nil } } c.asm.CmpString_coerce(&compiledCoercion{ - col: merged.Collation.Get(), + col: colldata.Lookup(merged.Collation), left: coerceLeft, right: coerceRight, }) @@ -367,7 +368,7 @@ func (c *compiler) compareAsStrings(lt ctype, rt ctype) error { } func isEncodingJSONSafe(col collations.ID) bool { - switch col.Get().Charset().(type) { + switch colldata.Lookup(col).Charset().(type) { case charset.Charset_utf8mb4, charset.Charset_utf8mb3, charset.Charset_binary: return true default: diff --git a/go/vt/vtgate/evalengine/compiler_asm.go b/go/vt/vtgate/evalengine/compiler_asm.go index b700894b949..317423696db 100644 --- a/go/vt/vtgate/evalengine/compiler_asm.go +++ b/go/vt/vtgate/evalengine/compiler_asm.go @@ -35,6 +35,8 @@ import ( "github.com/google/uuid" + "vitess.io/vitess/go/mysql/collations/colldata" + "vitess.io/vitess/go/hack" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" @@ -692,7 +694,7 @@ func (asm *assembler) CmpString_coerce(coercion *compiledCoercion) { }, "CMP VARCHAR(SP-2), VARCHAR(SP-1) COERCE AND COLLATE '%s'", coercion.col.Name()) } -func (asm *assembler) CmpString_collate(collation collations.Collation) { +func (asm *assembler) CmpString_collate(collation colldata.Collation) { asm.adjustStack(-2) asm.emit(func(env *ExpressionEnv) int { @@ -2024,7 +2026,7 @@ func (asm *assembler) Fn_CONV_uc(t sqltypes.Type, col collations.TypedCollation) func (asm *assembler) Fn_COLLATION(col collations.TypedCollation) { asm.emit(func(env *ExpressionEnv) int { v := evalCollation(env.vm.stack[env.vm.sp-1]) - env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalText([]byte(v.Collation.Get().Name()), col) + env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalText([]byte(collations.Local().LookupName(v.Collation)), col) return 1 }, "FN COLLATION (SP-1)") } @@ -2292,7 +2294,7 @@ func (asm *assembler) Fn_CHAR_LENGTH() { if sqltypes.IsBinary(arg.SQLType()) { env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalInt64(int64(len(arg.bytes))) } else { - coll := arg.col.Collation.Get() + coll := colldata.Lookup(arg.col.Collation) count := charset.Length(coll.Charset(), arg.bytes) env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalInt64(int64(count)) } @@ -2321,8 +2323,8 @@ func (asm *assembler) Fn_LUCASE(upcase bool) { asm.emit(func(env *ExpressionEnv) int { str := env.vm.stack[env.vm.sp-1].(*evalBytes) - coll := str.col.Collation.Get() - csa, ok := coll.(collations.CaseAwareCollation) + coll := colldata.Lookup(str.col.Collation) + csa, ok := coll.(colldata.CaseAwareCollation) if !ok { env.vm.err = vterrors.Errorf(vtrpc.Code_UNIMPLEMENTED, "not implemented") } else { @@ -2335,8 +2337,8 @@ func (asm *assembler) Fn_LUCASE(upcase bool) { asm.emit(func(env *ExpressionEnv) int { str := env.vm.stack[env.vm.sp-1].(*evalBytes) - coll := str.col.Collation.Get() - csa, ok := coll.(collations.CaseAwareCollation) + coll := colldata.Lookup(str.col.Collation) + csa, ok := coll.(colldata.CaseAwareCollation) if !ok { env.vm.err = vterrors.Errorf(vtrpc.Code_UNIMPLEMENTED, "not implemented") } else { @@ -2366,7 +2368,7 @@ func (asm *assembler) Fn_MULTICMP_b(args int, lessThan bool) { } func (asm *assembler) Fn_MULTICMP_c(args int, lessThan bool, tc collations.TypedCollation) { - col := tc.Collation.Get() + col := colldata.Lookup(tc.Collation) asm.adjustStack(-(args - 1)) asm.emit(func(env *ExpressionEnv) int { @@ -2495,7 +2497,7 @@ func (asm *assembler) Fn_LEFT(col collations.TypedCollation) { return 1 } - cs := col.Collation.Get().Charset() + cs := colldata.Lookup(col.Collation).Charset() strLen := charset.Length(cs, str.bytes) str.tt = int16(sqltypes.VarChar) @@ -2526,7 +2528,7 @@ func (asm *assembler) Fn_RIGHT(col collations.TypedCollation) { return 1 } - cs := col.Collation.Get().Charset() + cs := colldata.Lookup(col.Collation).Charset() strLen := charset.Length(cs, str.bytes) str.tt = int16(sqltypes.VarChar) @@ -2563,7 +2565,7 @@ func (asm *assembler) Fn_LPAD(col collations.TypedCollation) { return 1 } - cs := col.Collation.Get().Charset() + cs := colldata.Lookup(col.Collation).Charset() strLen := charset.Length(cs, str.bytes) l := int(length.i) @@ -2617,7 +2619,7 @@ func (asm *assembler) Fn_RPAD(col collations.TypedCollation) { return 1 } - cs := col.Collation.Get().Charset() + cs := colldata.Lookup(col.Collation).Charset() strLen := charset.Length(cs, str.bytes) l := int(length.i) @@ -2968,7 +2970,7 @@ func (asm *assembler) Like_coerce(expr *LikeExpr, coercion *compiledCoercion) { }, "LIKE VARCHAR(SP-2), VARCHAR(SP-1) COERCE AND COLLATE '%s'", coercion.col.Name()) } -func (asm *assembler) Like_collate(expr *LikeExpr, collation collations.Collation) { +func (asm *assembler) Like_collate(expr *LikeExpr, collation colldata.Collation) { asm.adjustStack(-1) asm.emit(func(env *ExpressionEnv) int { @@ -4304,7 +4306,7 @@ func (asm *assembler) Fn_REGEXP_LIKE(m *icuregex.Matcher, negate bool, c charset }, "FN REGEXP_LIKE VARCHAR(SP-2), VARCHAR(SP-1)") } -func (asm *assembler) Fn_REGEXP_LIKE_slow(negate bool, c collations.Charset, flags icuregex.RegexpFlag, offset int) { +func (asm *assembler) Fn_REGEXP_LIKE_slow(negate bool, c colldata.Charset, flags icuregex.RegexpFlag, offset int) { asm.adjustStack(-offset) asm.emit(func(env *ExpressionEnv) int { var err error @@ -4406,7 +4408,7 @@ func (asm *assembler) Fn_REGEXP_INSTR(m *icuregex.Matcher, c charset.Charset, of }, "FN REGEXP_INSTR VARCHAR(SP-2), VARCHAR(SP-1)") } -func (asm *assembler) Fn_REGEXP_INSTR_slow(c collations.Charset, flags icuregex.RegexpFlag, offset int) { +func (asm *assembler) Fn_REGEXP_INSTR_slow(c colldata.Charset, flags icuregex.RegexpFlag, offset int) { asm.adjustStack(-offset) asm.emit(func(env *ExpressionEnv) int { input := env.vm.stack[env.vm.sp-offset-1].(*evalBytes) @@ -4488,7 +4490,7 @@ func (asm *assembler) Fn_REGEXP_SUBSTR(m *icuregex.Matcher, merged collations.Ty asm.adjustStack(-offset) asm.emit(func(env *ExpressionEnv) int { input := env.vm.stack[env.vm.sp-offset-1].(*evalBytes) - c := merged.Collation.Get().Charset() + c := colldata.Lookup(merged.Collation).Charset() runes := charset.Expand(nil, input.bytes, c) pos := int64(1) @@ -4542,7 +4544,7 @@ func (asm *assembler) Fn_REGEXP_SUBSTR_slow(merged collations.TypedCollation, fl asm.emit(func(env *ExpressionEnv) int { input := env.vm.stack[env.vm.sp-offset-1].(*evalBytes) pattern := env.vm.stack[env.vm.sp-offset].(*evalBytes) - c := merged.Collation.Get().Charset() + c := colldata.Lookup(merged.Collation).Charset() runes := charset.Expand(nil, input.bytes, c) pos := int64(1) @@ -4614,7 +4616,7 @@ func (asm *assembler) Fn_REGEXP_REPLACE(m *icuregex.Matcher, merged collations.T input := env.vm.stack[env.vm.sp-offset-1].(*evalBytes) repl := env.vm.stack[env.vm.sp-offset+1].(*evalBytes) - c := merged.Collation.Get().Charset() + c := colldata.Lookup(merged.Collation).Charset() inputRunes := charset.Expand(nil, input.bytes, c) replRunes := charset.Expand(nil, repl.bytes, c) @@ -4640,7 +4642,8 @@ func (asm *assembler) Fn_REGEXP_REPLACE(m *icuregex.Matcher, merged collations.T m.Reset(inputRunes[pos-1:]) - b, replaced, err := regexpReplace(m, inputRunes, replRunes, pos, occ, merged.Collation.Get().Charset()) + cs := colldata.Lookup(merged.Collation).Charset() + b, replaced, err := regexpReplace(m, inputRunes, replRunes, pos, occ, cs) if err != nil { env.vm.err = err env.vm.sp -= offset @@ -4663,7 +4666,7 @@ func (asm *assembler) Fn_REGEXP_REPLACE_slow(merged collations.TypedCollation, f pattern := env.vm.stack[env.vm.sp-offset].(*evalBytes) repl := env.vm.stack[env.vm.sp-offset+1].(*evalBytes) - c := merged.Collation.Get().Charset() + c := colldata.Lookup(merged.Collation).Charset() inputRunes := charset.Expand(nil, input.bytes, c) replRunes := charset.Expand(nil, repl.bytes, c) @@ -4706,7 +4709,7 @@ func (asm *assembler) Fn_REGEXP_REPLACE_slow(merged collations.TypedCollation, f m := icuregex.NewMatcher(p) m.Reset(inputRunes[pos-1:]) - b, replaced, err := regexpReplace(m, inputRunes, replRunes, pos, occ, merged.Collation.Get().Charset()) + b, replaced, err := regexpReplace(m, inputRunes, replRunes, pos, occ, colldata.Lookup(merged.Collation).Charset()) if err != nil { env.vm.err = err env.vm.sp -= offset diff --git a/go/vt/vtgate/evalengine/eval_bytes.go b/go/vt/vtgate/evalengine/eval_bytes.go index adc9cb32f2d..455394e31e4 100644 --- a/go/vt/vtgate/evalengine/eval_bytes.go +++ b/go/vt/vtgate/evalengine/eval_bytes.go @@ -22,6 +22,7 @@ import ( "vitess.io/vitess/go/hack" "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/datetime" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vthash" @@ -82,8 +83,8 @@ func evalToVarchar(e eval, col collations.ID, convert bool) (*evalBytes, error) typedcol.Collation = col if col != collations.CollationBinaryID { - fromCollation := b.col.Collation.Get() - toCollation := col.Get() + fromCollation := colldata.Lookup(b.col.Collation) + toCollation := colldata.Lookup(col) var err error bytes, err = charset.Convert(nil, toCollation.Charset(), bytes, fromCollation.Charset()) @@ -109,7 +110,7 @@ func (e *evalBytes) Hash(h *vthash.Hasher) { _, _ = h.Write(e.bytes) default: h.Write16(hashPrefixBytes) - col := e.col.Collation.Get() + col := colldata.Lookup(e.col.Collation) col.Hash(h, e.bytes, 0) } } @@ -153,7 +154,7 @@ func (e *evalBytes) truncateInPlace(size int) { e.bytes = e.bytes[:size] } case sqltypes.IsText(tt): - collation := e.col.Collation.Get() + collation := colldata.Lookup(e.col.Collation) e.bytes = charset.Slice(collation.Charset(), e.bytes, 0, size) default: panic("called EvalResult.truncate on non-quoted") diff --git a/go/vt/vtgate/evalengine/eval_json.go b/go/vt/vtgate/evalengine/eval_json.go index 01cf69e5e99..8b19a27f92b 100644 --- a/go/vt/vtgate/evalengine/eval_json.go +++ b/go/vt/vtgate/evalengine/eval_json.go @@ -23,6 +23,7 @@ import ( "vitess.io/vitess/go/hack" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/sqltypes" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -98,7 +99,7 @@ func evalConvert_nj(e evalNumeric) *evalJSON { } func evalConvert_cj(e *evalBytes) (*evalJSON, error) { - jsonText, err := charset.Convert(nil, charset.Charset_utf8mb4{}, e.bytes, e.col.Collation.Get().Charset()) + jsonText, err := charset.Convert(nil, charset.Charset_utf8mb4{}, e.bytes, colldata.Lookup(e.col.Collation).Charset()) if err != nil { return nil, err } @@ -107,7 +108,7 @@ func evalConvert_cj(e *evalBytes) (*evalJSON, error) { } func evalConvertArg_cj(e *evalBytes) (*evalJSON, error) { - jsonText, err := charset.Convert(nil, charset.Charset_utf8mb4{}, e.bytes, e.col.Collation.Get().Charset()) + jsonText, err := charset.Convert(nil, charset.Charset_utf8mb4{}, e.bytes, colldata.Lookup(e.col.Collation).Charset()) if err != nil { return nil, err } diff --git a/go/vt/vtgate/evalengine/eval_result.go b/go/vt/vtgate/evalengine/eval_result.go index fa48ce19a25..19a6ea59220 100644 --- a/go/vt/vtgate/evalengine/eval_result.go +++ b/go/vt/vtgate/evalengine/eval_result.go @@ -21,6 +21,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -39,7 +40,7 @@ func (er EvalResult) Value(id collations.ID) sqltypes.Value { return evalToSQLValue(er.v) } - dst, err := charset.Convert(nil, id.Get().Charset(), str.bytes, str.col.Collation.Get().Charset()) + dst, err := charset.Convert(nil, colldata.Lookup(id).Charset(), str.bytes, colldata.Lookup(str.col.Collation).Charset()) if err != nil { // If we can't convert, we just return what we have, but it's going // to be invalidly encoded. Should normally never happen as only utf8mb4 diff --git a/go/vt/vtgate/evalengine/expr_collate.go b/go/vt/vtgate/evalengine/expr_collate.go index 6e705a7081e..9828a1d8722 100644 --- a/go/vt/vtgate/evalengine/expr_collate.go +++ b/go/vt/vtgate/evalengine/expr_collate.go @@ -18,6 +18,7 @@ package evalengine import ( "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -139,7 +140,7 @@ func evalCollation(e eval) collations.TypedCollation { } } -func mergeCollations(c1, c2 collations.TypedCollation, t1, t2 sqltypes.Type) (collations.TypedCollation, collations.Coercion, collations.Coercion, error) { +func mergeCollations(c1, c2 collations.TypedCollation, t1, t2 sqltypes.Type) (collations.TypedCollation, colldata.Coercion, colldata.Coercion, error) { if c1.Collation == c2.Collation { return c1, nil, nil, nil } @@ -157,7 +158,7 @@ func mergeCollations(c1, c2 collations.TypedCollation, t1, t2 sqltypes.Type) (co } env := collations.Local() - return env.MergeCollations(c1, c2, collations.CoercionOptions{ + return colldata.Merge(env, c1, c2, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -202,7 +203,7 @@ func (ca *collationAggregation) add(env *collations.Environment, tc collations.T ca.cur = tc } else { var err error - ca.cur, _, _, err = env.MergeCollations(ca.cur, tc, collations.CoercionOptions{ConvertToSuperset: true, ConvertWithCoercion: true}) + ca.cur, _, _, err = colldata.Merge(env, ca.cur, tc, colldata.CoercionOptions{ConvertToSuperset: true, ConvertWithCoercion: true}) if err != nil { return err } diff --git a/go/vt/vtgate/evalengine/expr_compare.go b/go/vt/vtgate/evalengine/expr_compare.go index eab7d1e6b7b..e7490370a1b 100644 --- a/go/vt/vtgate/evalengine/expr_compare.go +++ b/go/vt/vtgate/evalengine/expr_compare.go @@ -20,6 +20,7 @@ import ( "bytes" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -41,7 +42,7 @@ type ( LikeExpr struct { BinaryExpr Negate bool - Match collations.WildcardPattern + Match colldata.WildcardPattern MatchCollation collations.ID } @@ -569,7 +570,7 @@ func (l *LikeExpr) matchWildcard(left, right []byte, coll collations.ID) bool { if l.Match != nil && l.MatchCollation == coll { return l.Match.Match(left) } - fullColl := coll.Get() + fullColl := colldata.Lookup(coll) wc := fullColl.Wildcard(right, 0, 0, 0) return wc.Match(left) } @@ -639,12 +640,12 @@ func (expr *LikeExpr) compile(c *compiler) (ctype, error) { } var merged collations.TypedCollation - var coerceLeft collations.Coercion - var coerceRight collations.Coercion + var coerceLeft colldata.Coercion + var coerceRight colldata.Coercion var env = collations.Local() if lt.Col.Collation != rt.Col.Collation { - merged, coerceLeft, coerceRight, err = env.MergeCollations(lt.Col, rt.Col, collations.CoercionOptions{ + merged, coerceLeft, coerceRight, err = colldata.Merge(env, lt.Col, rt.Col, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -656,7 +657,7 @@ func (expr *LikeExpr) compile(c *compiler) (ctype, error) { } if coerceLeft == nil && coerceRight == nil { - c.asm.Like_collate(expr, merged.Collation.Get()) + c.asm.Like_collate(expr, colldata.Lookup(merged.Collation)) } else { if coerceLeft == nil { coerceLeft = func(dst, in []byte) ([]byte, error) { return in, nil } @@ -665,7 +666,7 @@ func (expr *LikeExpr) compile(c *compiler) (ctype, error) { coerceRight = func(dst, in []byte) ([]byte, error) { return in, nil } } c.asm.Like_coerce(expr, &compiledCoercion{ - col: merged.Collation.Get(), + col: colldata.Lookup(merged.Collation), left: coerceLeft, right: coerceRight, }) diff --git a/go/vt/vtgate/evalengine/expr_convert.go b/go/vt/vtgate/evalengine/expr_convert.go index 54961dd3774..6531cdd6fae 100644 --- a/go/vt/vtgate/evalengine/expr_convert.go +++ b/go/vt/vtgate/evalengine/expr_convert.go @@ -18,6 +18,7 @@ package evalengine import ( "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -195,7 +196,7 @@ func (c *ConvertExpr) convertToBinaryType(tt sqltypes.Type) sqltypes.Type { func (c *ConvertExpr) convertToCharType(tt sqltypes.Type) sqltypes.Type { if c.HasLength { - col := c.Collation.Get() + col := colldata.Lookup(c.Collation) length := c.Length * col.Charset().MaxWidth() if length > 64*1024 { return sqltypes.Text diff --git a/go/vt/vtgate/evalengine/fn_compare.go b/go/vt/vtgate/evalengine/fn_compare.go index ef8d7f4e3c8..ee4f61cb596 100644 --- a/go/vt/vtgate/evalengine/fn_compare.go +++ b/go/vt/vtgate/evalengine/fn_compare.go @@ -21,6 +21,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/proto/vtrpc" @@ -215,11 +216,11 @@ func compareAllText(args []eval, cmp int) (eval, error) { if err := ca.add(env, col); err != nil { return nil, err } - charsets = append(charsets, col.Collation.Get().Charset()) + charsets = append(charsets, colldata.Lookup(col.Collation).Charset()) } tc := ca.result() - col := tc.Collation.Get() + col := colldata.Lookup(tc.Collation) cs := col.Charset() b1, err := charset.Convert(nil, cs, args[0].ToRawBytes(), charsets[0]) diff --git a/go/vt/vtgate/evalengine/fn_regexp.go b/go/vt/vtgate/evalengine/fn_regexp.go index 5886a5c3765..2ba5b97573f 100644 --- a/go/vt/vtgate/evalengine/fn_regexp.go +++ b/go/vt/vtgate/evalengine/fn_regexp.go @@ -22,6 +22,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/icuregex" icuerrors "vitess.io/vitess/go/mysql/icuregex/errors" "vitess.io/vitess/go/sqltypes" @@ -101,8 +102,9 @@ func evalRegexpCollation(input, pat eval, f string) (eval, eval, collations.Type patCol := patBytes.col.Collation if (inputCol == collations.CollationBinaryID && patCol != collations.CollationBinaryID) || (inputCol != collations.CollationBinaryID && patCol == collations.CollationBinaryID) { - inputColName := inputCol.Get().Name() - patColName := patCol.Get().Name() + env := collations.Local() + inputColName := env.LookupName(inputCol) + patColName := env.LookupName(patCol) return nil, nil, typedCol, 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.CharacterSetMismatch, "Character set '%s' cannot be used in conjunction with '%s' in call to %s.", inputColName, patColName, f) } } @@ -114,8 +116,8 @@ func evalRegexpCollation(input, pat eval, f string) (eval, eval, collations.Type } var flags icuregex.RegexpFlag - var collation = typedCol.Collation.Get() - if strings.Contains(collation.Name(), "_ci") { + var collation = collations.Local().LookupName(typedCol.Collation) + if strings.Contains(collation, "_ci") { flags |= icuregex.CaseInsensitive } @@ -126,13 +128,14 @@ func compileRegexpCollation(input, pat ctype, f string) (collations.TypedCollati var merged collations.TypedCollation var err error + env := collations.Local() if input.isTextual() && pat.isTextual() { inputCol := input.Col.Collation patCol := pat.Col.Collation if (inputCol == collations.CollationBinaryID && patCol != collations.CollationBinaryID) || (inputCol != collations.CollationBinaryID && patCol == collations.CollationBinaryID) { - inputColName := inputCol.Get().Name() - patColName := patCol.Get().Name() + inputColName := env.LookupName(inputCol) + patColName := env.LookupName(patCol) return input.Col, 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.CharacterSetMismatch, "Character set '%s' cannot be used in conjunction with '%s' in call to %s.", inputColName, patColName, f) } } @@ -147,14 +150,13 @@ func compileRegexpCollation(input, pat ctype, f string) (collations.TypedCollati } var flags icuregex.RegexpFlag - var collation = merged.Collation.Get() - if strings.Contains(collation.Name(), "_ci") { + if strings.Contains(env.LookupName(merged.Collation), "_ci") { flags |= icuregex.CaseInsensitive } return merged, flags, nil } -func compileRegex(pat eval, c collations.Charset, flags icuregex.RegexpFlag) (*icuregex.Pattern, error) { +func compileRegex(pat eval, c colldata.Charset, flags icuregex.RegexpFlag) (*icuregex.Pattern, error) { patRunes := charset.Expand(nil, pat.ToRawBytes(), c) if len(patRunes) == 0 { @@ -247,7 +249,7 @@ func compileConstantRegex(c *compiler, args TupleExpr, pat, mt int, cs collation return nil, err } - return compileRegex(innerPat, cs.Collation.Get().Charset(), flags) + return compileRegex(innerPat, colldata.Lookup(cs.Collation).Charset(), flags) } // resultCollation returns the collation to use for the result of a regexp. @@ -281,7 +283,7 @@ func (r *builtinRegexpLike) eval(env *ExpressionEnv) (eval, error) { if err != nil { return nil, err } - collation := typedCol.Collation.Get() + collation := colldata.Lookup(typedCol.Collation) if len(r.Arguments) > 2 { m, err := r.Arguments[2].eval(env) @@ -328,7 +330,7 @@ func (r *builtinRegexpLike) compileSlow(c *compiler, input, pat, fl ctype, merge c.asm.Convert_xce(len(r.Arguments)-1, sqltypes.VarChar, merged.Collation) } - c.asm.Fn_REGEXP_LIKE_slow(r.Negate, merged.Collation.Get().Charset(), flags, len(r.Arguments)-1) + c.asm.Fn_REGEXP_LIKE_slow(r.Negate, colldata.Lookup(merged.Collation).Charset(), flags, len(r.Arguments)-1) c.asm.jumpDestination(skips...) return ctype{Type: sqltypes.Int64, Col: collationNumeric, Flag: input.Flag | pat.Flag | fl.Flag | flagIsBoolean}, nil } @@ -373,7 +375,7 @@ func (r *builtinRegexpLike) compile(c *compiler) (ctype, error) { return r.compileSlow(c, input, pat, f, merged, flags, skips...) } - c.asm.Fn_REGEXP_LIKE(icuregex.NewMatcher(p), r.Negate, merged.Collation.Get().Charset(), len(r.Arguments)-1) + c.asm.Fn_REGEXP_LIKE(icuregex.NewMatcher(p), r.Negate, colldata.Lookup(merged.Collation).Charset(), len(r.Arguments)-1) c.asm.jumpDestination(skips...) return ctype{Type: sqltypes.Int64, Col: collationNumeric, Flag: input.Flag | pat.Flag | f.Flag | flagIsBoolean}, nil @@ -433,7 +435,7 @@ func (r *builtinRegexpInstr) eval(env *ExpressionEnv) (eval, error) { } } - collation := typedCol.Collation.Get() + collation := colldata.Lookup(typedCol.Collation) pos := int64(1) occ := int64(1) @@ -520,7 +522,7 @@ func (r *builtinRegexpInstr) compileSlow(c *compiler, input, pat, pos, occ, retu c.asm.Convert_xce(len(r.Arguments)-1, sqltypes.VarChar, merged.Collation) } - c.asm.Fn_REGEXP_INSTR_slow(merged.Collation.Get().Charset(), flags, len(r.Arguments)-1) + c.asm.Fn_REGEXP_INSTR_slow(colldata.Lookup(merged.Collation).Charset(), flags, len(r.Arguments)-1) c.asm.jumpDestination(skips...) return ctype{Type: sqltypes.Int64, Col: collationNumeric, Flag: input.Flag | pat.Flag | pos.Flag | occ.Flag | returnOption.Flag | matchType.Flag}, nil } @@ -599,7 +601,7 @@ func (r *builtinRegexpInstr) compile(c *compiler) (ctype, error) { return r.compileSlow(c, input, pat, pos, occ, returnOpt, matchType, merged, flags, skips...) } - c.asm.Fn_REGEXP_INSTR(icuregex.NewMatcher(p), merged.Collation.Get().Charset(), len(r.Arguments)-1) + c.asm.Fn_REGEXP_INSTR(icuregex.NewMatcher(p), colldata.Lookup(merged.Collation).Charset(), len(r.Arguments)-1) c.asm.jumpDestination(skips...) return ctype{Type: sqltypes.Int64, Col: collationNumeric, Flag: input.Flag | pat.Flag | flagIsBoolean}, nil @@ -652,7 +654,7 @@ func (r *builtinRegexpSubstr) eval(env *ExpressionEnv) (eval, error) { } } - collation := typedCol.Collation.Get() + collation := colldata.Lookup(typedCol.Collation) pos := int64(1) occ := int64(1) inputRunes := charset.Expand(nil, input.ToRawBytes(), collation.Charset()) @@ -804,7 +806,7 @@ type builtinRegexpReplace struct { CallExpr } -func regexpReplace(m *icuregex.Matcher, inputRunes, replRunes []rune, pos, occ int64, c collations.Charset) ([]byte, bool, error) { +func regexpReplace(m *icuregex.Matcher, inputRunes, replRunes []rune, pos, occ int64, c colldata.Charset) ([]byte, bool, error) { var err error found := false if occ > 0 { @@ -902,7 +904,7 @@ func (r *builtinRegexpReplace) eval(env *ExpressionEnv) (eval, error) { } } - collation := typedCol.Collation.Get() + collation := colldata.Lookup(typedCol.Collation) repl, ok := replArg.(*evalBytes) if !ok { @@ -914,7 +916,7 @@ func (r *builtinRegexpReplace) eval(env *ExpressionEnv) (eval, error) { pos := int64(1) occ := int64(0) inputRunes := charset.Expand(nil, input.ToRawBytes(), collation.Charset()) - replRunes := charset.Expand(nil, repl.ToRawBytes(), repl.col.Collation.Get().Charset()) + replRunes := charset.Expand(nil, repl.ToRawBytes(), colldata.Lookup(repl.col.Collation).Charset()) if posExpr != nil { pos, err = position(evalToInt64(posExpr), int64(len(inputRunes)), "regexp_replace") diff --git a/go/vt/vtgate/evalengine/fn_string.go b/go/vt/vtgate/evalengine/fn_string.go index c6fc75c9d70..b34618b00d2 100644 --- a/go/vt/vtgate/evalengine/fn_string.go +++ b/go/vt/vtgate/evalengine/fn_string.go @@ -21,6 +21,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -117,8 +118,8 @@ func (call *builtinChangeCase) eval(env *ExpressionEnv) (eval, error) { return evalToVarchar(e, call.collate, false) case *evalBytes: - coll := e.col.Collation.Get() - csa, ok := coll.(collations.CaseAwareCollation) + coll := colldata.Lookup(e.col.Collation) + csa, ok := coll.(colldata.CaseAwareCollation) if !ok { return nil, vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "not implemented") } @@ -172,7 +173,7 @@ func (call *builtinCharLength) eval(env *ExpressionEnv) (eval, error) { if sqltypes.IsBinary(e.SQLType()) { return newEvalInt64(int64(len(e.bytes))), nil } - coll := e.col.Collation.Get() + coll := colldata.Lookup(e.col.Collation) count := charset.Length(coll.Charset(), e.bytes) return newEvalInt64(int64(count)), nil default: @@ -277,7 +278,7 @@ func charOrd(b []byte, coll collations.ID) int64 { if len(b) == 0 { return 0 } - cs := coll.Get().Charset() + cs := colldata.Lookup(coll).Charset() _, l := cs.DecodeRune(b) var r int64 for i := 0; i < l; i++ { @@ -429,11 +430,11 @@ func (c *builtinCollation) eval(env *ExpressionEnv) (eval, error) { return nil, err } - col := evalCollation(arg).Collation.Get() + col := evalCollation(arg) // the collation of a `COLLATION` expr is hardcoded to `utf8mb3_general_ci`, // not to the default collation of our connection. this is probably a bug in MySQL, but we match it - return newEvalText([]byte(col.Name()), collationUtf8mb3), nil + return newEvalText([]byte(collations.Local().LookupName(col.Collation)), collationUtf8mb3), nil } func (*builtinCollation) typeof(env *ExpressionEnv, fields []*querypb.Field) (sqltypes.Type, typeFlag) { @@ -612,7 +613,7 @@ func (call builtinLeftRight) eval(env *ExpressionEnv) (eval, error) { } // LEFT / RIGHT operates on characters, not bytes - cs := text.col.Collation.Get().Charset() + cs := colldata.Lookup(text.col.Collation).Charset() strLen := charset.Length(cs, text.bytes) if strLen <= int(length) { @@ -682,9 +683,9 @@ func (call builtinPad) eval(env *ExpressionEnv) (eval, error) { } } - cs := text.col.Collation.Get().Charset() + cs := colldata.Lookup(text.col.Collation).Charset() pad, ok := p.(*evalBytes) - if !ok || pad.col.Collation.Get().Charset() != cs { + if !ok || colldata.Lookup(pad.col.Collation).Charset() != cs { pad, err = evalToVarchar(p, text.col.Collation, true) if err != nil { return nil, err @@ -768,8 +769,8 @@ func (call builtinPad) compile(c *compiler) (ctype, error) { switch { case pad.isTextual(): - fromCharset := pad.Col.Collation.Get().Charset() - toCharset := col.Collation.Get().Charset() + fromCharset := colldata.Lookup(pad.Col.Collation).Charset() + toCharset := colldata.Lookup(col.Collation).Charset() if fromCharset != toCharset && !toCharset.IsSuperset(fromCharset) { c.asm.Convert_xce(1, sqltypes.VarChar, col.Collation) } @@ -787,7 +788,7 @@ func (call builtinPad) compile(c *compiler) (ctype, error) { } func strcmpCollate(left, right []byte, col collations.ID) int64 { - cmp := col.Get().Collate(left, right, false) + cmp := colldata.Lookup(col).Collate(left, right, false) switch { case cmp == 0: return 0 @@ -819,7 +820,7 @@ func (l *builtinStrcmp) eval(env *ExpressionEnv) (eval, error) { col1 := evalCollation(left) col2 := evalCollation(right) - mcol, _, _, err := collations.Local().MergeCollations(col1, col2, collations.CoercionOptions{ + mcol, _, _, err := colldata.Merge(collations.Local(), col1, col2, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -866,7 +867,7 @@ func (expr *builtinStrcmp) compile(c *compiler) (ctype, error) { if sqltypes.IsNumber(lt.Type) || sqltypes.IsNumber(rt.Type) { mcol = collationNumeric } else { - mcol, _, _, err = collations.Local().MergeCollations(lt.Col, rt.Col, collations.CoercionOptions{ + mcol, _, _, err = colldata.Merge(collations.Local(), lt.Col, rt.Col, colldata.CoercionOptions{ ConvertToSuperset: true, ConvertWithCoercion: true, }) @@ -926,7 +927,7 @@ func (call builtinTrim) eval(env *ExpressionEnv) (eval, error) { } pat, ok := p.(*evalBytes) - if !ok || pat.col.Collation.Get().Charset() != text.col.Collation.Get().Charset() { + if !ok || colldata.Lookup(pat.col.Collation).Charset() != colldata.Lookup(text.col.Collation).Charset() { pat, err = evalToVarchar(p, text.col.Collation, true) if err != nil { return nil, err @@ -986,8 +987,8 @@ func (call builtinTrim) compile(c *compiler) (ctype, error) { switch { case pat.isTextual(): - fromCharset := pat.Col.Collation.Get().Charset() - toCharset := col.Collation.Get().Charset() + fromCharset := colldata.Lookup(pat.Col.Collation).Charset() + toCharset := colldata.Lookup(col.Collation).Charset() if fromCharset != toCharset && !toCharset.IsSuperset(fromCharset) { c.asm.Convert_xce(1, sqltypes.VarChar, col.Collation) } @@ -1033,8 +1034,8 @@ func concatConvert(buf []byte, str *evalBytes, tc collations.TypedCollation) ([] if tc.Collation == collations.CollationBinaryID { return append(buf, str.bytes...), nil } - fromCharset := str.col.Collation.Get().Charset() - toCharset := tc.Collation.Get().Charset() + fromCharset := colldata.Lookup(str.col.Collation).Charset() + toCharset := colldata.Lookup(tc.Collation).Charset() if fromCharset != toCharset { return charset.Convert(buf, toCharset, str.bytes, fromCharset) } @@ -1138,8 +1139,8 @@ func (call *builtinConcat) compile(c *compiler) (ctype, error) { c.asm.Convert_xce(len(args)-i, arg.Type, tc.Collation) } case sqltypes.VarChar, sqltypes.Char, sqltypes.Text: - fromCharset := arg.Col.Collation.Get().Charset() - toCharset := tc.Collation.Get().Charset() + fromCharset := colldata.Lookup(arg.Col.Collation).Charset() + toCharset := colldata.Lookup(tc.Collation).Charset() if fromCharset != toCharset && !toCharset.IsSuperset(fromCharset) { c.asm.Convert_xce(len(args)-i, arg.Type, tc.Collation) } @@ -1286,8 +1287,8 @@ func (call *builtinConcatWs) compile(c *compiler) (ctype, error) { c.asm.Convert_xce(offset, arg.Type, tc.Collation) } case sqltypes.VarChar, sqltypes.Char, sqltypes.Text: - fromCharset := arg.Col.Collation.Get().Charset() - toCharset := tc.Collation.Get().Charset() + fromCharset := colldata.Lookup(arg.Col.Collation).Charset() + toCharset := colldata.Lookup(tc.Collation).Charset() if fromCharset != toCharset && !toCharset.IsSuperset(fromCharset) { c.asm.Convert_xce(offset, arg.Type, tc.Collation) } diff --git a/go/vt/vtgate/evalengine/format.go b/go/vt/vtgate/evalengine/format.go index fe641e0954a..446d3e0f28f 100644 --- a/go/vt/vtgate/evalengine/format.go +++ b/go/vt/vtgate/evalengine/format.go @@ -117,15 +117,13 @@ func (t TupleExpr) format(w *formatter, depth int) { func (c *CollateExpr) format(w *formatter, depth int) { c.Inner.format(w, depth) - coll := c.TypedCollation.Collation.Get() w.WriteString(" COLLATE ") - w.WriteString(coll.Name()) + w.WriteString(collations.Local().LookupName(c.TypedCollation.Collation)) } func (i *IntroducerExpr) format(w *formatter, depth int) { w.WriteString("_") - coll := i.TypedCollation.Collation.Get() - w.WriteString(coll.Name()) + w.WriteString(collations.Local().LookupName(i.TypedCollation.Collation)) i.Inner.format(w, depth) } @@ -206,7 +204,7 @@ func (c *ConvertExpr) format(buf *formatter, depth int) { } if c.Collation != collations.Unknown { buf.WriteString(" CHARACTER SET ") - buf.WriteString(c.Collation.Get().Name()) + buf.WriteString(collations.Local().LookupName(c.Collation)) } buf.WriteByte(')') } @@ -215,6 +213,6 @@ func (c *ConvertUsingExpr) format(buf *formatter, depth int) { buf.WriteString("CONVERT(") c.Inner.format(buf, depth) buf.WriteString(" USING ") - buf.WriteString(c.Collation.Get().Name()) + buf.WriteString(collations.Local().LookupName(c.Collation)) buf.WriteByte(')') } diff --git a/go/vt/vtgate/evalengine/integration/comparison_test.go b/go/vt/vtgate/evalengine/integration/comparison_test.go index a28c7f40a96..bde8435f688 100644 --- a/go/vt/vtgate/evalengine/integration/comparison_test.go +++ b/go/vt/vtgate/evalengine/integration/comparison_test.go @@ -179,7 +179,7 @@ func compareRemoteExprEnv(t *testing.T, env *evalengine.ExpressionEnv, conn *mys // TODO: passthrough proper collations for nullable fields remoteCollation = collations.CollationBinaryID } else { - remoteCollation = collationEnv.LookupByName(remote.Rows[0][1].ToString()).ID() + remoteCollation = collationEnv.LookupByName(remote.Rows[0][1].ToString()) } } } diff --git a/go/vt/vtgate/evalengine/integration/fuzz_test.go b/go/vt/vtgate/evalengine/integration/fuzz_test.go index dea6f295bcf..ebfaa486b19 100644 --- a/go/vt/vtgate/evalengine/integration/fuzz_test.go +++ b/go/vt/vtgate/evalengine/integration/fuzz_test.go @@ -344,11 +344,12 @@ func compareResult(local, remote Result, cmp *testcases.Comparison) error { var localCollationName string var remoteCollationName string - if coll := local.Collation.Get(); coll != nil { - localCollationName = coll.Name() + env := collations.Local() + if coll := local.Collation; coll != collations.Unknown { + localCollationName = env.LookupName(coll) } - if coll := remote.Collation.Get(); coll != nil { - remoteCollationName = coll.Name() + if coll := remote.Collation; coll != collations.Unknown { + remoteCollationName = env.LookupName(coll) } equals, err := cmp.Equals(local.Value, remote.Value) diff --git a/go/vt/vtgate/evalengine/mysql_test.go b/go/vt/vtgate/evalengine/mysql_test.go index 00067cb5c28..d63962f78c2 100644 --- a/go/vt/vtgate/evalengine/mysql_test.go +++ b/go/vt/vtgate/evalengine/mysql_test.go @@ -147,5 +147,5 @@ func TestMySQLGolden(t *testing.T) { func TestDebug1(t *testing.T) { // Debug eval, err := testSingle(t, `SELECT _latin1 0xFF regexp _latin1 '[[:lower:]]' COLLATE latin1_bin`) - t.Logf("eval=%s err=%v coll=%s", eval.String(), err, eval.Collation().Get().Name()) + t.Logf("eval=%s err=%v coll=%s", eval.String(), err, collations.Local().LookupName(eval.Collation())) } diff --git a/go/vt/vtgate/evalengine/translate.go b/go/vt/vtgate/evalengine/translate.go index e07b138f848..3af97a183e3 100644 --- a/go/vt/vtgate/evalengine/translate.go +++ b/go/vt/vtgate/evalengine/translate.go @@ -325,13 +325,13 @@ func (ast *astCompiler) translateCollateExpr(collate *sqlparser.CollateExpr) (Ex return nil, err } coll := collations.Local().LookupByName(collate.Collation) - if coll == nil { + if coll == collations.Unknown { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Unknown collation: '%s'", collate.Collation) } return &CollateExpr{ UnaryExpr: UnaryExpr{expr}, TypedCollation: collations.TypedCollation{ - Collation: coll.ID(), + Collation: coll, Coercibility: collations.CoerceExplicit, Repertoire: collations.RepertoireUnicode, }, @@ -349,10 +349,10 @@ func (ast *astCompiler) translateIntroducerExpr(introduced *sqlparser.Introducer collation = collations.CollationBinaryID } else { defaultCollation := collations.Local().DefaultCollationForCharset(introduced.CharacterSet[1:]) - if defaultCollation == nil { + if defaultCollation == collations.Unknown { panic(fmt.Sprintf("unknown character set: %s", introduced.CharacterSet)) } - collation = defaultCollation.ID() + collation = defaultCollation } switch lit := expr.(type) { diff --git a/go/vt/vtgate/evalengine/translate_convert.go b/go/vt/vtgate/evalengine/translate_convert.go index 12b1fdfc9ab..5560315f8e2 100644 --- a/go/vt/vtgate/evalengine/translate_convert.go +++ b/go/vt/vtgate/evalengine/translate_convert.go @@ -20,6 +20,7 @@ import ( "strings" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/decimal" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/sqlparser" @@ -27,15 +28,11 @@ import ( ) func (ast *astCompiler) binaryCollationForCollation(collation collations.ID) collations.ID { - binary := collation.Get() + binary := colldata.Lookup(collation) if binary == nil { return collations.Unknown } - binaryCollation := collations.Local().BinaryCollationForCharset(binary.Charset().Name()) - if binaryCollation == nil { - return collations.Unknown - } - return binaryCollation.ID() + return collations.Local().BinaryCollationForCharset(binary.Charset().Name()) } func (ast *astCompiler) translateConvertCharset(charset string, binary bool) (collations.ID, error) { @@ -50,11 +47,10 @@ func (ast *astCompiler) translateConvertCharset(charset string, binary bool) (co return collation, nil } charset = strings.ToLower(charset) - collation := collations.Local().DefaultCollationForCharset(charset) - if collation == nil { + collationID := collations.Local().DefaultCollationForCharset(charset) + if collationID == collations.Unknown { return collations.Unknown, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Unknown character set: '%s'", charset) } - collationID := collation.ID() if binary { collationID = ast.binaryCollationForCollation(collationID) if collationID == collations.Unknown { diff --git a/go/vt/vtgate/evalengine/translate_simplify.go b/go/vt/vtgate/evalengine/translate_simplify.go index d851114751d..7f2261b4790 100644 --- a/go/vt/vtgate/evalengine/translate_simplify.go +++ b/go/vt/vtgate/evalengine/translate_simplify.go @@ -16,6 +16,8 @@ limitations under the License. package evalengine +import "vitess.io/vitess/go/mysql/collations/colldata" + func (expr *Literal) constant() bool { return true } @@ -78,7 +80,7 @@ func (expr *LikeExpr) simplify(env *ExpressionEnv) error { if lit, ok := expr.Right.(*Literal); ok { if b, ok := lit.inner.(*evalBytes); ok && (b.isVarChar() || b.isBinary()) { expr.MatchCollation = b.col.Collation - coll := expr.MatchCollation.Get() + coll := colldata.Lookup(expr.MatchCollation) expr.Match = coll.Wildcard(b.bytes, 0, 0, 0) } } diff --git a/go/vt/vtgate/evalengine/weights.go b/go/vt/vtgate/evalengine/weights.go index 4be4e6eb7eb..08ec844f357 100644 --- a/go/vt/vtgate/evalengine/weights.go +++ b/go/vt/vtgate/evalengine/weights.go @@ -22,6 +22,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/mysql/decimal" "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/sqltypes" @@ -91,7 +92,7 @@ func WeightString(dst []byte, v sqltypes.Value, coerceTo sqltypes.Type, col coll return append(dst, b...), false, nil case sqltypes.IsText(coerceTo): - coll := col.Get() + coll := colldata.Lookup(col) if coll == nil { return dst, false, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "cannot hash unsupported collation") } @@ -158,7 +159,7 @@ func evalWeightString(dst []byte, e eval, length, precision int) ([]byte, bool, } return append(dst, b...), false, nil } - coll := e.col.Collation.Get() + coll := colldata.Lookup(e.col.Collation) if coll == nil { return dst, false, vterrors.Errorf(vtrpcpb.Code_INTERNAL, "cannot hash unsupported collation") } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 99a40b2aef4..591782bbba3 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -667,7 +667,7 @@ func TestExecutorShow(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Default().Get().Name()), + collations.Local().LookupName(collations.Default())), sqltypes.NewUint32(4)), }, } @@ -712,7 +712,7 @@ func TestExecutorShow(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Default().Get().Name()), + collations.Local().LookupName(collations.Default())), sqltypes.NewUint32(4)), }, } diff --git a/go/vt/vtgate/planbuilder/collations_test.go b/go/vt/vtgate/planbuilder/collations_test.go index f597f45562d..24fb038b4c2 100644 --- a/go/vt/vtgate/planbuilder/collations_test.go +++ b/go/vt/vtgate/planbuilder/collations_test.go @@ -67,7 +67,7 @@ func (tc *collationTestCase) addCollationsToSchema(vschema *vschemawrapper.VSche func TestOrderedAggregateCollations(t *testing.T) { collid := func(collname string) collations.ID { - return collations.Local().LookupByName(collname).ID() + return collations.Local().LookupByName(collname) } testCases := []collationTestCase{ { diff --git a/go/vt/vtgate/planbuilder/show_test.go b/go/vt/vtgate/planbuilder/show_test.go index 04f8c571764..b36133bb1c7 100644 --- a/go/vt/vtgate/planbuilder/show_test.go +++ b/go/vt/vtgate/planbuilder/show_test.go @@ -76,7 +76,7 @@ func TestGenerateCharsetRows(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Default().Get().Name()), + collations.Local().LookupName(collations.Default())), sqltypes.NewUint32(4)), } rows2 := [][]sqltypes.Value{ @@ -88,7 +88,7 @@ func TestGenerateCharsetRows(t *testing.T) { append(buildVarCharRow( "utf8mb4", "UTF-8 Unicode", - collations.Default().Get().Name()), + collations.Local().LookupName(collations.Default())), sqltypes.NewUint32(4)), } diff --git a/go/vt/vtgate/vcursor_impl.go b/go/vt/vtgate/vcursor_impl.go index ca93c97ce3d..8f65884dba3 100644 --- a/go/vt/vtgate/vcursor_impl.go +++ b/go/vt/vtgate/vcursor_impl.go @@ -1048,7 +1048,7 @@ func (vc *vcursorImpl) keyForPlan(ctx context.Context, query string, buf io.Stri _, _ = buf.WriteString(vc.keyspace) _, _ = buf.WriteString(vindexes.TabletTypeSuffix[vc.tabletType]) _, _ = buf.WriteString("+Collate:") - _, _ = buf.WriteString(vc.collation.Get().Name()) + _, _ = buf.WriteString(collations.Local().LookupName(vc.collation)) if vc.destination != nil { switch vc.destination.(type) { diff --git a/go/vt/vttablet/onlineddl/vrepl.go b/go/vt/vttablet/onlineddl/vrepl.go index 24711df0f14..cc669e11c11 100644 --- a/go/vt/vttablet/onlineddl/vrepl.go +++ b/go/vt/vttablet/onlineddl/vrepl.go @@ -33,6 +33,7 @@ import ( "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/textutil" "vitess.io/vitess/go/vt/dbconnpool" @@ -499,12 +500,12 @@ func (v *VRepl) generateFilterQuery(ctx context.Context) error { // Check source and target charset/encoding. If needed, create // a binlogdatapb.CharsetConversion entry (later written to vreplication) fromCollation := collations.Local().DefaultCollationForCharset(sourceCol.Charset) - if fromCollation == nil { + if fromCollation == collations.Unknown { return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", sourceCol.Charset, sourceCol.Name) } toCollation := collations.Local().DefaultCollationForCharset(targetCol.Charset) // Let's see if target col is at all textual - if targetCol.Type == vrepl.StringColumnType && toCollation == nil { + if targetCol.Type == vrepl.StringColumnType && toCollation == collations.Unknown { return vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", targetCol.Charset, targetCol.Name) } @@ -533,12 +534,12 @@ func (v *VRepl) generateFilterQuery(ctx context.Context) error { return nil } -func trivialCharset(c collations.Collation) bool { - if c == nil { +func trivialCharset(c collations.ID) bool { + if c == collations.Unknown { return true } utf8mb4Charset := charset.Charset_utf8mb4{} - return utf8mb4Charset.IsSuperset(c.Charset()) || c.ID() == collations.CollationBinaryID + return utf8mb4Charset.IsSuperset(colldata.Lookup(c).Charset()) || c == collations.CollationBinaryID } func (v *VRepl) analyzeBinlogSource(ctx context.Context) { diff --git a/go/vt/vttablet/tabletmanager/vdiff/table_differ.go b/go/vt/vttablet/tabletmanager/vdiff/table_differ.go index bab857a29b7..8bdfd97e422 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/table_differ.go +++ b/go/vt/vttablet/tabletmanager/vdiff/table_differ.go @@ -54,9 +54,9 @@ var BackgroundOperationTimeout = topo.RemoteOperationTimeout * 4 // compareColInfo contains the metadata for a column of the table being diffed type compareColInfo struct { - colIndex int // index of the column in the filter's select - collation collations.Collation // is the collation of the column, if any - isPK bool // is this column part of the primary key + colIndex int // index of the column in the filter's select + collation collations.ID // is the collation of the column, if any + isPK bool // is this column part of the primary key colName string } @@ -637,10 +637,9 @@ func (td *tableDiffer) compare(sourceRow, targetRow []sqltypes.Value, cols []com collationID collations.ID ) // If the collation is nil or unknown, use binary collation to compare as bytes. - if col.collation == nil { + collationID = col.collation + if collationID == collations.Unknown { collationID = collations.CollationBinaryID - } else { - collationID = col.collation.ID() } c, err = evalengine.NullsafeCompare(sourceRow[compareIndex], targetRow[compareIndex], collationID) if err != nil { diff --git a/go/vt/vttablet/tabletmanager/vdiff/utils.go b/go/vt/vttablet/tabletmanager/vdiff/utils.go index 34a9b3f164b..12ea1e8a68c 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/utils.go +++ b/go/vt/vttablet/tabletmanager/vdiff/utils.go @@ -40,10 +40,10 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare for i, cpk := range comparePKs { weightStringCol := -1 // if the collation is nil or unknown, use binary collation to compare as bytes - if cpk.collation == nil { + if cpk.collation == collations.Unknown { ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: collations.CollationBinaryID} } else { - ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: cpk.collation.ID()} + ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: cpk.collation} } } return &engine.MergeSort{ diff --git a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go index daf7360b895..10c6406f046 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go +++ b/go/vt/vttablet/tabletmanager/vdiff/workflow_differ_test.go @@ -483,10 +483,11 @@ func TestBuildPlanSuccess(t *testing.T) { dbc.ExpectRequestRE("select vdt.lastpk as lastpk, vdt.mismatch as mismatch, vdt.report as report", noResults, nil) columnList := make([]string, len(tcase.tablePlan.comparePKs)) collationList := make([]string, len(tcase.tablePlan.comparePKs)) + env := collations.Local() for i := range tcase.tablePlan.comparePKs { columnList[i] = tcase.tablePlan.comparePKs[i].colName - if tcase.tablePlan.comparePKs[i].collation != nil { - collationList[i] = tcase.tablePlan.comparePKs[i].collation.Name() + if tcase.tablePlan.comparePKs[i].collation != collations.Unknown { + collationList[i] = env.LookupName(tcase.tablePlan.comparePKs[i].collation) } else { collationList[i] = sqltypes.NULL.String() } diff --git a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go index fc0f0149098..2ce5a1b7720 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go +++ b/go/vt/vttablet/tabletmanager/vreplication/replicator_plan.go @@ -23,6 +23,7 @@ import ( "strings" "vitess.io/vitess/go/mysql/collations/charset" + "vitess.io/vitess/go/mysql/collations/colldata" "vitess.io/vitess/go/vt/vttablet" "google.golang.org/protobuf/proto" @@ -318,10 +319,10 @@ func (tp *TablePlan) bindFieldVal(field *querypb.Field, val *sqltypes.Value) (*q if conversion, ok := tp.ConvertCharset[field.Name]; ok && !val.IsNull() { // Non-null string value, for which we have a charset conversion instruction fromCollation := collations.Local().DefaultCollationForCharset(conversion.FromCharset) - if fromCollation == nil { + if fromCollation == collations.Unknown { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Character set %s not supported for column %s", conversion.FromCharset, field.Name) } - out, err := charset.Convert(nil, charset.Charset_utf8mb4{}, val.Raw(), fromCollation.Charset()) + out, err := charset.Convert(nil, charset.Charset_utf8mb4{}, val.Raw(), colldata.Lookup(fromCollation).Charset()) if err != nil { return nil, err } diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index fb3e9e133df..1ab4e9009c9 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -114,9 +114,9 @@ type vdiff struct { // compareColInfo contains the metadata for a column of the table being diffed type compareColInfo struct { - colIndex int // index of the column in the filter's select - collation collations.Collation // is the collation of the column, if any - isPK bool // is this column part of the primary key + colIndex int // index of the column in the filter's select + collation collations.ID // is the collation of the column, if any + isPK bool // is this column part of the primary key } // tableDiffer performs a diff for one table in the workflow. @@ -530,7 +530,7 @@ func findPKs(table *tabletmanagerdatapb.TableDefinition, targetSelect *sqlparser // getColumnCollations determines the proper collation to use for each // column in the table definition leveraging MySQL's collation inheritence // rules. -func getColumnCollations(table *tabletmanagerdatapb.TableDefinition) (map[string]collations.Collation, error) { +func getColumnCollations(table *tabletmanagerdatapb.TableDefinition) (map[string]collations.ID, error) { collationEnv := collations.Local() createstmt, err := sqlparser.Parse(table.Schema) if err != nil { @@ -548,7 +548,7 @@ func getColumnCollations(table *tabletmanagerdatapb.TableDefinition) (map[string tableCollation := tableschema.GetCollation() // If no explicit collation is specified for the column then we need // to walk the inheritence tree. - getColumnCollation := func(column *sqlparser.ColumnDefinition) collations.Collation { + getColumnCollation := func(column *sqlparser.ColumnDefinition) collations.ID { // If there's an explicit collation listed then use that. if column.Type.Options.Collate != "" { return collationEnv.LookupByName(strings.ToLower(column.Type.Options.Collate)) @@ -569,14 +569,14 @@ func getColumnCollations(table *tabletmanagerdatapb.TableDefinition) (map[string } // The table is using the global default charset and collation and // we inherite that. - return collations.Default().Get() + return collations.Default() } - columnCollations := make(map[string]collations.Collation) + columnCollations := make(map[string]collations.ID) for _, column := range tableschema.TableSpec.Columns { // If it's not a character based type then no collation is used. if !sqltypes.IsQuoted(column.Type.SQLType()) { - columnCollations[column.Name.Lowered()] = nil + columnCollations[column.Name.Lowered()] = collations.Unknown continue } columnCollations[column.Name.Lowered()] = getColumnCollation(column) @@ -784,10 +784,10 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare for _, cpk := range comparePKs { weightStringCol := -1 // if the collation is nil or unknown, use binary collation to compare as bytes - if cpk.collation == nil { + if cpk.collation == collations.Unknown { ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: collations.CollationBinaryID}) } else { - ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: cpk.collation.ID()}) + ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: cpk.collation}) } } return &engine.MergeSort{ @@ -1294,10 +1294,9 @@ func (td *tableDiffer) compare(sourceRow, targetRow []sqltypes.Value, cols []com var err error var collationID collations.ID // if the collation is nil or unknown, use binary collation to compare as bytes - if col.collation == nil { + collationID = col.collation + if col.collation == collations.Unknown { collationID = collations.CollationBinaryID - } else { - collationID = col.collation.ID() } c, err = evalengine.NullsafeCompare(sourceRow[compareIndex], targetRow[compareIndex], collationID) if err != nil { diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index df38bb5c752..ac57c9bcf68 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -91,12 +91,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { input: &binlogdatapb.Rule{ @@ -108,12 +108,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { input: &binlogdatapb.Rule{ @@ -125,12 +125,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { input: &binlogdatapb.Rule{ @@ -142,12 +142,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c2, c1 from t1 order by c1 asc", targetExpression: "select c2, c1 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), false}, {1, collations.Collation(nil), true}}, - comparePKs: []compareColInfo{{1, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Unknown, true}}, + comparePKs: []compareColInfo{{1, collations.Unknown, true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), }, }, { input: &binlogdatapb.Rule{ @@ -159,12 +159,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c0 as c1, c2 from t2 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // non-pk text column. @@ -177,12 +177,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "nonpktext", sourceExpression: "select c1, textcol from nonpktext order by c1 asc", targetExpression: "select c1, textcol from nonpktext order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // non-pk text column, different order. @@ -195,12 +195,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "nonpktext", sourceExpression: "select textcol, c1 from nonpktext order by c1 asc", targetExpression: "select textcol, c1 from nonpktext order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), false}, {1, collations.Collation(nil), true}}, - comparePKs: []compareColInfo{{1, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Unknown, true}}, + comparePKs: []compareColInfo{{1, collations.Unknown, true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Unknown, true}}), }, }, { // pk text column. @@ -213,12 +213,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "pktext", sourceExpression: "select textcol, c2 from pktext order by textcol asc", targetExpression: "select textcol, c2 from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Default().Get(), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Default().Get(), true}}, + compareCols: []compareColInfo{{0, collations.Default(), true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Default(), true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Default().Get(), false}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Default().Get(), false}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Default(), false}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Default(), false}}), }, }, { // pk text column, different order. @@ -231,12 +231,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "pktext", sourceExpression: "select c2, textcol from pktext order by textcol asc", targetExpression: "select c2, textcol from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), false}, {1, collations.Default().Get(), true}}, - comparePKs: []compareColInfo{{1, collations.Default().Get(), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Default(), true}}, + comparePKs: []compareColInfo{{1, collations.Default(), true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default().Get(), false}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default().Get(), false}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), }, }, { // text column as expression. @@ -249,12 +249,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "pktext", sourceExpression: "select c2, a + b as textcol from pktext order by textcol asc", targetExpression: "select c2, textcol from pktext order by textcol asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), false}, {1, collations.Default().Get(), true}}, - comparePKs: []compareColInfo{{1, collations.Default().Get(), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Default(), true}}, + comparePKs: []compareColInfo{{1, collations.Default(), true}}, pkCols: []int{1}, selectPks: []int{1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default().Get(), false}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default().Get(), false}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{1, collations.Default(), false}}), }, }, { input: &binlogdatapb.Rule{ @@ -265,12 +265,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "multipk", sourceExpression: "select c1, c2 from multipk order by c1 asc, c2 asc", targetExpression: "select c1, c2 from multipk order by c1 asc, c2 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), true}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}, pkCols: []int{0, 1}, selectPks: []int{0, 1}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, true}}), }, }, { // in_keyrange @@ -283,12 +283,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // in_keyrange on RHS of AND. @@ -302,12 +302,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 where c2 = 2 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // in_keyrange on LHS of AND. @@ -321,12 +321,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 where c2 = 2 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // in_keyrange on cascaded AND expression @@ -340,12 +340,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 where c2 = 2 and c1 = 1 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // in_keyrange parenthesized @@ -359,12 +359,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 where c2 = 2 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // group by @@ -377,12 +377,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "t1", sourceExpression: "select c1, c2 from t1 group by c1 order by c1 asc", targetExpression: "select c1, c2 from t1 order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { // aggregations @@ -395,8 +395,8 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "aggr", sourceExpression: "select c1, c2, count(*) as c3, sum(c4) as c4 from t1 group by c1 order by c1 asc", targetExpression: "select c1, c2, c3, c4 from aggr order by c1 asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}, {2, collations.Collation(nil), false}, {3, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}, {2, collations.Unknown, false}, {3, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, sourcePrimitive: &engine.OrderedAggregate{ @@ -405,9 +405,9 @@ func TestVDiffPlanSuccess(t *testing.T) { engine.NewAggregateParam(opcode.AggregateSum, 3, ""), }, GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1, Type: sqltypes.Unknown}}, - Input: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + Input: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }, { input: &binlogdatapb.Rule{ @@ -419,12 +419,12 @@ func TestVDiffPlanSuccess(t *testing.T) { targetTable: "datze", sourceExpression: "select id, dt from datze order by id asc", targetExpression: "select id, convert_tz(dt, 'UTC', 'US/Pacific') as dt from datze order by id asc", - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, - sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), - targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), + sourcePrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), + targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Unknown, true}}), }, }} @@ -1036,13 +1036,13 @@ func TestVDiffFindPKs(t *testing.T) { }, }, tdIn: &tableDiffer{ - compareCols: []compareColInfo{{0, collations.Collation(nil), false}, {1, collations.Collation(nil), false}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Unknown, false}}, comparePKs: []compareColInfo{}, pkCols: []int{}, }, tdOut: &tableDiffer{ - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}}, pkCols: []int{0}, selectPks: []int{0}, }, @@ -1064,13 +1064,13 @@ func TestVDiffFindPKs(t *testing.T) { }, }, tdIn: &tableDiffer{ - compareCols: []compareColInfo{{0, collations.Collation(nil), false}, {1, collations.Collation(nil), false}, {2, collations.Collation(nil), false}, {3, collations.Collation(nil), false}}, + compareCols: []compareColInfo{{0, collations.Unknown, false}, {1, collations.Unknown, false}, {2, collations.Unknown, false}, {3, collations.Unknown, false}}, comparePKs: []compareColInfo{}, pkCols: []int{}, }, tdOut: &tableDiffer{ - compareCols: []compareColInfo{{0, collations.Collation(nil), true}, {1, collations.Collation(nil), false}, {2, collations.Collation(nil), false}, {3, collations.Collation(nil), true}}, - comparePKs: []compareColInfo{{0, collations.Collation(nil), true}, {3, collations.Collation(nil), true}}, + compareCols: []compareColInfo{{0, collations.Unknown, true}, {1, collations.Unknown, false}, {2, collations.Unknown, false}, {3, collations.Unknown, true}}, + comparePKs: []compareColInfo{{0, collations.Unknown, true}, {3, collations.Unknown, true}}, pkCols: []int{0, 3}, selectPks: []int{0, 3}, }, @@ -1143,7 +1143,7 @@ func TestGetColumnCollations(t *testing.T) { tests := []struct { name string table *tabletmanagerdatapb.TableDefinition - want map[string]collations.Collation + want map[string]collations.ID wantErr bool }{ { @@ -1158,9 +1158,9 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 int, name varchar(10), primary key(c1))", }, - want: map[string]collations.Collation{ - "c1": collations.Collation(nil), - "name": collations.Default().Get(), + want: map[string]collations.ID{ + "c1": collations.Unknown, + "name": collations.Default(), }, }, { @@ -1168,9 +1168,9 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 varchar(10), name varchar(10), primary key(c1))", }, - want: map[string]collations.Collation{ - "c1": collations.Default().Get(), - "name": collations.Default().Get(), + want: map[string]collations.ID{ + "c1": collations.Default(), + "name": collations.Default(), }, }, { @@ -1178,9 +1178,9 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 int, name varchar(10), primary key(c1, name))", }, - want: map[string]collations.Collation{ - "c1": collations.Collation(nil), - "name": collations.Default().Get(), + want: map[string]collations.ID{ + "c1": collations.Unknown, + "name": collations.Default(), }, }, { @@ -1188,7 +1188,7 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 varchar(10), name varchar(10), primary key(c1)) default character set ucs2", }, - want: map[string]collations.Collation{ + want: map[string]collations.ID{ "c1": collationEnv.DefaultCollationForCharset("ucs2"), "name": collationEnv.DefaultCollationForCharset("ucs2"), }, @@ -1198,7 +1198,7 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 varchar(10), name varchar(10), primary key(c1)) charset=utf32 collate=utf32_icelandic_ci", }, - want: map[string]collations.Collation{ + want: map[string]collations.ID{ "c1": collationEnv.LookupByName("utf32_icelandic_ci"), "name": collationEnv.LookupByName("utf32_icelandic_ci"), }, @@ -1208,7 +1208,7 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 varchar(10) charset sjis, name varchar(10), primary key(c1)) character set=utf8", }, - want: map[string]collations.Collation{ + want: map[string]collations.ID{ "c1": collationEnv.DefaultCollationForCharset("sjis"), "name": collationEnv.DefaultCollationForCharset("utf8mb3"), }, @@ -1218,7 +1218,7 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 varchar(10) collate hebrew_bin, name varchar(10), primary key(c1)) charset=hebrew", }, - want: map[string]collations.Collation{ + want: map[string]collations.ID{ "c1": collationEnv.LookupByName("hebrew_bin"), "name": collationEnv.DefaultCollationForCharset("hebrew"), }, @@ -1228,9 +1228,9 @@ func TestGetColumnCollations(t *testing.T) { table: &tabletmanagerdatapb.TableDefinition{ Schema: "create table t1 (c1 varchar(10) collate utf16_turkish_ci, c2 int, name varchar(10), primary key(c1, c2)) charset=utf16 collate=utf16_icelandic_ci", }, - want: map[string]collations.Collation{ + want: map[string]collations.ID{ "c1": collationEnv.LookupByName("utf16_turkish_ci"), - "c2": collations.Collation(nil), + "c2": collations.Unknown, "name": collationEnv.LookupByName("utf16_icelandic_ci"), }, },