From 6425ce8a8912089bd5276a14a55fbfbfc33fa728 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Mon, 1 Jul 2024 23:11:13 +0800 Subject: [PATCH] [Fix](Prepared Statment) use fixed charset to init StringLiteral (#37084) picked from #36860 --- .../org/apache/doris/analysis/StringLiteral.java | 5 ++++- .../trees/expressions/literal/Literal.java | 9 +++++++-- .../apache/doris/qe/MysqlConnectProcessor.java | 4 +++- .../data/prepared_stmt_p0/prepared_stmt.out | 9 ++++++--- .../suites/prepared_stmt_p0/prepared_stmt.groovy | 16 +++++++++++++++- 5 files changed, 35 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 36e980ba4b2909..d5083049d23adf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -40,6 +40,7 @@ import java.io.UnsupportedEncodingException; import java.math.BigDecimal; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Objects; public class StringLiteral extends LiteralExpr { @@ -346,7 +347,9 @@ public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { } byte[] bytes = new byte[strLen]; data.get(bytes); - value = new String(bytes); + // ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in + // different environment + value = new String(bytes, StandardCharsets.UTF_8); if (LOG.isDebugEnabled()) { LOG.debug("parsed value '{}'", value); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java index 4619e35d5e06b0..fe53cbf0e2ab46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java @@ -45,6 +45,7 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -582,7 +583,9 @@ private static Literal handleStringLiteral(ByteBuffer data) { strLen = Math.min(strLen, data.remaining()); byte[] bytes = new byte[strLen]; data.get(bytes); - return new StringLiteral(new String(bytes)); + // ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in + // different environment + return new StringLiteral(new String(bytes, StandardCharsets.UTF_8)); } private static Literal handleVarcharLiteral(ByteBuffer data) { @@ -590,6 +593,8 @@ private static Literal handleVarcharLiteral(ByteBuffer data) { strLen = Math.min(strLen, data.remaining()); byte[] bytes = new byte[strLen]; data.get(bytes); - return new VarcharLiteral(new String(bytes)); + // ATTN: use fixed StandardCharsets.UTF_8 to avoid unexpected charset in + // different environment + return new VarcharLiteral(new String(bytes, StandardCharsets.UTF_8)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java index 6637f62f1efad0..df732f93c0426f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java @@ -213,7 +213,9 @@ private void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedS // process COM_EXECUTE, parse binary row data // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html private void handleExecute() { - // debugPacket(); + if (LOG.isDebugEnabled()) { + debugPacket(); + } packetBuf = packetBuf.order(ByteOrder.LITTLE_ENDIAN); // parse stmt_id, flags, params int stmtId = packetBuf.getInt(); diff --git a/regression-test/data/prepared_stmt_p0/prepared_stmt.out b/regression-test/data/prepared_stmt_p0/prepared_stmt.out index c95f732edf514a..257a5c62203166 100644 --- a/regression-test/data/prepared_stmt_p0/prepared_stmt.out +++ b/regression-test/data/prepared_stmt_p0/prepared_stmt.out @@ -74,6 +74,12 @@ a -- !select7 -- 2 1 user1 \N 1111111 1111111 +-- !select6_1 -- +2 1 user1 \N 1234.1111 xxxlalala + +-- !select7_1 -- +2 1 user1 \N 1111111 1111111 + -- !select8 -- 1 1 @@ -99,6 +105,3 @@ a 1236 100320.111390000 laa ddd Will we ignore LIMIT ?,? 2021-01-01 2020-01-01T12:36:38 2.7692 6022-01-01 [null] 1237 120939.111300000 a ddd Will we ignore LIMIT ?,? 2021-01-01 2020-01-01T12:36:38 22.822 7022-01-01 ["2025-01-01 11:30:38"] --- !select16 -- -mytable1 CREATE TABLE `mytable1` (\n `siteid` INT NULL DEFAULT "10",\n `citycode` SMALLINT NULL,\n `username` VARCHAR(32) NULL DEFAULT "",\n `pv` BIGINT SUM NULL DEFAULT "0"\n) ENGINE=OLAP\nAGGREGATE KEY(`siteid`, `citycode`, `username`)\nDISTRIBUTED BY HASH(`siteid`) BUCKETS 10\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V2",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); - diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index 522062e603a499..0ed0a48e65c847 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -57,6 +57,7 @@ suite("test_prepared_stmt", "nonConcurrent") { qt_sql """select * from ${tableName} order by 1, 2, 3""" qt_sql """select * from ${tableName} order by 1, 2, 3""" + sql "set global max_prepared_stmt_count = 10000" def stmt_read = prepareStatement "select * from ${tableName} where k1 = ? order by k1" assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); @@ -103,6 +104,7 @@ suite("test_prepared_stmt", "nonConcurrent") { sql """insert into mytable1 values(1,1,'user1',10);""" sql """insert into mytable1 values(1,1,'user1',10);""" sql "sync" + stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where citycode = ?) AS `SpotfireCustomQuery1` WHERE 1 = 1" assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); stmt_read.setInt(1, 12345) @@ -147,6 +149,9 @@ suite("test_prepared_stmt", "nonConcurrent") { qe_select5 stmt_read sql """insert into mytable1 values(2,1,'user1',null);""" + + // sql "set experimental_enable_nereids_planner = false" + stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where pv is null) AS `SpotfireCustomQuery1` WHERE 1 = 1" assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement); stmt_read.setString(1, "xxxlalala") @@ -155,7 +160,16 @@ suite("test_prepared_stmt", "nonConcurrent") { stmt_read.setString(1, "1111111") stmt_read.setString(2, "1111111") qe_select7 stmt_read - stmt_read.close() + // stmt_read.close() + + // sql "set experimental_enable_nereids_planner = true" + + stmt_read.setString(1, "xxxlalala") + stmt_read.setDouble(2, 1234.1111) + qe_select6_1 stmt_read + stmt_read.setString(1, "1111111") + stmt_read.setString(2, "1111111") + qe_select7_1 stmt_read stmt_read = prepareStatement "SELECT COUNT() from mytable1 WHERE citycode = ? GROUP BY siteid" assertEquals(stmt_read.class, com.mysql.cj.jdbc.ServerPreparedStatement);