Skip to content

Commit

Permalink
[fix](string function) fix coredump cause by not supported charset of…
Browse files Browse the repository at this point in the history
… convert function (apache#17032)

sql select convert('a' using utf8) cause coredump of BE:

Thread 338 "FragmentMgrThre" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7f0b2c21a700 (LWP 2865259)]
0x00007fff04f8a015 in __gconv_close () from /lib64/libc.so.6
(gdb) bt
#0  0x00007fff04f8a015 in __gconv_close () from /lib64/libc.so.6
#1  0x00007fff04f899e3 in iconv_close () from /lib64/libc.so.6
#2  0x000055556f92f5a7 in doris::vectorized::FunctionConvertTo::close (this=0x606003f28f50, context=0x6020039fe710, 
    scope=doris_udf::FunctionContext::THREAD_LOCAL) at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/functions/function_string.h:2442
apache#3  0x000055556db0598d in doris::vectorized::DefaultFunction::close (this=0x607000748f00, context=0x6020039fe710, 
    scope=doris_udf::FunctionContext::THREAD_LOCAL) at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/functions/function.h:524
apache#4  0x000055556da00b66 in doris::vectorized::VExpr::close_function_context (this=0x619004ca1d80, context=0x606003fad300, 
    scope=doris_udf::FunctionContext::FRAGMENT_LOCAL, function=...) at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/exprs/vexpr.cpp:373
apache#5  0x000055556da16455 in doris::vectorized::VectorizedFnCall::close (this=0x619004ca1d80, state=0x61c00026c880, context=0x606003fad300, 
    scope=doris_udf::FunctionContext::FRAGMENT_LOCAL) at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/exprs/vectorized_fn_call.cpp:92
apache#6  0x000055556da26afd in doris::vectorized::VExprContext::close (this=0x606003fad300, state=0x61c00026c880)
    at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/exprs/vexpr_context.cpp:72
apache#7  0x000055556d9fcff1 in doris::vectorized::VExpr::close (ctxs=..., state=0x61c00026c880)
    at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/exprs/vexpr.cpp:242
apache#8  0x000055556aa32c9a in doris::vectorized::VSortExecExprs::close (this=0x618000345f00, state=0x61c00026c880)
    at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/common/sort/vsort_exec_exprs.cpp:77
apache#9  0x000055556aa26dae in doris::vectorized::VSortNode::close (this=0x618000345c80, state=0x61c00026c880)
    at /mnt/disk2/tengjianping/doris-1.2/be/src/vec/exec/vsort_node.cpp:129
apache#10 0x000055556794b5ef in doris::PlanFragmentExecutor::close (this=0x6170005a35f0)
    at /mnt/disk2/tengjianping/doris-1.2/be/src/runtime/plan_fragment_executor.cpp:676
apache#11 0x00005555678ce1fb in doris::FragmentExecState::execute (this=0x6170005a3580)
    at /mnt/disk2/tengjianping/doris-1.2/be/src/runtime/fragment_mgr.cpp:258
apache#12 0x00005555678d6633 in doris::FragmentMgr::_exec_actual(std::shared_ptr<doris::FragmentExecState>, std::function<void (doris::PlanFragmentExecutor*)>) (this=0x627000003900, exec_state=..., cb=...) at /mnt/disk2/tengjianping/doris-1.2/be/src/runtime/fragment_mgr.cpp:497
apache#13 0x00005555678d8aa3 in operator() (__closure=0x60800014cda0) at /mnt/disk2/tengjianping/doris-1.2/be/src/runtime/fragment_mgr.cpp:721
  • Loading branch information
jacktengg authored Feb 22, 2023
1 parent e294248 commit dc21fcb
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 7 deletions.
17 changes: 10 additions & 7 deletions be/src/vec/functions/function_string.h
Original file line number Diff line number Diff line change
Expand Up @@ -2387,15 +2387,15 @@ class FunctionConvertTo : public IFunction {
const auto& character_data = context->get_constant_col(1)->column_ptr->get_data_at(0);
if (doris::iequal(character_data.to_string(), "gbk")) {
iconv_t cd = iconv_open("gb2312", "utf-8");
if (cd == nullptr) {
return Status::RuntimeError("function {} is convert to gbk failed in iconv_open",
if (cd == (iconv_t)-1) {
LOG(WARNING) << "iconv_open error: " << strerror(errno);
return Status::RuntimeError("function {} converting to gbk failed in iconv_open",
get_name());
}
context->set_function_state(scope, cd);
} else {
return Status::RuntimeError(
"Illegal second argument column of function convert. now only support "
"convert to character set of gbk");
return Status::RuntimeError("Not supported: convert to character set " +
character_data.to_string());
}

return Status::OK();
Expand Down Expand Up @@ -2425,7 +2425,8 @@ class FunctionConvertTo : public IFunction {
char* in = const_cast<char*>(value_data);
out_len = in_len;
if (iconv(cd, &in, &in_len, &out, &out_len) == -1) {
return Status::RuntimeError("function {} is convert to gbk failed in iconv",
LOG(WARNING) << "iconv failed, error: " << strerror(errno);
return Status::RuntimeError("function {} converting to gbk failed in iconv",
get_name());
} else {
res_offset[i] = res_chars.size();
Expand All @@ -2439,7 +2440,9 @@ class FunctionConvertTo : public IFunction {
if (scope == FunctionContext::THREAD_LOCAL) {
iconv_t cd = reinterpret_cast<iconv_t>(
context->get_function_state(FunctionContext::THREAD_LOCAL));
iconv_close(cd);
if (cd) {
iconv_close(cd);
}
context->set_function_state(FunctionContext::THREAD_LOCAL, nullptr);
}
return Status::OK();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !convert_const_to_gbk --
a ˿�

Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_convert") {
test {
sql "select convert('a' using utf8);"
exception "errCode = 2, detailMessage = Not supported"
}

qt_convert_const_to_gbk """select convert("a" using gbk), convert("丝" using gbk);"""
}

0 comments on commit dc21fcb

Please sign in to comment.