Skip to content

Commit

Permalink
[fix](memory) Fix jemalloc hook deadlock at BE start (#44009)
Browse files Browse the repository at this point in the history
It should be caused by the third-party library, but in any case, this
fix should be possible.

```
#0  0x00007f738432318c in __lll_lock_wait_private () from /lib64/libc.so.6
#1  0x00007f73843505ed in __internal_atexit () from /lib64/libc.so.6
#2  0x0000563f6a54db30 in malloc ()
#3  0x0000563f74e82b08 in operator new(unsigned long) ()
#4  0x0000563f6a6144bd in doris::ThreadContextPtr::ThreadContextPtr() ()
#5  0x0000563f6a54e21c in doris_calloc ()
#6  0x00007f7384350557 in __new_exitfn () from /lib64/libc.so.6
#7  0x00007f73843505fc in __internal_atexit () from /lib64/libc.so.6
#8  0x0000563f72c79c03 in ?? ()
#9  0x0000563f74e3b5fd in __libc_csu_init ()
#10 0x00007f7384339d18 in __libc_start_main () from /lib64/libc.so.6
#11 0x0000563f69ae302a in _start ()
```
  • Loading branch information
xinyiZzz authored Nov 16, 2024
1 parent 466c309 commit c2732d4
Show file tree
Hide file tree
Showing 4 changed files with 9 additions and 4 deletions.
1 change: 1 addition & 0 deletions be/src/common/daemon.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,7 @@ static void init_doris_metrics(const std::vector<StorePath>& store_paths) {
void signal_handler(int signal) {
if (signal == SIGINT || signal == SIGTERM) {
k_doris_exit = true;
k_doris_start = false;
LOG(INFO) << "doris start to exit";
}
}
Expand Down
1 change: 1 addition & 0 deletions be/src/common/daemon.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ namespace doris {

struct StorePath;
inline bool k_doris_exit = false;
inline bool k_doris_start = false;

class Daemon {
public:
Expand Down
9 changes: 5 additions & 4 deletions be/src/runtime/thread_context.h
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ class MemTracker;
class RuntimeState;

extern bool k_doris_exit;
extern bool k_doris_start;
extern bthread_key_t btls_key;

// Using gcc11 compiles thread_local variable on lower versions of GLIBC will report an error,
Expand Down Expand Up @@ -388,17 +389,17 @@ class AddThreadMemTrackerConsumer {
// which is different from the previous behavior.
#define CONSUME_MEM_TRACKER(size) \
do { \
if (doris::thread_context_ptr.init) { \
if (doris::k_doris_start && doris::thread_context_ptr.init) { \
doris::thread_context()->consume_memory(size); \
} else if (doris::ExecEnv::GetInstance()->initialized()) { \
} else if (doris::k_doris_start && doris::ExecEnv::GetInstance()->initialized()) { \
doris::ExecEnv::GetInstance()->orphan_mem_tracker_raw()->consume_no_update_peak(size); \
} \
} while (0)
#define RELEASE_MEM_TRACKER(size) \
do { \
if (doris::thread_context_ptr.init) { \
if (doris::k_doris_start && doris::thread_context_ptr.init) { \
doris::thread_context()->consume_memory(-size); \
} else if (doris::ExecEnv::GetInstance()->initialized()) { \
} else if (doris::k_doris_start && doris::ExecEnv::GetInstance()->initialized()) { \
doris::ExecEnv::GetInstance()->orphan_mem_tracker_raw()->consume_no_update_peak( \
-size); \
} \
Expand Down
2 changes: 2 additions & 0 deletions be/src/service/doris_main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ int __llvm_profile_write_file();

namespace doris {
extern bool k_doris_exit;
extern bool k_doris_start;

static void thrift_output(const char* x) {
LOG(WARNING) << "thrift internal message: " << x;
Expand Down Expand Up @@ -463,6 +464,7 @@ int main(int argc, char** argv) {

// init exec env
auto exec_env = doris::ExecEnv::GetInstance();
doris::k_doris_start = true;
doris::ExecEnv::init(exec_env, paths);
doris::TabletSchemaCache::create_global_schema_cache();

Expand Down

0 comments on commit c2732d4

Please sign in to comment.