Skip to content

Commit

Permalink
[fix](memory) Refactor MemCounter (#40542)
Browse files Browse the repository at this point in the history 
1. Add a new class MemCounter.
2. MemTracker and MemTrackerLimiter no longer have inheritance and
dependencies.
3. MemTrackerLimiter is used to count process memory, the BE web page
`/mem_tracker` also only displays MemTrackerLimiter.
4. MemTracker is used to count the memory of operators and some data
structures. It is not counted in the process memory and is used for
logic control and profile.

In addition, it seems that the crash is caused by memory abnormality,
not sure whether this PR can fix the problem, but this will help locate
the problem and more elegant.

```
==6641==ERROR: AddressSanitizer: heap-use-after-free on address 0x603000980e70 at pc 0x559be21880be bp 0x7fd7cfc75070 sp 0x7fd7cfc75068
READ of size 8 at 0x603000980e70 thread T1453 (memory_maintena)
    #0 0x559be21880bd in std::__atomic_base<long>::load(std::memory_order) const /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/atomic_base.h:481:9
    #1 0x559be21880bd in doris::MemTracker::MemCounter::current_value() const /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker.h:139:63
    #2 0x559be21880bd in doris::MemTracker::consumption() const /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker.h:165:56
    #3 0x559be3985133 in doris::MemTrackerLimiter::refresh_global_counter() /home/zcp/repo_center/doris_master/doris/be/src/runtime/memory/mem_tracker_limiter.cpp:245:59
    #4 0x559bdfcd52d5 in doris::Daemon::memory_maintenance_thread() /home/zcp/repo_center/doris_master/doris/be/src/common/daemon.cpp:239:13
    #5 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5
    #6 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8
    #7 0x7fdf5647584f  misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81

0x603000980e74 is located 0 bytes after 20-byte region [0x603000980e60,0x603000980e74)
04:19:48   freed by thread T1491 (Scan_normal [wo) here:
04:19:48       #0 0x559bdfb0dd9d in operator delete(void*) (/mnt/hdd01/ci/master-deploy/be/lib/doris_be+0x33546d9d) (BuildId: f170e92ad3c55512)
04:19:48       #1 0x559bdfb1d42b in __gnu_cxx::new_allocator<char>::deallocate(char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/ext/new_allocator.h:139:2
04:19:48       #2 0x559bdfb1d42b in std::allocator<char>::deallocate(char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/allocator.h:187:27
04:19:48       #3 0x559bdfb1d42b in std::allocator_traits<std::allocator<char>>::deallocate(std::allocator<char>&, char*, unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:492:13
04:19:48       #4 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_destroy(unsigned long) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:237:9
04:19:48       #5 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_dispose() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:232:4
04:19:48       #6 0x559bdfb1d42b in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::~basic_string() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:658:9
04:19:48       #7 0x559bdfb22501 in void std::destroy_at<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:88:15
04:19:48       #8 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:138:7
04:19:48       #9 0x559bdfb22501 in void std::_Destroy_aux<false>::__destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:152:6
04:19:48       #10 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:184:7
04:19:48       #11 0x559bdfb22501 in void std::_Destroy<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:746:7
04:19:48       #12 0x559bdfb22501 in std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::~vector() /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_vector.h:680:2
04:19:48       #13 0x559be5515262 in doris::TCondition::~TCondition() /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/PaloInternalService_types.cpp:9867:1
04:19:48       #14 0x559be5515262 in doris::TCondition::~TCondition() /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/PaloInternalService_types.cpp:9866:36
04:19:48       #15 0x559be0047ee6 in doris::Status doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB>(std::shared_ptr<doris::TabletSchema>, std::shared_ptr<doris::TabletSchema>, google::protobuf::RepeatedPtrField<doris::DeleteSubPredicatePB> const&, doris::DeleteConditions*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:372:5
04:19:48       #16 0x559be003ba59 in doris::DeleteHandler::init(std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetMeta>, std::allocator<std::shared_ptr<doris::RowsetMeta>>> const&, long, bool) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:404:13
04:19:48       #17 0x559be30ed6e3 in doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:636:28
04:19:48       #18 0x559be30e23d1 in doris::TabletReader::_init_params(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:290:18
04:19:48       #19 0x559be30e1592 in doris::TabletReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:124:18
04:19:48       #20 0x559c129840c8 in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/vec/olap/block_reader.cpp:210:5
04:19:48       #21 0x559c15a4ea09 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:232:32
04:19:48       #22 0x559bfb4acd94 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:236:5
04:19:48       #23 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:176:21
04:19:48       #24 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:175:31
04:19:48       #25 0x559bfb4b10be in void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14
04:19:48       #26 0x559bfb4b10be in std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2
04:19:48       #27 0x559bfb4b10be in std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9
04:19:48       #28 0x559be410ed1b in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:543:24
04:19:48       #29 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5
04:19:48       #30 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8
04:19:48   
04:19:48   previously allocated by thread T1491 (Scan_normal [wo) here:
04:19:48       #0 0x559bdfb0d53d in operator new(unsigned long) (/mnt/hdd01/ci/master-deploy/be/lib/doris_be+0x3354653d) (BuildId: f170e92ad3c55512)
04:19:48       #1 0x559bdfb33f74 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct<char*>(char*, char*, std::forward_iterator_tag) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.tcc:219:14
04:19:48       #2 0x559bdfb32705 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct_aux<char*>(char*, char*, std::__false_type) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:247:11
04:19:48       #3 0x559bdfb32705 in void std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::_M_construct<char*>(char*, char*) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:266:4
04:19:48       #4 0x559bdfb32705 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>::basic_string(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/basic_string.h:451:9
04:19:48       #5 0x559bdfc326e6 in decltype(::new((void*)(0)) std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>(std::declval<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>())) std::construct_at<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/stl_construct.h:97:39
04:19:48       #6 0x559bdfc326e6 in void std::allocator_traits<std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::construct<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/alloc_traits.h:514:4
04:19:48       #7 0x559bdfc326e6 in void std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>::_M_realloc_insert<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&>(__gnu_cxx::__normal_iterator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>*, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>>>>>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char>> const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/vector.tcc:449:4
04:19:48       #8 0x559be0032206 in doris::DeleteHandler::parse_condition(doris::DeleteSubPredicatePB const&, doris::TCondition*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:300:33
04:19:48       #9 0x559be0047aa3 in doris::Status doris::DeleteHandler::_parse_column_pred<doris::DeleteSubPredicatePB>(std::shared_ptr<doris::TabletSchema>, std::shared_ptr<doris::TabletSchema>, google::protobuf::RepeatedPtrField<doris::DeleteSubPredicatePB> const&, doris::DeleteConditions*) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:355:9
04:19:48       #10 0x559be003ba59 in doris::DeleteHandler::init(std::shared_ptr<doris::TabletSchema>, std::vector<std::shared_ptr<doris::RowsetMeta>, std::allocator<std::shared_ptr<doris::RowsetMeta>>> const&, long, bool) /home/zcp/repo_center/doris_master/doris/be/src/olap/delete_handler.cpp:404:13
04:19:48       #11 0x559be30ed6e3 in doris::TabletReader::_init_delete_condition(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:636:28
04:19:48       #12 0x559be30e23d1 in doris::TabletReader::_init_params(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:290:18
04:19:48       #13 0x559be30e1592 in doris::TabletReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/olap/tablet_reader.cpp:124:18
04:19:48       #14 0x559c129840c8 in doris::vectorized::BlockReader::init(doris::TabletReader::ReaderParams const&) /home/zcp/repo_center/doris_master/doris/be/src/vec/olap/block_reader.cpp:210:5
04:19:48       #15 0x559c15a4ea09 in doris::vectorized::NewOlapScanner::open(doris::RuntimeState*) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/new_olap_scanner.cpp:232:32
04:19:48       #16 0x559bfb4acd94 in doris::vectorized::ScannerScheduler::_scanner_scan(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>) /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:236:5
04:19:48       #17 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:176:21
04:19:48       #18 0x559bfb4b10be in doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()::operator()() const /home/zcp/repo_center/doris_master/doris/be/src/vec/exec/scan/scanner_scheduler.cpp:175:31
04:19:48       #19 0x559bfb4b10be in void std::__invoke_impl<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(std::__invoke_other, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:61:14
04:19:48       #20 0x559bfb4b10be in std::enable_if<is_invocable_r_v<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>, void>::type std::__invoke_r<void, doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&>(doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/invoke.h:111:2
04:19:48       #21 0x559bfb4b10be in std::_Function_handler<void (), doris::vectorized::ScannerScheduler::submit(std::shared_ptr<doris::vectorized::ScannerContext>, std::shared_ptr<doris::vectorized::ScanTask>)::$_1::operator()() const::'lambda'()>::_M_invoke(std::_Any_data const&) /var/local/ldb-toolchain/bin/../lib/gcc/x86_64-linux-gnu/11/../../../../include/c++/11/bits/std_function.h:291:9
04:19:48       #22 0x559be410ed1b in doris::ThreadPool::dispatch_thread() /home/zcp/repo_center/doris_master/doris/be/src/util/threadpool.cpp:543:24
04:19:48       #23 0x559be40e6c17 in doris::Thread::supervise_thread(void*) /home/zcp/repo_center/doris_master/doris/be/src/util/thread.cpp:498:5
04:19:48       #24 0x7fdf563e3ac2 in start_thread nptl/pthread_create.c:442:8
```
  • Loading branch information
@xinyiZzz
xinyiZzz authored Sep 16, 2024
1 parent b92b63e commit b58570a
Show file tree
Hide file tree
Showing 49 changed files with 623 additions and 791 deletions.
2 changes: 1 addition & 1 deletion be/src/cloud/cloud_tablet_mgr.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ class CloudTabletMgr::TabletMap {
CloudTabletMgr::CloudTabletMgr(CloudStorageEngine& engine)
: _engine(engine),
_tablet_map(std::make_unique<TabletMap>()),
_cache(std::make_unique<LRUCachePolicyTrackingManual>(
_cache(std::make_unique<LRUCachePolicy>(
CachePolicy::CacheType::CLOUD_TABLET_CACHE, config::tablet_cache_capacity,
LRUCacheType::NUMBER, 0, config::tablet_cache_shards)) {}

Expand Down
4 changes: 2 additions & 2 deletions be/src/cloud/cloud_txn_delete_bitmap_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@
namespace doris {

CloudTxnDeleteBitmapCache::CloudTxnDeleteBitmapCache(size_t size_in_bytes)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE,
size_in_bytes, LRUCacheType::SIZE, 86400, 4),
: LRUCachePolicy(CachePolicy::CacheType::CLOUD_TXN_DELETE_BITMAP_CACHE, size_in_bytes,
LRUCacheType::SIZE, 86400, 4),
_stop_latch(1) {}

CloudTxnDeleteBitmapCache::~CloudTxnDeleteBitmapCache() {
Expand Down
2 changes: 1 addition & 1 deletion be/src/cloud/cloud_txn_delete_bitmap_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
namespace doris {

// Record transaction related delete bitmaps using a lru cache.
class CloudTxnDeleteBitmapCache : public LRUCachePolicyTrackingManual {
class CloudTxnDeleteBitmapCache : public LRUCachePolicy {
public:
CloudTxnDeleteBitmapCache(size_t size_in_bytes);

Expand Down
9 changes: 4 additions & 5 deletions be/src/http/default_path_handlers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ void display_tablets_callback(const WebPageHandler::ArgumentMap& args, EasyJson*
// Registered to handle "/mem_tracker", and prints out memory tracker information.
void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstream* output) {
(*output) << "<h1>Memory usage by subsystem</h1>\n";
std::vector<MemTracker::Snapshot> snapshots;
std::vector<MemTrackerLimiter::Snapshot> snapshots;
auto iter = args.find("type");
if (iter != args.end()) {
if (iter->second == "global") {
Expand All @@ -159,7 +159,7 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
} else if (iter->second == "other") {
MemTrackerLimiter::make_type_snapshots(&snapshots, MemTrackerLimiter::Type::OTHER);
} else if (iter->second == "reserved_memory") {
GlobalMemoryArbitrator::make_reserved_memory_snapshots(&snapshots);
MemTrackerLimiter::make_all_reserved_trackers_snapshots(&snapshots);
} else if (iter->second == "all") {
MemTrackerLimiter::make_all_memory_state_snapshots(&snapshots);
}
Expand Down Expand Up @@ -191,7 +191,6 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
(*output) << "<thead><tr>"
"<th data-sortable='true'>Type</th>"
"<th data-sortable='true'>Label</th>"
"<th data-sortable='true'>Parent Label</th>"
"<th>Limit</th>"
"<th data-sortable='true' "
">Current Consumption(Bytes)</th>"
Expand All @@ -207,8 +206,8 @@ void mem_tracker_handler(const WebPageHandler::ArgumentMap& args, std::stringstr
string peak_consumption_normalize = AccurateItoaKMGT(item.peak_consumption);
(*output) << strings::Substitute(
"<tr><td>$0</td><td>$1</td><td>$2</td><td>$3</td><td>$4</td><td>$5</td><td>$6</"
"td><td>$7</td></tr>\n",
item.type, item.label, item.parent_label, limit_str, item.cur_consumption,
"td></tr>\n",
item.type, item.label, limit_str, item.cur_consumption,
current_consumption_normalize, item.peak_consumption, peak_consumption_normalize);
}
(*output) << "</tbody></table>\n";
Expand Down
5 changes: 2 additions & 3 deletions be/src/olap/memtable.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,9 +161,8 @@ MemTable::~MemTable() {
std::for_each(_row_in_blocks.begin(), _row_in_blocks.end(), std::default_delete<RowInBlock>());
_insert_mem_tracker->release(_mem_usage);
_flush_mem_tracker->set_consumption(0);
DCHECK_EQ(_insert_mem_tracker->consumption(), 0)
<< std::endl
<< MemTracker::log_usage(_insert_mem_tracker->make_snapshot());
DCHECK_EQ(_insert_mem_tracker->consumption(), 0) << std::endl
<< _insert_mem_tracker->log_usage();
DCHECK_EQ(_flush_mem_tracker->consumption(), 0);
_arena.reset();
_agg_buffer_pool.clear();
Expand Down
5 changes: 1 addition & 4 deletions be/src/olap/memtable_memory_limiter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,10 +62,7 @@ Status MemTableMemoryLimiter::init(int64_t process_mem_limit) {
_load_hard_mem_limit * config::load_process_safe_mem_permit_percent / 100;
g_load_hard_mem_limit.set_value(_load_hard_mem_limit);
g_load_soft_mem_limit.set_value(_load_soft_mem_limit);
_memtable_tracker_set =
MemTrackerLimiter::create_shared(MemTrackerLimiter::Type::LOAD, "MemTableTrackerSet");
_mem_tracker = std::make_unique<MemTracker>("AllMemTableMemory",
ExecEnv::GetInstance()->details_mem_tracker_set());
_mem_tracker = std::make_unique<MemTracker>("AllMemTableMemory");
REGISTER_HOOK_METRIC(memtable_memory_limiter_mem_consumption,
[this]() { return _mem_tracker->consumption(); });
_log_timer.start();
Expand Down
5 changes: 1 addition & 4 deletions be/src/olap/memtable_memory_limiter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
#include <stdint.h>

#include "common/status.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "runtime/memory/mem_tracker.h"
#include "util/countdown_latch.h"
#include "util/stopwatch.hpp"

Expand All @@ -45,7 +45,6 @@ class MemTableMemoryLimiter {

void refresh_mem_tracker();

MemTrackerLimiter* memtable_tracker_set() { return _memtable_tracker_set.get(); }
MemTracker* mem_tracker() { return _mem_tracker.get(); }

int64_t mem_usage() const { return _mem_usage; }
Expand All @@ -68,8 +67,6 @@ class MemTableMemoryLimiter {
int64_t _write_mem_usage = 0;
int64_t _active_mem_usage = 0;

// mem tracker collection of all mem tables.
std::shared_ptr<MemTrackerLimiter> _memtable_tracker_set;
// sum of all mem table memory.
std::unique_ptr<MemTracker> _mem_tracker;
int64_t _load_hard_mem_limit = -1;
Expand Down
13 changes: 0 additions & 13 deletions be/src/olap/memtable_writer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,25 +187,12 @@ Status MemTableWriter::wait_flush() {
}

void MemTableWriter::_reset_mem_table() {
#ifndef BE_TEST
auto mem_table_insert_tracker = std::make_shared<MemTracker>(
fmt::format("MemTableManualInsert:TabletId={}:MemTableNum={}#loadID={}",
std::to_string(tablet_id()), _mem_table_num,
UniqueId(_req.load_id).to_string()),
ExecEnv::GetInstance()->memtable_memory_limiter()->memtable_tracker_set());
auto mem_table_flush_tracker = std::make_shared<MemTracker>(
fmt::format("MemTableHookFlush:TabletId={}:MemTableNum={}#loadID={}",
std::to_string(tablet_id()), _mem_table_num++,
UniqueId(_req.load_id).to_string()),
ExecEnv::GetInstance()->memtable_memory_limiter()->memtable_tracker_set());
#else
auto mem_table_insert_tracker = std::make_shared<MemTracker>(fmt::format(
"MemTableManualInsert:TabletId={}:MemTableNum={}#loadID={}",
std::to_string(tablet_id()), _mem_table_num, UniqueId(_req.load_id).to_string()));
auto mem_table_flush_tracker = std::make_shared<MemTracker>(fmt::format(
"MemTableHookFlush:TabletId={}:MemTableNum={}#loadID={}", std::to_string(tablet_id()),
_mem_table_num++, UniqueId(_req.load_id).to_string()));
#endif
{
std::lock_guard<SpinLock> l(_mem_table_tracker_lock);
_mem_table_insert_trackers.push_back(mem_table_insert_tracker);
Expand Down
26 changes: 13 additions & 13 deletions be/src/olap/page_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,28 +92,28 @@ class StoragePageCache {
}
};

class DataPageCache : public LRUCachePolicyTrackingAllocator {
class DataPageCache : public LRUCachePolicy {
public:
DataPageCache(size_t capacity, uint32_t num_shards)
: LRUCachePolicyTrackingAllocator(
CachePolicy::CacheType::DATA_PAGE_CACHE, capacity, LRUCacheType::SIZE,
config::data_page_cache_stale_sweep_time_sec, num_shards) {}
: LRUCachePolicy(CachePolicy::CacheType::DATA_PAGE_CACHE, capacity,
LRUCacheType::SIZE, config::data_page_cache_stale_sweep_time_sec,
num_shards) {}
};

class IndexPageCache : public LRUCachePolicyTrackingAllocator {
class IndexPageCache : public LRUCachePolicy {
public:
IndexPageCache(size_t capacity, uint32_t num_shards)
: LRUCachePolicyTrackingAllocator(
CachePolicy::CacheType::INDEXPAGE_CACHE, capacity, LRUCacheType::SIZE,
config::index_page_cache_stale_sweep_time_sec, num_shards) {}
: LRUCachePolicy(CachePolicy::CacheType::INDEXPAGE_CACHE, capacity,
LRUCacheType::SIZE, config::index_page_cache_stale_sweep_time_sec,
num_shards) {}
};

class PKIndexPageCache : public LRUCachePolicyTrackingAllocator {
class PKIndexPageCache : public LRUCachePolicy {
public:
PKIndexPageCache(size_t capacity, uint32_t num_shards)
: LRUCachePolicyTrackingAllocator(
CachePolicy::CacheType::PK_INDEX_PAGE_CACHE, capacity, LRUCacheType::SIZE,
config::pk_index_page_cache_stale_sweep_time_sec, num_shards) {}
: LRUCachePolicy(CachePolicy::CacheType::PK_INDEX_PAGE_CACHE, capacity,
LRUCacheType::SIZE,
config::pk_index_page_cache_stale_sweep_time_sec, num_shards) {}
};

static constexpr uint32_t kDefaultNumShards = 16;
Expand Down Expand Up @@ -164,7 +164,7 @@ class StoragePageCache {
// delete bitmap in unique key with mow
std::unique_ptr<PKIndexPageCache> _pk_index_page_cache;

LRUCachePolicyTrackingAllocator* _get_page_cache(segment_v2::PageTypePB page_type) {
LRUCachePolicy* _get_page_cache(segment_v2::PageTypePB page_type) {
switch (page_type) {
case segment_v2::DATA_PAGE: {
return _data_page_cache.get();
Expand Down
6 changes: 3 additions & 3 deletions be/src/olap/rowset/segment_v2/inverted_index_cache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,9 +135,9 @@ void InvertedIndexQueryCache::insert(const CacheKey& key, std::shared_ptr<roarin
return;
}

auto* lru_handle = LRUCachePolicyTrackingManual::insert(
key.encode(), (void*)cache_value_ptr.release(), bitmap->getSizeInBytes(),
bitmap->getSizeInBytes(), CachePriority::NORMAL);
auto* lru_handle = LRUCachePolicy::insert(key.encode(), (void*)cache_value_ptr.release(),
bitmap->getSizeInBytes(), bitmap->getSizeInBytes(),
CachePriority::NORMAL);
*handle = InvertedIndexQueryCacheHandle(this, lru_handle);
}

Expand Down
31 changes: 15 additions & 16 deletions be/src/olap/rowset/segment_v2/inverted_index_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,23 +99,23 @@ class InvertedIndexSearcherCache {
private:
InvertedIndexSearcherCache() = default;

class InvertedIndexSearcherCachePolicy : public LRUCachePolicyTrackingManual {
class InvertedIndexSearcherCachePolicy : public LRUCachePolicy {
public:
InvertedIndexSearcherCachePolicy(size_t capacity, uint32_t num_shards,
uint32_t element_count_capacity)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE,
capacity, LRUCacheType::SIZE,
config::inverted_index_cache_stale_sweep_time_sec,
num_shards, element_count_capacity, true) {}
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
LRUCacheType::SIZE,
config::inverted_index_cache_stale_sweep_time_sec, num_shards,
element_count_capacity, true) {}
InvertedIndexSearcherCachePolicy(size_t capacity, uint32_t num_shards,
uint32_t element_count_capacity,
CacheValueTimeExtractor cache_value_time_extractor,
bool cache_value_check_timestamp)
: LRUCachePolicyTrackingManual(
CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
LRUCacheType::SIZE, config::inverted_index_cache_stale_sweep_time_sec,
num_shards, element_count_capacity, cache_value_time_extractor,
cache_value_check_timestamp, true) {}
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_SEARCHER_CACHE, capacity,
LRUCacheType::SIZE,
config::inverted_index_cache_stale_sweep_time_sec, num_shards,
element_count_capacity, cache_value_time_extractor,
cache_value_check_timestamp, true) {}
};
// Insert a cache entry by key.
// And the cache entry will be returned in handle.
Expand Down Expand Up @@ -179,9 +179,9 @@ class InvertedIndexCacheHandle {

class InvertedIndexQueryCacheHandle;

class InvertedIndexQueryCache : public LRUCachePolicyTrackingManual {
class InvertedIndexQueryCache : public LRUCachePolicy {
public:
using LRUCachePolicyTrackingManual::insert;
using LRUCachePolicy::insert;

// cache key
struct CacheKey {
Expand Down Expand Up @@ -227,10 +227,9 @@ class InvertedIndexQueryCache : public LRUCachePolicyTrackingManual {
InvertedIndexQueryCache() = delete;

InvertedIndexQueryCache(size_t capacity, uint32_t num_shards)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE,
capacity, LRUCacheType::SIZE,
config::inverted_index_cache_stale_sweep_time_sec,
num_shards) {}
: LRUCachePolicy(CachePolicy::CacheType::INVERTEDINDEX_QUERY_CACHE, capacity,
LRUCacheType::SIZE, config::inverted_index_cache_stale_sweep_time_sec,
num_shards) {}

bool lookup(const CacheKey& key, InvertedIndexQueryCacheHandle* handle);

Expand Down
7 changes: 3 additions & 4 deletions be/src/olap/schema_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ using SegmentIteratorUPtr = std::unique_ptr<SegmentIterator>;
// eliminating the need for frequent allocation and deallocation during usage.
// This caching mechanism proves immensely advantageous, particularly in scenarios
// with high concurrency, where queries are executed simultaneously.
class SchemaCache : public LRUCachePolicyTrackingManual {
class SchemaCache : public LRUCachePolicy {
public:
static SchemaCache* instance();

Expand Down Expand Up @@ -86,9 +86,8 @@ class SchemaCache : public LRUCachePolicyTrackingManual {
};

SchemaCache(size_t capacity)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::SCHEMA_CACHE, capacity,
LRUCacheType::NUMBER,
config::schema_cache_sweep_time_sec) {}
: LRUCachePolicy(CachePolicy::CacheType::SCHEMA_CACHE, capacity, LRUCacheType::NUMBER,
config::schema_cache_sweep_time_sec) {}

private:
static constexpr char SCHEMA_DELIMITER = '-';
Expand Down
6 changes: 3 additions & 3 deletions be/src/olap/segment_loader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,9 @@ bool SegmentCache::lookup(const SegmentCache::CacheKey& key, SegmentCacheHandle*

void SegmentCache::insert(const SegmentCache::CacheKey& key, SegmentCache::CacheValue& value,
SegmentCacheHandle* handle) {
auto* lru_handle = LRUCachePolicyTrackingManual::insert(
key.encode(), &value, value.segment->meta_mem_usage(), value.segment->meta_mem_usage(),
CachePriority::NORMAL);
auto* lru_handle =
LRUCachePolicy::insert(key.encode(), &value, value.segment->meta_mem_usage(),
value.segment->meta_mem_usage(), CachePriority::NORMAL);
handle->push_segment(this, lru_handle);
}

Expand Down
11 changes: 5 additions & 6 deletions be/src/olap/segment_loader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ class BetaRowset;
// Make sure that cache_handle is valid during the segment usage period.
using BetaRowsetSharedPtr = std::shared_ptr<BetaRowset>;

class SegmentCache : public LRUCachePolicyTrackingManual {
class SegmentCache : public LRUCachePolicy {
public:
using LRUCachePolicyTrackingManual::insert;
using LRUCachePolicy::insert;
// The cache key or segment lru cache
struct CacheKey {
CacheKey(RowsetId rowset_id_, int64_t segment_id_)
Expand All @@ -81,10 +81,9 @@ class SegmentCache : public LRUCachePolicyTrackingManual {
};

SegmentCache(size_t memory_bytes_limit, size_t segment_num_limit)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::SEGMENT_CACHE,
memory_bytes_limit, LRUCacheType::SIZE,
config::tablet_rowset_stale_sweep_time_sec,
DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit) {}
: LRUCachePolicy(CachePolicy::CacheType::SEGMENT_CACHE, memory_bytes_limit,
LRUCacheType::SIZE, config::tablet_rowset_stale_sweep_time_sec,
DEFAULT_LRU_CACHE_NUM_SHARDS * 2, segment_num_limit) {}

// Lookup the given segment in the cache.
// If the segment is found, the cache entry will be written into handle.
Expand Down
8 changes: 4 additions & 4 deletions be/src/olap/storage_engine.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ class StorageEngine final : public BaseStorageEngine {
// lru cache for create tabelt round robin in disks
// key: partitionId_medium
// value: index
class CreateTabletIdxCache : public LRUCachePolicyTrackingManual {
class CreateTabletIdxCache : public LRUCachePolicy {
public:
// get key, delimiter with DELIMITER '-'
static std::string get_key(int64_t partition_id, TStorageMedium::type medium) {
Expand All @@ -558,9 +558,9 @@ class CreateTabletIdxCache : public LRUCachePolicyTrackingManual {
};

CreateTabletIdxCache(size_t capacity)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::CREATE_TABLET_RR_IDX_CACHE,
capacity, LRUCacheType::NUMBER,
/*stale_sweep_time_s*/ 30 * 60) {}
: LRUCachePolicy(CachePolicy::CacheType::CREATE_TABLET_RR_IDX_CACHE, capacity,
LRUCacheType::NUMBER,
/*stale_sweep_time_s*/ 30 * 60) {}
};

struct DirInfo {
Expand Down
3 changes: 1 addition & 2 deletions be/src/olap/tablet_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,7 @@ bvar::Adder<int64_t> g_tablet_meta_schema_columns_count("tablet_meta_schema_colu

TabletManager::TabletManager(StorageEngine& engine, int32_t tablet_map_lock_shard_size)
: _engine(engine),
_tablet_meta_mem_tracker(std::make_shared<MemTracker>(
"TabletMeta(experimental)", ExecEnv::GetInstance()->details_mem_tracker_set())),
_tablet_meta_mem_tracker(std::make_shared<MemTracker>("TabletMeta(experimental)")),
_tablets_shards_size(tablet_map_lock_shard_size),
_tablets_shards_mask(tablet_map_lock_shard_size - 1) {
CHECK_GT(_tablets_shards_size, 0);
Expand Down
9 changes: 4 additions & 5 deletions be/src/olap/tablet_meta.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -520,13 +520,12 @@ class DeleteBitmap {

void remove_sentinel_marks();

class AggCachePolicy : public LRUCachePolicyTrackingManual {
class AggCachePolicy : public LRUCachePolicy {
public:
AggCachePolicy(size_t capacity)
: LRUCachePolicyTrackingManual(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE,
capacity, LRUCacheType::SIZE,
config::delete_bitmap_agg_cache_stale_sweep_time_sec,
256) {}
: LRUCachePolicy(CachePolicy::CacheType::DELETE_BITMAP_AGG_CACHE, capacity,
LRUCacheType::SIZE,
config::delete_bitmap_agg_cache_stale_sweep_time_sec, 256) {}
};

class AggCache {
Expand Down
Loading

0 comments on commit b58570a

Please sign in to comment.