diff --git a/src/tbb/arena.h b/src/tbb/arena.h index 5e74aca50f..8df7cb84e0 100644 --- a/src/tbb/arena.h +++ b/src/tbb/arena.h @@ -633,7 +633,7 @@ void arena::advertise_new_work() { my_permit_manager->adjust_demand(*this->my_client, my_max_num_workers, /* mandatory = */ false); // Notify all sleeping threads that work has appeared in the arena. - my_permit_manager->get_wait_list().notify(is_related_arena); + governor::get_wait_list().notify(is_related_arena); } } } diff --git a/src/tbb/governor.cpp b/src/tbb/governor.cpp index 3111ab3e7b..23407825f5 100644 --- a/src/tbb/governor.cpp +++ b/src/tbb/governor.cpp @@ -21,6 +21,7 @@ #include "arena.h" #include "dynamic_link.h" #include "concurrent_monitor.h" +#include "market_concurrent_monitor.h" #include "oneapi/tbb/task_group.h" #include "oneapi/tbb/global_control.h" @@ -68,6 +69,7 @@ void governor::acquire_resources () { detect_cpu_features(cpu_features); is_rethrow_broken = gcc_rethrow_exception_broken(); + sleep_monitor = new (cache_aligned_allocate(sizeof(market_concurrent_monitor))) market_concurrent_monitor; } void governor::release_resources () { @@ -79,6 +81,8 @@ void governor::release_resources () { int status = theTLS.destroy(); if( status ) runtime_warning("failed to destroy task scheduler TLS: %s", std::strerror(status)); + sleep_monitor->~market_concurrent_monitor(); + cache_aligned_deallocate(sleep_monitor); clear_address_waiter_table(); system_topology::destroy(); diff --git a/src/tbb/governor.h b/src/tbb/governor.h index 3d861e5323..8ded4d72d1 100644 --- a/src/tbb/governor.h +++ b/src/tbb/governor.h @@ -29,6 +29,7 @@ namespace r1 { class market; class thread_data; class __TBB_InitOnce; +class market_concurrent_monitor; #if __TBB_USE_ITT_NOTIFY //! Defined in profiling.cpp @@ -62,6 +63,8 @@ class governor { static cpu_features_type cpu_features; static bool is_rethrow_broken; + static market_concurrent_monitor* sleep_monitor; + //! Create key for thread-local storage and initialize RML. static void acquire_resources (); @@ -146,6 +149,11 @@ class governor { return false; #endif } + + //! Return wait list + static market_concurrent_monitor& get_wait_list() { + return *sleep_monitor; + } }; // class governor } // namespace r1 diff --git a/src/tbb/market.h b/src/tbb/market.h index c722703df3..3e8243f5cd 100644 --- a/src/tbb/market.h +++ b/src/tbb/market.h @@ -84,9 +84,6 @@ class market : public permit_manager, rml::tbb_client { //! Pointer to the RML server object that services this TBB instance. rml::tbb_server* my_server; - //! Waiting object for external and coroutine waiters. - market_concurrent_monitor my_sleep_monitor; - //! Maximal number of workers allowed for use by the underlying resource manager /** It can't be changed after market creation. **/ unsigned my_num_workers_hard_limit; @@ -239,9 +236,6 @@ class market : public permit_manager, rml::tbb_client { //! Decrements market's refcount and destroys it in the end bool release ( bool is_public, bool blocking_terminate ); - //! Return wait list - market_concurrent_monitor& get_wait_list() { return my_sleep_monitor; } - #if __TBB_ENQUEUE_ENFORCED_CONCURRENCY //! Imlpementation of mandatory concurrency enabling void enable_mandatory_concurrency_impl (tbb_permit_manager_client*a ); diff --git a/src/tbb/task.cpp b/src/tbb/task.cpp index cfa728728c..143d2a137e 100644 --- a/src/tbb/task.cpp +++ b/src/tbb/task.cpp @@ -186,7 +186,7 @@ void task_dispatcher::do_post_resume_action() { auto is_our_suspend_point = [sp] (market_context ctx) { return std::uintptr_t(sp) == ctx.my_uniq_addr; }; - td->my_arena->my_market->get_wait_list().notify(is_our_suspend_point); + governor::get_wait_list().notify(is_our_suspend_point); break; } default: @@ -218,7 +218,7 @@ void notify_waiters(std::uintptr_t wait_ctx_addr) { return wait_ctx_addr == context.my_uniq_addr; }; - r1::governor::get_thread_data()->my_arena->my_market->get_wait_list().notify(is_related_wait_ctx); + governor::get_wait_list().notify(is_related_wait_ctx); } } // namespace r1 diff --git a/src/tbb/task_dispatcher.h b/src/tbb/task_dispatcher.h index f6ff3f173c..597242db8d 100644 --- a/src/tbb/task_dispatcher.h +++ b/src/tbb/task_dispatcher.h @@ -71,7 +71,7 @@ inline d1::task* suspend_point_type::resume_task::execute(d1::execution_data& ed thread_data* td = ed_ext.task_disp->m_thread_data; td->set_post_resume_action(task_dispatcher::post_resume_action::register_waiter, &monitor_node); - market_concurrent_monitor& wait_list = td->my_arena->my_market->get_wait_list(); + market_concurrent_monitor& wait_list = governor::get_wait_list(); if (wait_list.wait([&] { return !ed_ext.wait_ctx->continue_execution(); }, monitor_node)) { return nullptr; diff --git a/src/tbb/thread_data.h b/src/tbb/thread_data.h index 5bcceb76af..57611fcbb2 100644 --- a/src/tbb/thread_data.h +++ b/src/tbb/thread_data.h @@ -39,6 +39,7 @@ class task; class arena_slot; class task_group_context; class task_dispatcher; +class tbb_permit_manager_client; class context_list : public intrusive_list { public: