From 1408468b9fec7c0c59f70e186e6bd14b16134ba5 Mon Sep 17 00:00:00 2001 From: Ryan Daum Date: Wed, 3 May 2023 12:29:04 -0400 Subject: [PATCH] Hooks & modifications to GC to allow registration of externally allocated memory. Goal: allow a mechanism to make the GC aware of non-Julia process memory usage. Background in RAI-7988 * Adds calls to register allocations and frees of process memory unaccounted for by live Julia objects, but relevant for GC collection heuristics. * Adds a separate call to allow the Julia program to adjust the `default_collect_interval`. * Increases the default collection interval (to avoid excessive full sweeps once external memory is reported) --- src/gc.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++-- src/julia.h | 9 +++++++++ 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/gc.c b/src/gc.c index e1ac14d4772f2..29a90d70a1c63 100644 --- a/src/gc.c +++ b/src/gc.c @@ -604,10 +604,10 @@ static void gc_sweep_foreign_objs(void) static int64_t last_gc_total_bytes = 0; #ifdef _P64 -#define default_collect_interval (5600*1024*sizeof(void*)) +static size_t default_collect_interval = (5600*1024*sizeof(void*)); static size_t max_collect_interval = 1250000000UL; #else -#define default_collect_interval (3200*1024*sizeof(void*)) +static size_t default_collect_interval = (3200*1024*sizeof(void*)); static size_t max_collect_interval = 500000000UL; #endif @@ -1096,6 +1096,54 @@ void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); } +JL_DLLEXPORT void jl_gc_set_default_collect_interval(size_t collect_interval) JL_NOTSAFEPOINT { + default_collect_interval = collect_interval; +} + +JL_DLLEXPORT size_t jl_gc_default_collect_interval() JL_NOTSAFEPOINT { + return default_collect_interval; +} + +JL_DLLEXPORT void jl_gc_allocd_external(size_t sz_delta) JL_NOTSAFEPOINT +{ + // External memory is divided up for accounting among all threads; this way the + // per-thread quantity is kept balanced to help avoid full GC thrashing if only one + // thread goes over max_collect_interval. + int64_t per_tls_delta = sz_delta / jl_n_threads; + int64_t remainder = sz_delta % jl_n_threads; + + for (int i = 0; i < jl_n_threads; i++) { + jl_ptls_t ptls = jl_all_tls_states[i]; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + + per_tls_delta); + } + // Stash the remainder in the 1st thread. + jl_ptls_t ptls = jl_all_tls_states[0]; + jl_atomic_store_relaxed(&ptls->gc_num.allocd, + jl_atomic_load_relaxed(&ptls->gc_num.allocd) + remainder); +} + +JL_DLLEXPORT void jl_gc_freed_external(size_t sz_delta) JL_NOTSAFEPOINT +{ + // External memory is divided up for accounting among all threads; this way the + // per-thread quantity is kept balanced to help avoid full GC thrashing if only one + // thread goes over max_collect_interval. + int64_t per_tls_delta = sz_delta / jl_n_threads; + int64_t remainder = sz_delta % jl_n_threads; + + for (int i = 0; i < jl_n_threads; i++) { + jl_ptls_t ptls = jl_all_tls_states[i]; + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + + per_tls_delta); + } + // Subtract the remainder from the 1st thread. + jl_ptls_t ptls = jl_all_tls_states[0]; + jl_atomic_store_relaxed(&ptls->gc_num.freed, + jl_atomic_load_relaxed(&ptls->gc_num.freed) + remainder); +} + static void combine_thread_gc_counts(jl_gc_num_t *dest) JL_NOTSAFEPOINT { for (int i = 0; i < jl_n_threads; i++) { diff --git a/src/julia.h b/src/julia.h index 83b3f5f925fb6..cd4d82827ccfd 100644 --- a/src/julia.h +++ b/src/julia.h @@ -937,6 +937,15 @@ JL_DLLEXPORT void *jl_gc_managed_realloc(void *d, size_t sz, size_t oldsz, int isaligned, jl_value_t *owner); JL_DLLEXPORT void jl_gc_safepoint(void); +JL_DLLEXPORT void jl_gc_set_default_collect_interval(size_t collect_interval) JL_NOTSAFEPOINT; +JL_DLLEXPORT size_t jl_gc_default_collect_interval(void) JL_NOTSAFEPOINT; + +// Record accounting for memory that exists outside-of the Julia runtime, but which is +// nevertheless part of the same process RSS and which should be accounted for when +// calculating process memory pressure when triggering garbage collection. +JL_DLLEXPORT void jl_gc_allocd_external(size_t sz_delta) JL_NOTSAFEPOINT; +JL_DLLEXPORT void jl_gc_freed_external(size_t sz_delta) JL_NOTSAFEPOINT; + // object accessors ----------------------------------------------------------- #define jl_svec_len(t) (((jl_svec_t*)(t))->length)