From 2d24d3268735ddd09492d06c9c070de54fb77e6c Mon Sep 17 00:00:00 2001 From: Lin Yang Date: Mon, 11 Sep 2023 14:50:51 -0700 Subject: [PATCH] epoch: Fix false sharing in Local struct The perf c2c data shows lots of cacheline contention were caused by false sharing between epoch and other field in "Local" struct. So use CachePadded class to align it to another cacheline to resolve this issue. --- crossbeam-epoch/src/internal.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crossbeam-epoch/src/internal.rs b/crossbeam-epoch/src/internal.rs index 486993c63..79fbb9709 100644 --- a/crossbeam-epoch/src/internal.rs +++ b/crossbeam-epoch/src/internal.rs @@ -273,9 +273,6 @@ pub(crate) struct Local { /// A node in the intrusive linked list of `Local`s. entry: Entry, - /// The local epoch. - epoch: AtomicEpoch, - /// A reference to the global data. /// /// When all guards and handles get dropped, this reference is destroyed. @@ -294,6 +291,9 @@ pub(crate) struct Local { /// /// This is just an auxiliary counter that sometimes kicks off collection. pin_count: Cell>, + + /// The local epoch. + epoch: CachePadded, } // Make sure `Local` is less than or equal to 2048 bytes. @@ -320,12 +320,12 @@ impl Local { let local = Owned::new(Local { entry: Entry::default(), - epoch: AtomicEpoch::new(Epoch::starting()), collector: UnsafeCell::new(ManuallyDrop::new(collector.clone())), bag: UnsafeCell::new(Bag::new()), guard_count: Cell::new(0), handle_count: Cell::new(1), pin_count: Cell::new(Wrapping(0)), + epoch: CachePadded::new(AtomicEpoch::new(Epoch::starting())), }) .into_shared(unprotected()); collector.global.locals.insert(local, unprotected());