[vm, gc] Incremental compaction.

At the beginning of a major GC cycle, select some mostly-empty pages to be evacuated. Mark the pages and the objects on these pages. Apply a write barrier for stores creating old -> evacuation candidate pointers, and discover any such pointers that already exist during marking.

At the end of a major GC cycle, evacuate objects from these pages. Forward pointers of objects in the remembered set and new-space. Free the evacuated pages.

This compaction is incremental in the sense that creating the remembered set is interleaved with mutator execution. The evacuation step, however, is stop-the-world.

Write-barrier elimination for x.slot = x is removed. Write-barrier elimination for x.slot = constant is removed in the JIT, kept for AOT but snapshot pages are marked as never-evacuate.

TEST=ci
Bug: #52513
Change-Id: Icbc29ef7cb662ef8759b8c1d7a63b7af60766281
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/357760
Reviewed-by: Alexander Aprelev <[email protected]>
Commit-Queue: Ryan Macnak <[email protected]>

runtime/docs/gc.md

@@ -95,18 +95,18 @@ But we combine the generational and incremental checks with a shift-and-mask.
 ```c++
 enum HeaderBits {
   ...
-  kNotMarkedBit,            // Incremental barrier target.
-  kNewBit,                  // Generational barrier target.
-  kAlwaysSetBit,            // Incremental barrier source.
-  kOldAndNotRememberedBit,  // Generational barrier source.
+  kNotMarkedBit,                 // Incremental barrier target.
+  kNewOrEvacuationCandidateBit,  // Generational barrier target.
+  kAlwaysSetBit,                 // Incremental barrier source.
+  kOldAndNotRememberedBit,       // Generational barrier source.
   ...
 };
 
-static constexpr intptr_t kGenerationalBarrierMask = 1 << kNewBit;
+static constexpr intptr_t kGenerationalBarrierMask = 1 << kNewOrEvacuationCandidateBit;
 static constexpr intptr_t kIncrementalBarrierMask = 1 << kNotMarkedBit;
 static constexpr intptr_t kBarrierOverlapShift = 2;
 COMPILE_ASSERT(kNotMarkedBit + kBarrierOverlapShift == kAlwaysSetBit);
-COMPILE_ASSERT(kNewBit + kBarrierOverlapShift == kOldAndNotRememberedBit);
+COMPILE_ASSERT(kNewOrEvacuationCandidateBit + kBarrierOverlapShift == kOldAndNotRememberedBit);
 
 StorePointer(ObjectPtr source, ObjectPtr* slot, ObjectPtr target) {
   *slot = target;
@@ -178,7 +178,6 @@ We can eliminate these checks when the compiler can prove these cases cannot hap
 * `value` is a constant. Constants are always old, and they will be marked via the constant pools even if we fail to mark them via `container`.
 * `value` has the static type bool. All possible values of the bool type (null, false, true) are constants.
 * `value` is known to be a Smi. Smis are not heap objects.
-* `container` is the same object as `value`. The GC never needs to retain an additional object if it sees a self-reference, so ignoring a self-reference cannot cause us to free a reachable object.
 * `container` is known to be a new object or known to be an old object that is in the remembered set and is marked if marking is in progress.
 
 We can know that `container` meets the last property if `container` is the result of an allocation (instead of a heap load), and there is no instruction that can trigger a GC between the allocation and the store. This is because the allocation stubs ensure the result of AllocateObject is either a new-space object (common case, bump pointer allocation succeeds), or has been preemptively added to the remembered set and marking worklist (uncommon case, entered runtime to allocate object, possibly triggering GC).

runtime/platform/atomic.h

@@ -80,8 +80,8 @@ class RelaxedAtomic {
   }
   T operator+=(T arg) { return fetch_add(arg) + arg; }
   T operator-=(T arg) { return fetch_sub(arg) - arg; }
-  T& operator++() { return fetch_add(1) + 1; }
-  T& operator--() { return fetch_sub(1) - 1; }
+  T operator++() { return fetch_add(1) + 1; }
+  T operator--() { return fetch_sub(1) - 1; }
   T operator++(int) { return fetch_add(1); }
   T operator--(int) { return fetch_sub(1); }
 

runtime/vm/app_snapshot.cc

@@ -893,7 +893,7 @@ void Deserializer::InitializeHeader(ObjectPtr raw,
   tags = UntaggedObject::AlwaysSetBit::update(true, tags);
   tags = UntaggedObject::NotMarkedBit::update(true, tags);
   tags = UntaggedObject::OldAndNotRememberedBit::update(true, tags);
-  tags = UntaggedObject::NewBit::update(false, tags);
+  tags = UntaggedObject::NewOrEvacuationCandidateBit::update(false, tags);
   tags = UntaggedObject::ImmutableBit::update(is_immutable, tags);
   raw->untag()->tags_ = tags;
 }

runtime/vm/bitfield.h

@@ -36,6 +36,9 @@ class AtomicBitFieldContainer : AtomicBitFieldContainerBase {
   }
 
   T load(std::memory_order order) const { return field_.load(order); }
+  NO_SANITIZE_THREAD T load_ignore_race() const {
+    return *reinterpret_cast<const T*>(&field_);
+  }
   void store(T value, std::memory_order order) { field_.store(value, order); }
 
   bool compare_exchange_weak(T old_tags, T new_tags, std::memory_order order) {
@@ -48,11 +51,6 @@ class AtomicBitFieldContainer : AtomicBitFieldContainerBase {
     return TargetBitField::decode(field_.load(order));
   }
 
-  template <class TargetBitField>
-  NO_SANITIZE_THREAD typename TargetBitField::Type ReadIgnoreRace() const {
-    return TargetBitField::decode(*reinterpret_cast<const T*>(&field_));
-  }
-
   template <class TargetBitField,
             std::memory_order order = std::memory_order_relaxed>
   void UpdateBool(bool value) {

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -1933,7 +1933,7 @@ void Assembler::VerifyStoreNeedsNoWriteBarrier(Register object,
   Label done;
   BranchIfSmi(value, &done, kNearJump);
   ldrb(TMP, FieldAddress(value, target::Object::tags_offset()));
-  tst(TMP, Operand(1 << target::UntaggedObject::kNewBit));
+  tst(TMP, Operand(1 << target::UntaggedObject::kNewOrEvacuationCandidateBit));
   b(&done, ZERO);
   ldrb(TMP, FieldAddress(object, target::Object::tags_offset()));
   tst(TMP, Operand(1 << target::UntaggedObject::kOldAndNotRememberedBit));

runtime/vm/compiler/assembler/assembler_arm64.cc

@@ -1231,7 +1231,7 @@ void Assembler::VerifyStoreNeedsNoWriteBarrier(Register object,
   Label done;
   BranchIfSmi(value, &done, kNearJump);
   ldr(TMP, FieldAddress(value, target::Object::tags_offset()), kUnsignedByte);
-  tbz(&done, TMP, target::UntaggedObject::kNewBit);
+  tbz(&done, TMP, target::UntaggedObject::kNewOrEvacuationCandidateBit);
   ldr(TMP, FieldAddress(object, target::Object::tags_offset()), kUnsignedByte);
   tbz(&done, TMP, target::UntaggedObject::kOldAndNotRememberedBit);
   Stop("Write barrier is required");

runtime/vm/compiler/assembler/assembler_ia32.cc

@@ -2207,7 +2207,7 @@ void Assembler::VerifyStoreNeedsNoWriteBarrier(Register object,
   Label done;
   BranchIfSmi(value, &done, kNearJump);
   testb(FieldAddress(value, target::Object::tags_offset()),
-        Immediate(1 << target::UntaggedObject::kNewBit));
+        Immediate(1 << target::UntaggedObject::kNewOrEvacuationCandidateBit));
   j(ZERO, &done, Assembler::kNearJump);
   testb(FieldAddress(object, target::Object::tags_offset()),
         Immediate(1 << target::UntaggedObject::kOldAndNotRememberedBit));

runtime/vm/compiler/assembler/assembler_riscv.cc

@@ -3518,7 +3518,7 @@ void Assembler::VerifyStoreNeedsNoWriteBarrier(Register object,
   Label done;
   BranchIfSmi(value, &done, kNearJump);
   lbu(TMP2, FieldAddress(value, target::Object::tags_offset()));
-  andi(TMP2, TMP2, 1 << target::UntaggedObject::kNewBit);
+  andi(TMP2, TMP2, 1 << target::UntaggedObject::kNewOrEvacuationCandidateBit);
   beqz(TMP2, &done, kNearJump);
   lbu(TMP2, FieldAddress(object, target::Object::tags_offset()));
   andi(TMP2, TMP2, 1 << target::UntaggedObject::kOldAndNotRememberedBit);

runtime/vm/compiler/assembler/assembler_x64.cc

@@ -1684,7 +1684,7 @@ void Assembler::VerifyStoreNeedsNoWriteBarrier(Register object,
   Label done;
   BranchIfSmi(value, &done, kNearJump);
   testb(FieldAddress(value, target::Object::tags_offset()),
-        Immediate(1 << target::UntaggedObject::kNewBit));
+        Immediate(1 << target::UntaggedObject::kNewOrEvacuationCandidateBit));
   j(ZERO, &done, Assembler::kNearJump);
   testb(FieldAddress(object, target::Object::tags_offset()),
         Immediate(1 << target::UntaggedObject::kOldAndNotRememberedBit));

runtime/vm/compiler/backend/il.cc

@@ -1392,10 +1392,17 @@ bool Value::NeedsWriteBarrier() {
 
     // Strictly speaking, the incremental barrier can only be skipped for
     // immediate objects (Smis) or permanent objects (vm-isolate heap or
-    // image pages). Here we choose to skip the barrier for any constant on
-    // the assumption it will remain reachable through the object pool.
+    // image pages). For AOT, we choose to skip the barrier for any constant on
+    // the assumptions it will remain reachable through the object pool and it
+    // is on a page created by snapshot loading that is marked so as to never be
+    // evacuated.
     if (value->BindsToConstant()) {
-      return false;
+      if (FLAG_precompiled_mode) {
+        return false;
+      } else {
+        const Object& constant = value->BoundConstant();
+        return constant.ptr()->IsHeapObject() && !constant.InVMIsolateHeap();
+      }
     }
 
     // Follow the chain of redefinitions as redefined value could have a more

runtime/vm/compiler/backend/il.h

@@ -6417,11 +6417,6 @@ class StoreFieldInstr : public TemplateInstruction<2, NoThrow> {
       // The target field is native and unboxed, so not traversed by the GC.
       return false;
     }
-    if (instance()->definition() == value()->definition()) {
-      // `x.slot = x` cannot create an old->new or old&marked->old&unmarked
-      // reference.
-      return false;
-    }
 
     if (value()->definition()->Type()->IsBool()) {
       return false;
@@ -7074,12 +7069,6 @@ class StoreIndexedInstr : public TemplateInstruction<3, NoThrow> {
   bool aligned() const { return alignment_ == kAlignedAccess; }
 
   bool ShouldEmitStoreBarrier() const {
-    if (array()->definition() == value()->definition()) {
-      // `x[slot] = x` cannot create an old->new or old&marked->old&unmarked
-      // reference.
-      return false;
-    }
-
     if (value()->definition()->Type()->IsBool()) {
       return false;
     }

runtime/vm/compiler/frontend/base_flow_graph_builder.cc

@@ -522,11 +522,9 @@ Fragment BaseFlowGraphBuilder::StoreNativeField(
     StoreBarrierType emit_store_barrier /* = kEmitStoreBarrier */,
     compiler::Assembler::MemoryOrder memory_order /* = kRelaxed */) {
   Value* value = Pop();
-  if (value->BindsToConstant()) {
-    emit_store_barrier = kNoStoreBarrier;
-  }
+  Value* instance = Pop();
   StoreFieldInstr* store = new (Z)
-      StoreFieldInstr(slot, Pop(), value, emit_store_barrier,
+      StoreFieldInstr(slot, instance, value, emit_store_barrier,
                       stores_inner_pointer, InstructionSource(position), kind);
   return Fragment(store);
 }

runtime/vm/compiler/runtime_api.cc

@@ -361,7 +361,7 @@ uword MakeTagWordForNewSpaceObject(classid_t cid, uword instance_size) {
   return dart::UntaggedObject::SizeTag::encode(
              TranslateOffsetInWordsToHost(instance_size)) |
          dart::UntaggedObject::ClassIdTag::encode(cid) |
-         dart::UntaggedObject::NewBit::encode(true) |
+         dart::UntaggedObject::NewOrEvacuationCandidateBit::encode(true) |
          dart::UntaggedObject::AlwaysSetBit::encode(true) |
          dart::UntaggedObject::NotMarkedBit::encode(true) |
          dart::UntaggedObject::ImmutableBit::encode(
@@ -377,7 +377,8 @@ const word UntaggedObject::kCardRememberedBit =
 
 const word UntaggedObject::kCanonicalBit = dart::UntaggedObject::kCanonicalBit;
 
-const word UntaggedObject::kNewBit = dart::UntaggedObject::kNewBit;
+const word UntaggedObject::kNewOrEvacuationCandidateBit =
+    dart::UntaggedObject::kNewOrEvacuationCandidateBit;
 
 const word UntaggedObject::kOldAndNotRememberedBit =
     dart::UntaggedObject::kOldAndNotRememberedBit;

runtime/vm/compiler/runtime_api.h

@@ -418,7 +418,7 @@ class UntaggedObject : public AllStatic {
  public:
   static const word kCardRememberedBit;
   static const word kCanonicalBit;
-  static const word kNewBit;
+  static const word kNewOrEvacuationCandidateBit;
   static const word kOldAndNotRememberedBit;
   static const word kNotMarkedBit;
   static const word kImmutableBit;

runtime/vm/flag_list.h

@@ -127,8 +127,8 @@ constexpr bool FLAG_support_il_printer = false;
   R(log_marker_tasks, false, bool, false,                                      \
     "Log debugging information for old gen GC marking tasks.")                 \
   P(scavenger_tasks, int, 2,                                                   \
-    "The number of tasks to spawn during scavenging (0 means "                 \
-    "perform all marking on main thread).")                                    \
+    "The number of tasks to spawn during scavenging and incremental "          \
+    "compaction (0 means perform all work on the main thread).")               \
   P(mark_when_idle, bool, false,                                               \
     "The Dart thread will assist in concurrent marking during idle time and "  \
     "is counted as one marker task")                                           \
@@ -216,6 +216,8 @@ constexpr bool FLAG_support_il_printer = false;
   P(truncating_left_shift, bool, true,                                         \
     "Optimize left shift to truncate if possible")                             \
   P(use_compactor, bool, false, "Compact the heap during old-space GC.")       \
+  P(use_incremental_compactor, bool, true,                                     \
+    "Compact the heap during old-space GC.")                                   \
   P(use_cha_deopt, bool, true,                                                 \
     "Use class hierarchy analysis even if it can cause deoptimization.")       \
   P(use_field_guards, bool, true, "Use field guards and track field types")    \

runtime/vm/heap/become.cc

@@ -29,7 +29,7 @@ ForwardingCorpse* ForwardingCorpse::AsForwarder(uword addr, intptr_t size) {
   bool is_old = (addr & kNewObjectAlignmentOffset) == kOldObjectAlignmentOffset;
   tags = UntaggedObject::NotMarkedBit::update(true, tags);
   tags = UntaggedObject::OldAndNotRememberedBit::update(is_old, tags);
-  tags = UntaggedObject::NewBit::update(!is_old, tags);
+  tags = UntaggedObject::NewOrEvacuationCandidateBit::update(!is_old, tags);
 
   result->tags_ = tags;
   if (size > UntaggedObject::SizeTag::kMaxSizeTag) {

runtime/vm/heap/compactor.cc

@@ -6,9 +6,9 @@
 
 #include "platform/atomic.h"
 #include "vm/globals.h"
-#include "vm/heap/become.h"
 #include "vm/heap/heap.h"
 #include "vm/heap/pages.h"
+#include "vm/heap/sweeper.h"
 #include "vm/thread_barrier.h"
 #include "vm/timeline.h"
 
@@ -184,18 +184,52 @@ class CompactorTask : public ThreadPool::Task {
 void GCCompactor::Compact(Page* pages, FreeList* freelist, Mutex* pages_lock) {
   SetupImagePageBoundaries();
 
-  // Divide the heap.
+  Page* fixed_head = nullptr;
+  Page* fixed_tail = nullptr;
+
+  // Divide the heap, and set aside never-evacuate pages.
   // TODO(30978): Try to divide based on live bytes or with work stealing.
   intptr_t num_pages = 0;
-  for (Page* page = pages; page != nullptr; page = page->next()) {
-    num_pages++;
+  Page* page = pages;
+  Page* prev = nullptr;
+  while (page != nullptr) {
+    Page* next = page->next();
+    if (page->is_never_evacuate()) {
+      if (prev != nullptr) {
+        prev->set_next(next);
+      } else {
+        pages = next;
+      }
+      if (fixed_tail == nullptr) {
+        fixed_tail = page;
+      }
+      page->set_next(fixed_head);
+      fixed_head = page;
+    } else {
+      prev = page;
+      num_pages++;
+    }
+    page = next;
   }
+  fixed_pages_ = fixed_head;
 
   intptr_t num_tasks = FLAG_compactor_tasks;
   RELEASE_ASSERT(num_tasks >= 1);
   if (num_pages < num_tasks) {
     num_tasks = num_pages;
   }
+  if (num_tasks == 0) {
+    ASSERT(pages == nullptr);
+
+    // Move pages to sweeper work lists.
+    heap_->old_space()->pages_ = nullptr;
+    heap_->old_space()->pages_tail_ = nullptr;
+    heap_->old_space()->sweep_regular_ = fixed_head;
+
+    heap_->old_space()->Sweep(/*exclusive*/ true);
+    heap_->old_space()->SweepLarge();
+    return;
+  }
 
   Partition* partitions = new Partition[num_tasks];
 
@@ -206,6 +240,7 @@ void GCCompactor::Compact(Page* pages, FreeList* freelist, Mutex* pages_lock) {
     Page* page = pages;
     Page* prev = nullptr;
     while (task_index < num_tasks) {
+      ASSERT(!page->is_never_evacuate());
       if (page_index % pages_per_task == 0) {
         partitions[task_index].head = page;
         partitions[task_index].tail = nullptr;
@@ -352,6 +387,12 @@ void GCCompactor::Compact(Page* pages, FreeList* freelist, Mutex* pages_lock) {
     partitions[num_tasks - 1].tail->set_next(nullptr);
     heap_->old_space()->pages_ = pages = partitions[0].head;
     heap_->old_space()->pages_tail_ = partitions[num_tasks - 1].tail;
+    if (fixed_head != nullptr) {
+      fixed_tail->set_next(heap_->old_space()->pages_);
+      heap_->old_space()->pages_ = fixed_head;
+
+      ASSERT(heap_->old_space()->pages_tail_ != nullptr);
+    }
 
     delete[] partitions;
   }
@@ -486,6 +527,7 @@ void CompactorTask::RunEnteredIsolateGroup() {
 }
 
 void CompactorTask::PlanPage(Page* page) {
+  ASSERT(!page->is_never_evacuate());
   uword current = page->object_start();
   uword end = page->object_end();
 
@@ -498,6 +540,7 @@ void CompactorTask::PlanPage(Page* page) {
 }
 
 void CompactorTask::SlidePage(Page* page) {
+  ASSERT(!page->is_never_evacuate());
   uword current = page->object_start();
   uword end = page->object_end();
 
@@ -667,6 +710,11 @@ void GCCompactor::ForwardPointer(ObjectPtr* ptr) {
   if (forwarding_page == nullptr) {
     return;  // Not moved (VM isolate, large page, code page).
   }
+  if (page->is_never_evacuate()) {
+    // Forwarding page is non-NULL since one is still reserved for use as a
+    // counting page, but it doesn't have forwarding information.
+    return;
+  }
 
   ObjectPtr new_target =
       UntaggedObject::FromAddr(forwarding_page->Lookup(old_addr));
@@ -703,6 +751,11 @@ void GCCompactor::ForwardCompressedPointer(uword heap_base,
   if (forwarding_page == nullptr) {
     return;  // Not moved (VM isolate, large page, code page).
   }
+  if (page->is_never_evacuate()) {
+    // Forwarding page is non-NULL since one is still reserved for use as a
+    // counting page, but it doesn't have forwarding information.
+    return;
+  }
 
   ObjectPtr new_target =
       UntaggedObject::FromAddr(forwarding_page->Lookup(old_addr));
@@ -796,6 +849,24 @@ void GCCompactor::ForwardLargePages() {
     page->VisitObjectPointers(this);
     ml.Lock();
   }
+  while (fixed_pages_ != nullptr) {
+    Page* page = fixed_pages_;
+    fixed_pages_ = page->next();
+    ml.Unlock();
+
+    GCSweeper sweeper;
+    FreeList* freelist = heap_->old_space()->DataFreeList(0);
+    bool page_in_use;
+    {
+      MutexLocker ml(freelist->mutex());
+      page_in_use = sweeper.SweepPage(page, freelist);
+    }
+    ASSERT(page_in_use);
+
+    page->VisitObjectPointers(this);
+
+    ml.Lock();
+  }
 }
 
 void GCCompactor::ForwardStackPointers() {

runtime/vm/heap/compactor.h

@@ -74,6 +74,7 @@ class GCCompactor : public ValueObject,
 
   Mutex large_pages_mutex_;
   Page* large_pages_ = nullptr;
+  Page* fixed_pages_ = nullptr;
 
   // The typed data views whose inner pointer must be updated after sliding is
   // complete.