8376839: GenShen: Improve performance of evacuations into the old generation

Reviewed-by: kdnilsen

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp

@@ -112,6 +112,24 @@ void ShenandoahConcurrentGC::entry_concurrent_update_refs_prepare(ShenandoahHeap
   heap->concurrent_prepare_for_update_refs();
 }
 
+void ShenandoahConcurrentGC::entry_update_card_table() {
+  ShenandoahHeap* const heap = ShenandoahHeap::heap();
+  TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters());
+
+  static const char* msg = "Concurrent update cards";
+  ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_update_card_table);
+  EventMark em("%s", msg);
+
+  ShenandoahWorkerScope scope(heap->workers(),
+                              ShenandoahWorkerPolicy::calc_workers_for_conc_evac(),
+                              "concurrent update cards");
+
+  // Heap needs to be parsable here.
+  // Also, parallel heap region iterate must have a phase set.
+  assert(ShenandoahTimingsTracker::is_current_phase_valid(), "Current phase must be set");
+  ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
+}
+
 bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
   ShenandoahHeap* const heap = ShenandoahHeap::heap();
   _generation->ref_processor()->set_soft_reference_policy(
@@ -206,6 +224,11 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) {
 
     // Perform update-refs phase.
     entry_concurrent_update_refs_prepare(heap);
+
+    if (ShenandoahHeap::heap()->mode()->is_generational()) {
+      entry_update_card_table();
+    }
+
     if (ShenandoahVerify) {
       vmop_entry_init_update_refs();
     }

src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp

@@ -59,8 +59,6 @@ public:
   bool collect(GCCause::Cause cause) override;
   ShenandoahDegenPoint degen_point() const;
 
-  void entry_concurrent_update_refs_prepare(ShenandoahHeap* heap);
-
   // Return true if this cycle found enough immediate garbage to skip evacuation
   bool abbreviated() const { return _abbreviated; }
 
@@ -95,6 +93,8 @@ protected:
   void entry_cleanup_early();
   void entry_evacuate();
   void entry_update_thread_roots();
+  void entry_update_card_table();
+  void entry_concurrent_update_refs_prepare(ShenandoahHeap* heap);
   void entry_update_refs();
   void entry_cleanup_complete();
 

src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp

@@ -277,6 +277,11 @@ void ShenandoahDegenGC::op_degenerated() {
         _abbreviated = true;
       }
 
+      // labs are retired, walk the old regions and update remembered set
+      if (ShenandoahHeap::heap()->mode()->is_generational()) {
+        ShenandoahGenerationalHeap::heap()->old_generation()->update_card_table();
+      }
+
     case _degenerated_update_refs:
       if (heap->has_forwarded_objects()) {
         op_update_refs();

src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp

@@ -1569,7 +1569,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
   // We must call try_recycle_under_lock() even if !r->is_trash().  The reason is that if r is being recycled at this
   // moment by a GC worker thread, it may appear to be not trash even though it has not yet been fully recycled.  If
   // we proceed without waiting for the worker to finish recycling the region, the worker thread may overwrite the
-  // region's affiliation with FREE after we set the region's affiliation to req.afiliation() below
+  // region's affiliation with FREE after we set the region's affiliation to req.affiliation() below
   r->try_recycle_under_lock();
   in_new_region = r->is_empty();
   if (in_new_region) {
@@ -1585,7 +1585,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah
       // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any
       // coalesce-and-fill processing.
       r->end_preemptible_coalesce_and_fill();
-      _heap->old_generation()->clear_cards_for(r);
     }
 #ifdef ASSERT
     ShenandoahMarkingContext* const ctx = _heap->marking_context();

src/hotspot/share/gc/shenandoah/shenandoahGenerationalEvacuationTask.cpp

@@ -133,3 +133,4 @@ void ShenandoahGenerationalEvacuationTask::evacuate_and_promote_regions() {
     }
   }
 }
+

src/hotspot/share/gc/shenandoah/shenandoahGenerationalHeap.cpp

@@ -363,10 +363,6 @@ oop ShenandoahGenerationalHeap::try_evacuate_object(oop p, Thread* thread, uint
       // Record that the evacuation succeeded
       evac_tracker()->end_evacuation(thread, size * HeapWordSize, FROM_GENERATION, TO_GENERATION);
     }
-
-    if (TO_GENERATION == OLD_GENERATION) {
-      old_generation()->handle_evacuation(copy, size);
-    }
   }  else {
     // Failed to evacuate. We need to deal with the object that is left behind. Since this
     // new allocation is certainly after TAMS, it will be considered live in the next cycle.

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -1955,6 +1955,26 @@ void ShenandoahHeap::heap_region_iterate(ShenandoahHeapRegionClosure* blk) const
   }
 }
 
+class ShenandoahHeapRegionIteratorTask : public WorkerTask {
+private:
+  ShenandoahRegionIterator _regions;
+  ShenandoahHeapRegionClosure* _closure;
+
+public:
+  ShenandoahHeapRegionIteratorTask(ShenandoahHeapRegionClosure* closure)
+    : WorkerTask("Shenandoah Heap Region Iterator")
+    , _closure(closure) {}
+
+  void work(uint worker_id) override {
+    ShenandoahParallelWorkerSession worker_session(worker_id);
+    ShenandoahHeapRegion* region = _regions.next();
+    while (region != nullptr) {
+      _closure->heap_region_do(region);
+      region = _regions.next();
+    }
+  }
+};
+
 class ShenandoahParallelHeapRegionTask : public WorkerTask {
 private:
   ShenandoahHeap* const _heap;
@@ -2011,6 +2031,11 @@ void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* b
   }
 }
 
+void ShenandoahHeap::heap_region_iterator(ShenandoahHeapRegionClosure* closure) const {
+  ShenandoahHeapRegionIteratorTask task(closure);
+  workers()->run_task(&task);
+}
+
 class ShenandoahRendezvousHandshakeClosure : public HandshakeClosure {
 public:
   inline ShenandoahRendezvousHandshakeClosure(const char* name) : HandshakeClosure(name) {}

src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp

@@ -298,6 +298,7 @@ public:
 
   void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const;
+  void heap_region_iterator(ShenandoahHeapRegionClosure* blk) const;
 
   inline ShenandoahMmuTracker* mmu_tracker() { return &_mmu_tracker; };
 

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp

@@ -67,6 +67,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c
   _new_top(nullptr),
   _empty_time(os::elapsedTime()),
   _top_before_promoted(nullptr),
+  _top_at_evac_start(start),
   _state(committed ? _empty_committed : _empty_uncommitted),
   _top(start),
   _tlab_allocs(0),
@@ -565,12 +566,17 @@ void ShenandoahHeapRegion::recycle_internal() {
   assert(_recycling.is_set() && is_trash(), "Wrong state");
   ShenandoahHeap* heap = ShenandoahHeap::heap();
 
+  _top_at_evac_start = _bottom;
   _mixed_candidate_garbage_words = 0;
   set_top(bottom());
   clear_live_data();
   reset_alloc_metadata();
   heap->marking_context()->reset_top_at_mark_start(this);
   set_update_watermark(bottom());
+  if (is_old()) {
+    heap->old_generation()->clear_cards_for(this);
+  }
+
   if (ZapUnusedHeapArea) {
     SpaceMangler::mangle_region(MemRegion(bottom(), end()));
   }

src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp

@@ -246,6 +246,7 @@ private:
   double _empty_time;
 
   HeapWord* _top_before_promoted;
+  HeapWord* _top_at_evac_start;
 
   // Seldom updated fields
   Atomic<RegionState> _state;
@@ -365,12 +366,15 @@ public:
   }
 
   // Returns true iff this region was promoted in place subsequent to the most recent start of concurrent old marking.
-  inline bool was_promoted_in_place() {
+  bool was_promoted_in_place() const {
     return _promoted_in_place;
   }
   inline void restore_top_before_promote();
   inline size_t garbage_before_padded_for_promote() const;
 
+  HeapWord* get_top_at_evac_start() const { return _top_at_evac_start; }
+  void record_top_at_evac_start()         { _top_at_evac_start = _top; }
+
   // If next available memory is not aligned on address that is multiple of alignment, fill the empty space
   // so that returned object is aligned on an address that is a multiple of alignment_in_bytes.  Requested
   // size is in words.  It is assumed that this->is_old().  A pad object is allocated, filled, and registered

src/hotspot/share/gc/shenandoah/shenandoahHeapRegionClosures.cpp

@@ -80,6 +80,11 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR
     // Remember limit for updating refs. It's guaranteed that we get no
     // from-space-refs written from here on.
     r->set_update_watermark_at_safepoint(r->top());
+
+    if (r->is_old()) {
+      // Record where we need to start updating the remembered set
+      r->record_top_at_evac_start();
+    }
   } else {
     assert(!r->has_live(), "Region %zu should have no live data", r->index());
     assert(_ctx == nullptr || _ctx->top_at_mark_start(r) == r->top(),

src/hotspot/share/gc/shenandoah/shenandoahInPlacePromoter.cpp

@@ -238,6 +238,9 @@ void ShenandoahInPlacePromoter::promote(ShenandoahHeapRegion* region) const {
     // is_collector_free range.  We'll add it to that range below.
     region->restore_top_before_promote();
 
+    // We also need to record where those allocations begin so that we can later update the remembered set.
+    region->record_top_at_evac_start();
+
     assert(region->used() + pip_pad_bytes + pip_unpadded == region_size_bytes, "invariant");
 
     // The update_watermark was likely established while we had the artificially high value of top.  Make it sane now.

src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp

@@ -318,6 +318,11 @@ void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* c
   ShenandoahHeap::heap()->heap_region_iterate(&old_regions_cl);
 }
 
+void ShenandoahOldGeneration::heap_region_iterator(ShenandoahHeapRegionClosure* cl) {
+  ShenandoahIncludeRegionClosure<OLD_GENERATION> old_regions_cl(cl);
+  ShenandoahHeap::heap()->heap_region_iterator(&old_regions_cl);
+}
+
 void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) {
   ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress);
 }
@@ -326,6 +331,12 @@ bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() {
   return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress();
 }
 
+void ShenandoahOldGeneration::record_tops_at_evac_start() {
+  for_each_region([](ShenandoahHeapRegion* region) {
+    region->record_top_at_evac_start();
+  });
+}
+
 void ShenandoahOldGeneration::cancel_marking() {
   if (is_concurrent_mark_in_progress()) {
     log_debug(gc)("Abandon SATB buffers");
@@ -662,15 +673,19 @@ void ShenandoahOldGeneration::log_failed_promotion(LogStream& ls, Thread* thread
   }
 }
 
-void ShenandoahOldGeneration::handle_evacuation(HeapWord* obj, size_t words) const {
-  // Only register the copy of the object that won the evacuation race.
-  _card_scan->register_object_without_lock(obj);
-
-  // Mark the entire range of the evacuated object as dirty.  At next remembered set scan,
-  // we will clear dirty bits that do not hold interesting pointers.  It's more efficient to
-  // do this in batch, in a background GC thread than to try to carefully dirty only cards
-  // that hold interesting pointers right now.
-  _card_scan->mark_range_as_dirty(obj, words);
+void ShenandoahOldGeneration::update_card_table() {
+  for_each_region([this](ShenandoahHeapRegion* region) {
+    if (region->is_regular()) {
+      // Humongous regions are promoted in place, remembered set maintenance is handled there
+      // Regular regions that are promoted in place have their rset maintenance handled for
+      // the objects in the region when it was promoted. We record TEAS for such a region
+      // when the in-place-promotion is completed. Such a region may be used for additional
+      // promotions in the same cycle it was itself promoted.
+      if (region->top() > region->get_top_at_evac_start()) {
+        _card_scan->update_card_table(region->get_top_at_evac_start(), region->top());
+      }
+    }
+  });
 }
 
 bool ShenandoahOldGeneration::has_unprocessed_collection_candidates() {

src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp

@@ -172,8 +172,8 @@ public:
   void handle_failed_promotion(Thread* thread, size_t size) const;
   void log_failed_promotion(LogStream& ls, Thread* thread, size_t size) const;
 
-  // A successful evacuation re-dirties the cards and registers the object with the remembered set
-  void handle_evacuation(HeapWord* obj, size_t words) const;
+  // Iterate over recently promoted objects to update card table and object registrations
+  void update_card_table();
 
   // Clear the flag after it is consumed by the control thread
   bool clear_failed_evacuation() {
@@ -199,11 +199,36 @@ public:
   // Mark card for this location as dirty
   void mark_card_as_dirty(void* location);
 
+  template<typename T>
+  class ShenandoahHeapRegionLambda : public ShenandoahHeapRegionClosure {
+    T _region_lambda;
+  public:
+    explicit ShenandoahHeapRegionLambda(T region_lambda) : _region_lambda(region_lambda) {}
+
+    void heap_region_do(ShenandoahHeapRegion* r) override {
+      _region_lambda(r);
+    }
+
+    bool is_thread_safe() override {
+      return true;
+    }
+
+    size_t parallel_region_stride() override {
+      // Temporarily override to force parallelism when updating card table
+      return 8;
+    }
+  };
+
+  template<typename LambdaT>
+  void for_each_region(LambdaT lambda) {
+    ShenandoahHeapRegionLambda l(lambda);
+    heap_region_iterator(&l);
+  }
+
   void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override;
-
   void parallel_heap_region_iterate_free(ShenandoahHeapRegionClosure* cl) override;
-
   void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override;
+  void heap_region_iterator(ShenandoahHeapRegionClosure* cl);
 
   bool contains(ShenandoahAffiliation affiliation) const override;
   bool contains(ShenandoahHeapRegion* region) const override;
@@ -212,6 +237,10 @@ public:
   void set_concurrent_mark_in_progress(bool in_progress) override;
   bool is_concurrent_mark_in_progress() override;
 
+  // For old regions, objects between top at evac start and top represent promoted objects.
+  // These objects will need to have their cards dirtied and their offsets within the cards registered.
+  void record_tops_at_evac_start();
+
   bool entry_coalesce_and_fill();
 
   // Global collections touch old regions, so the old generation needs to be informed of this.

src/hotspot/share/gc/shenandoah/shenandoahPLAB.cpp

@@ -210,12 +210,4 @@ void ShenandoahPLAB::retire() {
   // plab->retire() overwrites unused memory between plab->top() and plab->hard_end() with a dummy object to make memory parsable.
   // It adds the size of this unused memory, in words, to plab->waste().
   _plab->retire();
-  if (top != nullptr && _plab->waste() > original_waste && _heap->is_in_old(top)) {
-    // If retiring the plab created a filler object, then we need to register it with our card scanner so it can
-    // safely walk the region backing the plab.
-    log_debug(gc, plab)("retire_plab() is registering remnant of size %zu at " PTR_FORMAT,
-                        (_plab->waste() - original_waste) * HeapWordSize, p2i(top));
-    // No lock is necessary because the PLAB memory is aligned on card boundaries.
-    _heap->old_generation()->card_scan()->register_object_without_lock(top);
-  }
 }

src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp

@@ -109,6 +109,7 @@ class outputStream;
   f(conc_strong_roots,                              "Concurrent Strong Roots")         \
   SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_,       "  CSR: ", f)                      \
   f(conc_evac,                                      "Concurrent Evacuation")           \
+  f(conc_update_card_table,                         "Concurrent Update Cards")         \
   f(conc_final_roots,                               "Concurrent Final Roots")          \
   f(promote_in_place,                               "  Promote Regions")               \
   f(final_roots_gross,                              "Pause Verify Final Roots (G)")    \
@@ -254,7 +255,7 @@ public:
   void flush_cycle_to_global();
 
   static const char* phase_name(Phase phase) {
-    assert(phase >= 0 && phase < _num_phases, "Out of bound");
+    assert(phase >= 0 && phase < _num_phases, "Out of bounds: %d", phase);
     return _phase_names[phase];
   }
 

src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp

@@ -31,6 +31,33 @@
 #include "logging/log.hpp"
 #include "runtime/threads.hpp"
 
+// A closure that takes an oop in the old generation and, if it's pointing
+// into the young generation, dirties the corresponding remembered set entry.
+class ShenandoahDirtyRememberedSetClosure : public BasicOopIterateClosure {
+protected:
+  ShenandoahGenerationalHeap* const _heap;
+  ShenandoahScanRemembered*   const _scanner;
+
+public:
+  ShenandoahDirtyRememberedSetClosure() :
+          _heap(ShenandoahGenerationalHeap::heap()),
+          _scanner(_heap->old_generation()->card_scan()) {}
+
+  template<class T>
+  void work(T* p) {
+    assert(_heap->is_in_old(p), "Expecting to get an old gen address");
+    if (T o = RawAccess<>::oop_load(p); !CompressedOops::is_null(o)) {
+      if (const oop obj = CompressedOops::decode_not_null(o); _heap->is_in_young(obj)) {
+        // Dirty the card containing the cross-generational pointer.
+        _scanner->mark_card_as_dirty((HeapWord*) p);
+      }
+    }
+  }
+
+  void do_oop(narrowOop* p) override { work(p); }
+  void do_oop(oop* p) override       { work(p); }
+};
+
 size_t ShenandoahDirectCardMarkRememberedSet::last_valid_index() const {
   return _card_table->last_valid_index();
 }
@@ -161,7 +188,6 @@ void ShenandoahCardCluster::register_object_without_lock(HeapWord* address) {
   uint8_t offset_in_card = checked_cast<uint8_t>(pointer_delta(address, card_start_address));
 
   if (!starts_object(card_at_start)) {
-    set_starts_object_bit(card_at_start);
     set_first_start(card_at_start, offset_in_card);
     set_last_start(card_at_start, offset_in_card);
   } else {
@@ -172,6 +198,49 @@ void ShenandoahCardCluster::register_object_without_lock(HeapWord* address) {
   }
 }
 
+void ShenandoahCardCluster::update_card_table(HeapWord* start, HeapWord* end) {
+  HeapWord* address = start;
+  HeapWord* previous_address = nullptr;
+  uint8_t previous_offset = 0;
+  size_t previous_card_index = -1;
+  ShenandoahDirtyRememberedSetClosure make_cards_dirty;
+
+  log_debug(gc, remset)("Update remembered set from " PTR_FORMAT ", to " PTR_FORMAT, p2i(start), p2i(end));
+  _rs->mark_range_as_dirty(start, pointer_delta(end, start));
+
+  while (address < end) {
+
+    // Compute card and offset in card for this object
+    const size_t object_card_index = _rs->card_index_for_addr(address);
+    const HeapWord* card_start_address = _rs->addr_for_card_index(object_card_index);
+    const uint8_t offset_in_card = checked_cast<uint8_t>(pointer_delta(address, card_start_address));
+
+    if (object_card_index != previous_card_index) {
+      if (previous_address != nullptr) {
+        // Register the previous object on the previous card, we are starting a new card here
+        set_last_start(previous_card_index, previous_offset);
+      }
+
+      previous_card_index = object_card_index;
+      if (!starts_object(object_card_index)) {
+        // The previous cycle may have recorded an earlier start in this card. Do not overwrite it.
+        set_first_start(object_card_index, offset_in_card);
+      }
+    }
+
+    previous_offset = offset_in_card;
+    previous_address = address;
+
+    const oop obj = cast_to_oop(address);
+    address += obj->size();
+  }
+
+  // Register the last object seen in this range.
+  if (previous_address != nullptr) {
+    set_last_start(previous_card_index, previous_offset);
+  }
+}
+
 void ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t length_in_words) {
 
   size_t card_at_start = _rs->card_index_for_addr(address);
@@ -641,36 +710,6 @@ void ShenandoahScanRemembered::merge_worker_card_stats_cumulative(
 }
 #endif
 
-// A closure that takes an oop in the old generation and, if it's pointing
-// into the young generation, dirties the corresponding remembered set entry.
-// This is only used to rebuild the remembered set after a full GC.
-class ShenandoahDirtyRememberedSetClosure : public BasicOopIterateClosure {
-protected:
-  ShenandoahGenerationalHeap* const _heap;
-  ShenandoahScanRemembered*   const _scanner;
-
-public:
-  ShenandoahDirtyRememberedSetClosure() :
-          _heap(ShenandoahGenerationalHeap::heap()),
-          _scanner(_heap->old_generation()->card_scan()) {}
-
-  template<class T>
-  inline void work(T* p) {
-    assert(_heap->is_in_old(p), "Expecting to get an old gen address");
-    T o = RawAccess<>::oop_load(p);
-    if (!CompressedOops::is_null(o)) {
-      oop obj = CompressedOops::decode_not_null(o);
-      if (_heap->is_in_young(obj)) {
-        // Dirty the card containing the cross-generational pointer.
-        _scanner->mark_card_as_dirty((HeapWord*) p);
-      }
-    }
-  }
-
-  virtual void do_oop(narrowOop* p) { work(p); }
-  virtual void do_oop(oop* p)       { work(p); }
-};
-
 ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(ShenandoahCardTable* card_table, size_t total_card_count) :
   LogCardValsPerIntPtr(log2i_exact(sizeof(intptr_t)) - log2i_exact(sizeof(CardValue))),
   LogCardSizeInWords(log2i_exact(CardTable::card_size_in_words())) {
@@ -1039,38 +1078,44 @@ void ShenandoahReconstructRememberedSetTask::work(uint worker_id) {
   ShenandoahDirtyRememberedSetClosure dirty_cards_for_cross_generational_pointers;
 
   while (r != nullptr) {
-    if (r->is_old() && r->is_active()) {
-      HeapWord* obj_addr = r->bottom();
-      if (r->is_humongous_start()) {
-        // First, clear the remembered set
-        oop obj = cast_to_oop(obj_addr);
-        size_t size = obj->size();
-
-        size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
-        size_t region_index = r->index();
-        ShenandoahHeapRegion* humongous_region = heap->get_region(region_index);
-        while (num_regions-- != 0) {
-          scanner->reset_object_range(humongous_region->bottom(), humongous_region->end());
-          region_index++;
-          humongous_region = heap->get_region(region_index);
-        }
-
-        // Then register the humongous object and DIRTY relevant remembered set cards
-        scanner->register_object_without_lock(obj_addr);
-        obj->oop_iterate(&dirty_cards_for_cross_generational_pointers);
-      } else if (!r->is_humongous()) {
-        scanner->reset_object_range(r->bottom(), r->end());
-
-        // Then iterate over all objects, registering object and DIRTYing relevant remembered set cards
-        HeapWord* t = r->top();
-        while (obj_addr < t) {
+    if (r->is_active()) {
+      if (r->is_old()) {
+        HeapWord* obj_addr = r->bottom();
+        if (r->is_humongous_start()) {
+          // First, clear the remembered set
           oop obj = cast_to_oop(obj_addr);
+          size_t size = obj->size();
+
+          size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize);
+          size_t region_index = r->index();
+          ShenandoahHeapRegion* humongous_region = heap->get_region(region_index);
+          while (num_regions-- != 0) {
+            scanner->reset_object_range(humongous_region->bottom(), humongous_region->end());
+            region_index++;
+            humongous_region = heap->get_region(region_index);
+          }
+
+          // Then register the humongous object and DIRTY relevant remembered set cards
           scanner->register_object_without_lock(obj_addr);
-          obj_addr += obj->oop_iterate_size(&dirty_cards_for_cross_generational_pointers);
-        }
-      } // else, ignore humongous continuation region
+          obj->oop_iterate(&dirty_cards_for_cross_generational_pointers);
+        } else if (!r->is_humongous()) {
+          scanner->reset_object_range(r->bottom(), r->end());
+
+          // Then iterate over all objects, registering object and DIRTYing relevant remembered set cards
+          HeapWord* t = r->top();
+          while (obj_addr < t) {
+            oop obj = cast_to_oop(obj_addr);
+            scanner->register_object_without_lock(obj_addr);
+            obj_addr += obj->oop_iterate_size(&dirty_cards_for_cross_generational_pointers);
+          }
+        } // else, ignore humongous continuation region
+      } else {
+        // The region is young, but it may become old again and we don't want stale remembered set data.
+        assert(r->is_young(), "Region: %zu, is active but free", r->index());
+        heap->old_generation()->clear_cards_for(r);
+      }
     }
-    // else, this region is FREE or YOUNG or inactive and we can ignore it.
+    // else, this region is FREE or inactive and we can ignore it.
     r = _regions->next();
   }
 }

src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp

@@ -603,6 +603,9 @@ public:
   // as address.
   void register_object_without_lock(HeapWord* address);
 
+  // Dirty cards and register objects for the given range in memory.
+  void update_card_table(HeapWord* start, HeapWord* end);
+
   // During the reference updates phase of GC, we walk through each old-gen memory region that was
   // not part of the collection set and we invalidate all unmarked objects.  As part of this effort,
   // we coalesce neighboring dead objects in order to make future remembered set scanning more
@@ -814,6 +817,10 @@ public:
     }
   }
 
+  void update_card_table(HeapWord* start, HeapWord* end) const {
+    _scc->update_card_table(start, end);
+  }
+
   // Return true iff this object is "properly" registered.
   bool verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx);