diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.cpp b/src/hotspot/share/gc/parallel/psParallelCompact.cpp index f4383e573af..484533619a4 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.cpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.cpp @@ -1660,21 +1660,19 @@ static void compaction_with_stealing_work(TaskTerminator* terminator, uint worke } class FillDensePrefixAndCompactionTask: public WorkerTask { - uint _num_workers; TaskTerminator _terminator; public: FillDensePrefixAndCompactionTask(uint active_workers) : WorkerTask("FillDensePrefixAndCompactionTask"), - _num_workers(active_workers), _terminator(active_workers, ParCompactionManager::region_task_queues()) { } virtual void work(uint worker_id) { - { + if (worker_id == 0) { auto start = Ticks::now(); - PSParallelCompact::fill_dead_objs_in_dense_prefix(worker_id, _num_workers); - log_trace(gc, phases)("Fill dense prefix by worker %u: %.3f ms", worker_id, (Ticks::now() - start).seconds() * 1000); + PSParallelCompact::fill_dead_objs_in_dense_prefix(); + log_trace(gc, phases)("Fill dense prefix by worker 0: %.3f ms", (Ticks::now() - start).seconds() * 1000); } compaction_with_stealing_work(&_terminator, worker_id); } @@ -1687,9 +1685,10 @@ void PSParallelCompact::fill_range_in_dense_prefix(HeapWord* start, HeapWord* en assert(mark_bitmap()->find_obj_beg(start, end) == end, "precondition"); HeapWord* bottom = _space_info[old_space_id].space()->bottom(); if (start != bottom) { + // The preceding live obj. HeapWord* obj_start = mark_bitmap()->find_obj_beg_reverse(bottom, start); - HeapWord* after_obj = obj_start + cast_to_oop(obj_start)->size(); - assert(after_obj == start, "precondition"); + HeapWord* obj_end = obj_start + cast_to_oop(obj_start)->size(); + assert(obj_end == start, "precondition"); } } #endif @@ -1703,58 +1702,51 @@ void PSParallelCompact::fill_range_in_dense_prefix(HeapWord* start, HeapWord* en } while (addr < end); } -void PSParallelCompact::fill_dead_objs_in_dense_prefix(uint worker_id, uint num_workers) { +void PSParallelCompact::fill_dead_objs_in_dense_prefix() { ParMarkBitMap* bitmap = mark_bitmap(); HeapWord* const bottom = _space_info[old_space_id].space()->bottom(); HeapWord* const prefix_end = dense_prefix(old_space_id); - if (bottom == prefix_end) { - return; - } + const size_t region_size = ParallelCompactData::RegionSize; - size_t bottom_region = _summary_data.addr_to_region_idx(bottom); - size_t prefix_end_region = _summary_data.addr_to_region_idx(prefix_end); + // Fill dead space in [start_addr, end_addr) + HeapWord* const start_addr = bottom; + HeapWord* const end_addr = prefix_end; - size_t start_region; - size_t end_region; - split_regions_for_worker(bottom_region, prefix_end_region, - worker_id, num_workers, - &start_region, &end_region); + for (HeapWord* cur_addr = start_addr; cur_addr < end_addr; /* empty */) { + RegionData* cur_region_ptr = _summary_data.addr_to_region_ptr(cur_addr); + if (cur_region_ptr->data_size() == region_size) { + // Full; no dead space. Next region. + if (_summary_data.is_region_aligned(cur_addr)) { + cur_addr += region_size; + } else { + cur_addr = _summary_data.region_align_up(cur_addr); + } + continue; + } - if (start_region == end_region) { - return; - } + // Fill dead space inside cur_region. + if (_summary_data.is_region_aligned(cur_addr)) { + cur_addr += cur_region_ptr->partial_obj_size(); + } - HeapWord* const start_addr = _summary_data.region_to_addr(start_region); - HeapWord* const end_addr = _summary_data.region_to_addr(end_region); - - // Skip live partial obj (if any) from previous region. - HeapWord* cur_addr; - RegionData* start_region_ptr = _summary_data.region(start_region); - if (start_region_ptr->partial_obj_size() != 0) { - HeapWord* partial_obj_start = start_region_ptr->partial_obj_addr(); - assert(bitmap->is_marked(partial_obj_start), "inv"); - cur_addr = partial_obj_start + cast_to_oop(partial_obj_start)->size(); - } else { - cur_addr = start_addr; - } - - // end_addr is inclusive to handle regions starting with dead space. - while (cur_addr <= end_addr) { - // Use prefix_end to handle trailing obj in each worker region-chunk. - HeapWord* live_start = bitmap->find_obj_beg(cur_addr, prefix_end); - if (cur_addr != live_start) { - // Only worker 0 handles proceeding dead space. - if (cur_addr != start_addr || worker_id == 0) { + HeapWord* region_end_addr = _summary_data.region_align_up(cur_addr + 1); + assert(region_end_addr <= end_addr, "inv"); + while (cur_addr < region_end_addr) { + // Use end_addr to allow filler-obj to cross region boundary. + HeapWord* live_start = bitmap->find_obj_beg(cur_addr, end_addr); + if (cur_addr != live_start) { + // Found dead space [cur_addr, live_start). fill_range_in_dense_prefix(cur_addr, live_start); } + if (live_start >= region_end_addr) { + cur_addr = live_start; + break; + } + assert(bitmap->is_marked(live_start), "inv"); + cur_addr = live_start + cast_to_oop(live_start)->size(); } - if (live_start >= end_addr) { - break; - } - assert(bitmap->is_marked(live_start), "inv"); - cur_addr = live_start + cast_to_oop(live_start)->size(); } } @@ -1787,15 +1779,38 @@ void PSParallelCompact::compact() { void PSParallelCompact::verify_filler_in_dense_prefix() { HeapWord* bottom = _space_info[old_space_id].space()->bottom(); HeapWord* dense_prefix_end = dense_prefix(old_space_id); - HeapWord* cur_addr = bottom; - while (cur_addr < dense_prefix_end) { - oop obj = cast_to_oop(cur_addr); - oopDesc::verify(obj); - if (!mark_bitmap()->is_marked(cur_addr)) { - Klass* k = cast_to_oop(cur_addr)->klass(); - assert(k == Universe::fillerArrayKlass() || k == vmClasses::FillerObject_klass(), "inv"); + + const size_t region_size = ParallelCompactData::RegionSize; + + for (HeapWord* cur_addr = bottom; cur_addr < dense_prefix_end; /* empty */) { + RegionData* cur_region_ptr = _summary_data.addr_to_region_ptr(cur_addr); + if (cur_region_ptr->data_size() == region_size) { + // Full; no dead space. Next region. + if (_summary_data.is_region_aligned(cur_addr)) { + cur_addr += region_size; + } else { + cur_addr = _summary_data.region_align_up(cur_addr); + } + continue; + } + + // This region contains filler objs. + if (_summary_data.is_region_aligned(cur_addr)) { + cur_addr += cur_region_ptr->partial_obj_size(); + } + + HeapWord* region_end_addr = _summary_data.region_align_up(cur_addr + 1); + assert(region_end_addr <= dense_prefix_end, "inv"); + + while (cur_addr < region_end_addr) { + oop obj = cast_to_oop(cur_addr); + oopDesc::verify(obj); + if (!mark_bitmap()->is_marked(cur_addr)) { + Klass* k = cast_to_oop(cur_addr)->klass(); + assert(k == Universe::fillerArrayKlass() || k == vmClasses::FillerObject_klass(), "inv"); + } + cur_addr += obj->size(); } - cur_addr += obj->size(); } } diff --git a/src/hotspot/share/gc/parallel/psParallelCompact.hpp b/src/hotspot/share/gc/parallel/psParallelCompact.hpp index 4d212499b4c..2297d720b35 100644 --- a/src/hotspot/share/gc/parallel/psParallelCompact.hpp +++ b/src/hotspot/share/gc/parallel/psParallelCompact.hpp @@ -755,7 +755,7 @@ private: static void fill_range_in_dense_prefix(HeapWord* start, HeapWord* end); public: - static void fill_dead_objs_in_dense_prefix(uint worker_id, uint num_workers); + static void fill_dead_objs_in_dense_prefix(); // This method invokes a full collection. // clear_all_soft_refs controls whether soft-refs should be cleared or not.