diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp index d8f6c68a51e..48617450c8b 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.cpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.cpp @@ -193,80 +193,56 @@ HeapRegion* G1CollectedHeap::new_region(size_t word_size, return res; } -HeapWord* -G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr, - uint num_regions, - size_t word_size) { - assert(first_hr != NULL, "pre-condition"); - assert(is_humongous(word_size), "word_size should be humongous"); - assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); - - // Index of last region in the series. - uint first = first_hr->hrm_index(); - uint last = first + num_regions - 1; - - // We need to initialize the region(s) we just discovered. This is - // a bit tricky given that it can happen concurrently with - // refinement threads refining cards on these regions and - // potentially wanting to refine the BOT as they are scanning - // those cards (this can happen shortly after a cleanup; see CR - // 6991377). So we have to set up the region(s) carefully and in - // a specific order. - - // The word size sum of all the regions we will allocate. - size_t word_size_sum = (size_t) num_regions * HeapRegion::GrainWords; +void G1CollectedHeap::set_humongous_metadata(HeapRegion* first_hr, + uint num_regions, + size_t word_size, + bool update_remsets) { + // Calculate the new top of the humongous object. + HeapWord* obj_top = first_hr->bottom() + word_size; + // The word size sum of all the regions used + size_t word_size_sum = num_regions * HeapRegion::GrainWords; assert(word_size <= word_size_sum, "sanity"); - // The passed in hr will be the "starts humongous" region. The header - // of the new object will be placed at the bottom of this region. - HeapWord* new_obj = first_hr->bottom(); - // This will be the new top of the new object. - HeapWord* obj_top = new_obj + word_size; - - // First, we need to zero the header of the space that we will be - // allocating. When we update top further down, some refinement - // threads might try to scan the region. By zeroing the header we - // ensure that any thread that will try to scan the region will - // come across the zero klass word and bail out. - // - // NOTE: It would not have been correct to have used - // CollectedHeap::fill_with_object() and make the space look like - // an int array. The thread that is doing the allocation will - // later update the object header to a potentially different array - // type and, for a very short period of time, the klass and length - // fields will be inconsistent. This could cause a refinement - // thread to calculate the object size incorrectly. - Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); - - // Next, pad out the unused tail of the last region with filler - // objects, for improved usage accounting. - // How many words we use for filler objects. - size_t word_fill_size = word_size_sum - word_size; - // How many words memory we "waste" which cannot hold a filler object. size_t words_not_fillable = 0; - if (word_fill_size >= min_fill_size()) { - fill_with_objects(obj_top, word_fill_size); - } else if (word_fill_size > 0) { + // Pad out the unused tail of the last region with filler + // objects, for improved usage accounting. + + // How many words can we use for filler objects. + size_t words_fillable = word_size_sum - word_size; + + if (words_fillable >= G1CollectedHeap::min_fill_size()) { + G1CollectedHeap::fill_with_objects(obj_top, words_fillable); + } else { // We have space to fill, but we cannot fit an object there. - words_not_fillable = word_fill_size; - word_fill_size = 0; + words_not_fillable = words_fillable; + words_fillable = 0; } // We will set up the first region as "starts humongous". This // will also update the BOT covering all the regions to reflect // that there is a single object that starts at the bottom of the // first region. - first_hr->set_starts_humongous(obj_top, word_fill_size); - _policy->remset_tracker()->update_at_allocate(first_hr); - // Then, if there are any, we will set up the "continues - // humongous" regions. - HeapRegion* hr = NULL; + first_hr->hr_clear(false /* clear_space */); + first_hr->set_starts_humongous(obj_top, words_fillable); + + if (update_remsets) { + _policy->remset_tracker()->update_at_allocate(first_hr); + } + + // Indices of first and last regions in the series. + uint first = first_hr->hrm_index(); + uint last = first + num_regions - 1; + + HeapRegion* hr = nullptr; for (uint i = first + 1; i <= last; ++i) { hr = region_at(i); + hr->hr_clear(false /* clear_space */); hr->set_continues_humongous(first_hr); - _policy->remset_tracker()->update_at_allocate(hr); + if (update_remsets) { + _policy->remset_tracker()->update_at_allocate(hr); + } } // Up to this point no concurrent thread would have been able to @@ -297,11 +273,57 @@ G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr, assert(words_not_fillable == 0 || first_hr->bottom() + word_size_sum - words_not_fillable == hr->top(), "Miscalculation in humongous allocation"); +} - increase_used((word_size_sum - words_not_fillable) * HeapWordSize); +HeapWord* +G1CollectedHeap::humongous_obj_allocate_initialize_regions(HeapRegion* first_hr, + uint num_regions, + size_t word_size) { + assert(first_hr != NULL, "pre-condition"); + assert(is_humongous(word_size), "word_size should be humongous"); + assert(num_regions * HeapRegion::GrainWords >= word_size, "pre-condition"); + + // Index of last region in the series. + uint first = first_hr->hrm_index(); + uint last = first + num_regions - 1; + + // We need to initialize the region(s) we just discovered. This is + // a bit tricky given that it can happen concurrently with + // refinement threads refining cards on these regions and + // potentially wanting to refine the BOT as they are scanning + // those cards (this can happen shortly after a cleanup; see CR + // 6991377). So we have to set up the region(s) carefully and in + // a specific order. + + // The passed in hr will be the "starts humongous" region. The header + // of the new object will be placed at the bottom of this region. + HeapWord* new_obj = first_hr->bottom(); + + // First, we need to zero the header of the space that we will be + // allocating. When we update top further down, some refinement + // threads might try to scan the region. By zeroing the header we + // ensure that any thread that will try to scan the region will + // come across the zero klass word and bail out. + // + // NOTE: It would not have been correct to have used + // CollectedHeap::fill_with_object() and make the space look like + // an int array. The thread that is doing the allocation will + // later update the object header to a potentially different array + // type and, for a very short period of time, the klass and length + // fields will be inconsistent. This could cause a refinement + // thread to calculate the object size incorrectly. + Copy::fill_to_words(new_obj, oopDesc::header_size(), 0); + + // Next, update the metadata for the regions. + set_humongous_metadata(first_hr, num_regions, word_size, true); + + HeapRegion* last_hr = region_at(last); + size_t used = byte_size(first_hr->bottom(), last_hr->top()); + + increase_used(used); for (uint i = first; i <= last; ++i) { - hr = region_at(i); + HeapRegion *hr = region_at(i); _humongous_set.add(hr); _hr_printer.alloc(hr); } diff --git a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp index 644098d88a7..60c08cc3cec 100644 --- a/src/hotspot/share/gc/g1/g1CollectedHeap.hpp +++ b/src/hotspot/share/gc/g1/g1CollectedHeap.hpp @@ -606,6 +606,11 @@ public: // Register the given region to be part of the collection set. inline void register_humongous_candidate_region_with_region_attr(uint index); + void set_humongous_metadata(HeapRegion* first_hr, + uint num_regions, + size_t word_size, + bool update_remsets); + // We register a region with the fast "in collection set" test. We // simply set to true the array slot corresponding to this region. void register_young_region_with_region_attr(HeapRegion* r) { diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp index 1f3a9461441..1fd30111c96 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.cpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp @@ -40,7 +40,7 @@ #include "gc/g1/g1Policy.hpp" #include "gc/g1/g1RegionMarkStatsCache.inline.hpp" #include "gc/shared/gcTraceTime.inline.hpp" -#include "gc/shared/preservedMarks.hpp" +#include "gc/shared/preservedMarks.inline.hpp" #include "gc/shared/referenceProcessor.hpp" #include "gc/shared/verifyOption.hpp" #include "gc/shared/weakProcessor.inline.hpp" @@ -119,12 +119,15 @@ G1FullCollector::G1FullCollector(G1CollectedHeap* heap, _scope(heap->monitoring_support(), explicit_gc, clear_soft_refs, do_maximal_compaction, tracer), _num_workers(calc_active_workers()), _has_compaction_targets(false), + _has_humongous(false), _oop_queue_set(_num_workers), _array_queue_set(_num_workers), _preserved_marks_set(true), _serial_compaction_point(this), + _humongous_compaction_point(this), _is_alive(this, heap->concurrent_mark()->mark_bitmap()), _is_alive_mutator(heap->ref_processor_stw(), &_is_alive), + _humongous_compaction_regions(8), _always_subject_to_discovery(), _is_subject_mutator(heap->ref_processor_stw(), &_always_subject_to_discovery), _region_attr_table() { @@ -155,6 +158,7 @@ G1FullCollector::~G1FullCollector() { delete _markers[i]; delete _compaction_points[i]; } + FREE_C_HEAP_ARRAY(G1FullGCMarker*, _markers); FREE_C_HEAP_ARRAY(G1FullGCCompactionPoint*, _compaction_points); FREE_C_HEAP_ARRAY(HeapWord*, _compaction_tops); @@ -246,6 +250,8 @@ void G1FullCollector::complete_collection() { _heap->gc_epilogue(true); _heap->verify_after_full_collection(); + + _heap->print_heap_after_full_collection(); } void G1FullCollector::before_marking_update_attribute_table(HeapRegion* hr) { @@ -343,6 +349,12 @@ void G1FullCollector::phase2_prepare_compaction() { // maximally compact the tail regions of the compaction queues serially. if (scope()->do_maximal_compaction() || !has_free_compaction_targets) { phase2c_prepare_serial_compaction(); + + if (scope()->do_maximal_compaction() && + has_humongous() && + serial_compaction_point()->has_regions()) { + phase2d_prepare_humongous_compaction(); + } } } @@ -418,6 +430,35 @@ void G1FullCollector::phase2c_prepare_serial_compaction() { serial_cp->update(); } +void G1FullCollector::phase2d_prepare_humongous_compaction() { + GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare humongous compaction", scope()->timer()); + G1FullGCCompactionPoint* serial_cp = serial_compaction_point(); + assert(serial_cp->has_regions(), "Sanity!" ); + + uint last_serial_target = serial_cp->current_region()->hrm_index(); + uint region_index = last_serial_target + 1; + uint max_reserved_regions = _heap->max_reserved_regions(); + + G1FullGCCompactionPoint* humongous_cp = humongous_compaction_point(); + + while (region_index < max_reserved_regions) { + HeapRegion* hr = _heap->region_at_or_null(region_index); + + if (hr == nullptr) { + region_index++; + continue; + } else if (hr->is_starts_humongous()) { + uint num_regions = humongous_cp->forward_humongous(hr); + region_index += num_regions; // Skip over the continues humongous regions. + continue; + } else if (is_compaction_target(region_index)) { + // Add the region to the humongous compaction point. + humongous_cp->add(hr); + } + region_index++; + } +} + void G1FullCollector::phase3_adjust_pointers() { // Adjust the pointers to reflect the new locations GCTraceTime(Info, gc, phases) info("Phase 3: Adjust pointers", scope()->timer()); @@ -436,6 +477,11 @@ void G1FullCollector::phase4_do_compaction() { if (serial_compaction_point()->has_regions()) { task.serial_compaction(); } + + if (!_humongous_compaction_regions.is_empty()) { + assert(scope()->do_maximal_compaction(), "Only compact humongous during maximal compaction"); + task.humongous_compaction(); + } } void G1FullCollector::phase5_reset_metadata() { diff --git a/src/hotspot/share/gc/g1/g1FullCollector.hpp b/src/hotspot/share/gc/g1/g1FullCollector.hpp index daa7dc34743..619a8d0548b 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.hpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.hpp @@ -76,15 +76,18 @@ class G1FullCollector : StackObj { G1FullGCScope _scope; uint _num_workers; bool _has_compaction_targets; + bool _has_humongous; G1FullGCMarker** _markers; G1FullGCCompactionPoint** _compaction_points; OopQueueSet _oop_queue_set; ObjArrayTaskQueueSet _array_queue_set; PreservedMarksSet _preserved_marks_set; G1FullGCCompactionPoint _serial_compaction_point; + G1FullGCCompactionPoint _humongous_compaction_point; G1IsAliveClosure _is_alive; ReferenceProcessorIsAliveMutator _is_alive_mutator; G1RegionMarkStats* _live_stats; + GrowableArrayCHeap _humongous_compaction_regions; static uint calc_active_workers(); @@ -115,6 +118,7 @@ public: ObjArrayTaskQueueSet* array_queue_set() { return &_array_queue_set; } PreservedMarksSet* preserved_mark_set() { return &_preserved_marks_set; } G1FullGCCompactionPoint* serial_compaction_point() { return &_serial_compaction_point; } + G1FullGCCompactionPoint* humongous_compaction_point() { return &_humongous_compaction_point; } G1CMBitMap* mark_bitmap(); ReferenceProcessor* reference_processor(); size_t live_words(uint region_index) const { @@ -134,6 +138,7 @@ public: inline void set_free(uint region_idx); inline bool is_free(uint region_idx) const; inline void update_from_compacting_to_skip_compacting(uint region_idx); + inline void update_from_skip_compacting_to_compacting(uint region_idx); inline void set_compaction_top(HeapRegion* r, HeapWord* value); inline HeapWord* compaction_top(HeapRegion* r) const; @@ -141,8 +146,14 @@ public: inline void set_has_compaction_targets(); inline bool has_compaction_targets() const; + inline void add_humongous_region(HeapRegion* hr); + inline GrowableArrayCHeap& humongous_compaction_regions(); + uint truncate_parallel_cps(); + inline void set_has_humongous(); + inline bool has_humongous(); + private: void phase1_mark_live_objects(); void phase2_prepare_compaction(); @@ -150,6 +161,7 @@ private: void phase2a_determine_worklists(); bool phase2b_forward_oops(); void phase2c_prepare_serial_compaction(); + void phase2d_prepare_humongous_compaction(); void phase3_adjust_pointers(); void phase4_do_compaction(); diff --git a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp index 9542161eb54..353143a6219 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.inline.hpp @@ -61,6 +61,11 @@ void G1FullCollector::update_from_compacting_to_skip_compacting(uint region_idx) _region_attr_table.set_skip_compacting(region_idx); } +void G1FullCollector::update_from_skip_compacting_to_compacting(uint region_idx) { + DEBUG_ONLY(_region_attr_table.verify_is_skip_compacting(region_idx);) + _region_attr_table.set_compacting(region_idx); +} + void G1FullCollector::set_compaction_top(HeapRegion* r, HeapWord* value) { Atomic::store(&_compaction_tops[r->hrm_index()], value); } @@ -79,5 +84,23 @@ bool G1FullCollector::has_compaction_targets() const { return _has_compaction_targets; } +void G1FullCollector::set_has_humongous() { + if (!_has_humongous) { + _has_humongous = true; + } +} + +bool G1FullCollector::has_humongous() { + return _has_humongous; +} + +void G1FullCollector::add_humongous_region(HeapRegion* hr) { + _humongous_compaction_regions.append(hr); +} + +GrowableArrayCHeap& G1FullCollector::humongous_compaction_regions() { + return _humongous_compaction_regions; +} + #endif // SHARE_GC_G1_G1FULLCOLLECTOR_INLINE_HPP diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp index 2fe65413ff6..8cc97c763f5 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.cpp @@ -42,16 +42,7 @@ void G1FullGCCompactTask::G1CompactRegionClosure::clear_in_bitmap(oop obj) { size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { size_t size = obj->size(); if (obj->is_forwarded()) { - HeapWord* destination = cast_from_oop(obj->forwardee()); - - // copy object and reinit its mark - HeapWord* obj_addr = cast_from_oop(obj); - assert(obj_addr != destination, "everything in this pass should be moving"); - Copy::aligned_conjoint_words(obj_addr, destination, size); - - // There is no need to transform stack chunks - marking already did that. - cast_to_oop(destination)->init_mark(); - assert(cast_to_oop(destination)->klass() != NULL, "should have a class"); + G1FullGCCompactTask::copy_object_to_new_location(obj); } // Clear the mark for the compacted object to allow reuse of the @@ -60,6 +51,21 @@ size_t G1FullGCCompactTask::G1CompactRegionClosure::apply(oop obj) { return size; } +void G1FullGCCompactTask::copy_object_to_new_location(oop obj) { + assert(obj->is_forwarded(), "Sanity!"); + assert(obj->forwardee() != obj, "Object must have a new location"); + + size_t size = obj->size(); + // Copy object and reinit its mark. + HeapWord* obj_addr = cast_from_oop(obj); + HeapWord* destination = cast_from_oop(obj->forwardee()); + Copy::aligned_conjoint_words(obj_addr, destination, size); + + // There is no need to transform stack chunks - marking already did that. + cast_to_oop(destination)->init_mark(); + assert(cast_to_oop(destination)->klass() != nullptr, "should have a class"); +} + void G1FullGCCompactTask::compact_region(HeapRegion* hr) { assert(!hr->is_pinned(), "Should be no pinned region in compaction queue"); assert(!hr->is_humongous(), "Should be no humongous regions in compaction queue"); @@ -98,3 +104,49 @@ void G1FullGCCompactTask::serial_compaction() { compact_region(*it); } } + +void G1FullGCCompactTask::humongous_compaction() { + GCTraceTime(Debug, gc, phases) tm("Phase 4: Humonguous Compaction", collector()->scope()->timer()); + + for (HeapRegion* hr : collector()->humongous_compaction_regions()) { + assert(collector()->is_compaction_target(hr->hrm_index()), "Sanity"); + compact_humongous_obj(hr); + } +} + +void G1FullGCCompactTask::compact_humongous_obj(HeapRegion* src_hr) { + assert(src_hr->is_starts_humongous(), "Should be start region of the humongous object"); + + oop obj = cast_to_oop(src_hr->bottom()); + size_t word_size = obj->size(); + + uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(word_size); + HeapWord* destination = cast_from_oop(obj->forwardee()); + + assert(collector()->mark_bitmap()->is_marked(obj), "Should only compact marked objects"); + collector()->mark_bitmap()->clear(obj); + + copy_object_to_new_location(obj); + + uint dest_start_idx = _g1h->addr_to_region(destination); + // Update the metadata for the destination regions. + _g1h->set_humongous_metadata(_g1h->region_at(dest_start_idx), num_regions, word_size, false); + + // Free the source regions that do not overlap with the destination regions. + uint src_start_idx = src_hr->hrm_index(); + free_non_overlapping_regions(src_start_idx, dest_start_idx, num_regions); +} + +void G1FullGCCompactTask::free_non_overlapping_regions(uint src_start_idx, uint dest_start_idx, uint num_regions) { + uint dest_end_idx = dest_start_idx + num_regions -1; + uint src_end_idx = src_start_idx + num_regions - 1; + + uint non_overlapping_start = dest_end_idx < src_start_idx ? + src_start_idx : + dest_end_idx + 1; + + for (uint i = non_overlapping_start; i <= src_end_idx; ++i) { + HeapRegion* hr = _g1h->region_at(i); + _g1h->free_humongous_region(hr, nullptr); + } +} diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp b/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp index 8cf3a017cbe..a7e2ea38c17 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactTask.hpp @@ -38,16 +38,24 @@ class G1FullCollector; class G1FullGCCompactTask : public G1FullGCTask { G1FullCollector* _collector; HeapRegionClaimer _claimer; + G1CollectedHeap* _g1h; void compact_region(HeapRegion* hr); + void compact_humongous_obj(HeapRegion* hr); + void free_non_overlapping_regions(uint src_start_idx, uint dest_start_idx, uint num_regions); + + static void copy_object_to_new_location(oop obj); public: G1FullGCCompactTask(G1FullCollector* collector) : G1FullGCTask("G1 Compact Task", collector), _collector(collector), - _claimer(collector->workers()) { } + _claimer(collector->workers()), + _g1h(G1CollectedHeap::heap()) { } + void work(uint worker_id); void serial_compaction(); + void humongous_compaction(); class G1CompactRegionClosure : public StackObj { G1CMBitMap* _bitmap; diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp index 3142fd1da59..b17b857c246 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp @@ -26,6 +26,7 @@ #include "gc/g1/g1FullCollector.inline.hpp" #include "gc/g1/g1FullGCCompactionPoint.hpp" #include "gc/g1/heapRegion.hpp" +#include "gc/shared/preservedMarks.inline.hpp" #include "oops/oop.inline.hpp" #include "utilities/debug.hpp" @@ -130,3 +131,84 @@ void G1FullGCCompactionPoint::remove_at_or_above(uint bottom) { assert(start_index >= 0, "Should have at least one region"); _compaction_regions->trunc_to(start_index); } + +void G1FullGCCompactionPoint::add_humongous(HeapRegion* hr) { + assert(hr->is_starts_humongous(), "Sanity!"); + + _collector->add_humongous_region(hr); + + G1CollectedHeap* g1h = G1CollectedHeap::heap(); + do { + add(hr); + _collector->update_from_skip_compacting_to_compacting(hr->hrm_index()); + hr = g1h->next_region_in_humongous(hr); + } while (hr != nullptr); +} + +uint G1FullGCCompactionPoint::forward_humongous(HeapRegion* hr) { + assert(hr->is_starts_humongous(), "Sanity!"); + + oop obj = cast_to_oop(hr->bottom()); + size_t obj_size = obj->size(); + uint num_regions = (uint)G1CollectedHeap::humongous_obj_size_in_regions(obj_size); + + if (!has_regions()) { + return num_regions; + } + + // Find contiguous compaction target regions for the humongous object. + uint range_begin = find_contiguous_before(hr, num_regions); + + if (range_begin == UINT_MAX) { + // No contiguous compaction target regions found, so the object cannot be moved. + return num_regions; + } + + // Preserve the mark for the humongous object as the region was initially not compacting. + _collector->marker(0)->preserved_stack()->push_if_necessary(obj, obj->mark()); + + HeapRegion* dest_hr = _compaction_regions->at(range_begin); + obj->forward_to(cast_to_oop(dest_hr->bottom())); + assert(obj->is_forwarded(), "Object must be forwarded!"); + + // Add the humongous object regions to the compaction point. + add_humongous(hr); + + // Remove covered regions from compaction target candidates. + _compaction_regions->remove_range(range_begin, (range_begin + num_regions)); + + return num_regions; +} + +uint G1FullGCCompactionPoint::find_contiguous_before(HeapRegion* hr, uint num_regions) { + assert(num_regions > 0, "Sanity!"); + assert(has_regions(), "Sanity!"); + + if (num_regions == 1) { + // If only one region, return the first region. + return 0; + } + + uint contiguous_region_count = 1; + + uint range_end = 1; + uint range_limit = (uint)_compaction_regions->length(); + + for (; range_end < range_limit; range_end++) { + if (contiguous_region_count == num_regions) { + break; + } + // Check if the current region and the previous region are contiguous. + bool regions_are_contiguous = (_compaction_regions->at(range_end)->hrm_index() - _compaction_regions->at(range_end - 1)->hrm_index()) == 1; + contiguous_region_count = regions_are_contiguous ? contiguous_region_count + 1 : 1; + } + + if (contiguous_region_count < num_regions && + hr->hrm_index() - _compaction_regions->at(range_end-1)->hrm_index() != 1) { + // We reached the end but the final region is not contiguous with the target region; + // no contiguous regions to move to. + return UINT_MAX; + } + // Return the index of the first region in the range of contiguous regions. + return range_end - contiguous_region_count; +} diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp index 4b9be9a12f2..ea8351f1299 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp @@ -28,6 +28,7 @@ #include "memory/allocation.hpp" #include "oops/oopsHierarchy.hpp" #include "utilities/growableArray.hpp" +#include "utilities/pair.hpp" class G1FullCollector; class HeapRegion; @@ -43,6 +44,7 @@ class G1FullGCCompactionPoint : public CHeapObj { void initialize_values(); void switch_region(); HeapRegion* next_region(); + uint find_contiguous_before(HeapRegion* hr, uint num_regions); public: G1FullGCCompactionPoint(G1FullCollector* collector); @@ -53,7 +55,9 @@ public: void initialize(HeapRegion* hr); void update(); void forward(oop object, size_t size); + uint forward_humongous(HeapRegion* hr); void add(HeapRegion* hr); + void add_humongous(HeapRegion* hr); void remove_at_or_above(uint bottom); HeapRegion* current_region(); diff --git a/src/hotspot/share/gc/g1/g1FullGCHeapRegionAttr.hpp b/src/hotspot/share/gc/g1/g1FullGCHeapRegionAttr.hpp index 017877785f4..ff613c9e4af 100644 --- a/src/hotspot/share/gc/g1/g1FullGCHeapRegionAttr.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCHeapRegionAttr.hpp @@ -84,6 +84,8 @@ public: void verify_is_compacting(uint idx) { assert(get_by_index(idx) == Compacting, "invariant"); } + void verify_is_skip_compacting(uint idx) { assert(get_by_index(idx) == SkipCompacting, "invariant"); } + void verify_is_invalid(uint idx) { assert(get_by_index(idx) == Invalid, "invariant"); } }; diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp index b03f8f8182c..70fcc4b2c19 100644 --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp @@ -89,6 +89,8 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(HeapRegion* hr) { bool is_empty = !_collector->mark_bitmap()->is_marked(obj); if (is_empty) { free_pinned_region(hr); + } else { + _collector->set_has_humongous(); } } else if (hr->is_open_archive()) { bool is_empty = _collector->live_words(hr->hrm_index()) == 0; diff --git a/src/hotspot/share/gc/g1/heapRegion.cpp b/src/hotspot/share/gc/g1/heapRegion.cpp index f232b12a4ee..1c47b92b5f7 100644 --- a/src/hotspot/share/gc/g1/heapRegion.cpp +++ b/src/hotspot/share/gc/g1/heapRegion.cpp @@ -116,9 +116,7 @@ void HeapRegion::unlink_from_list() { } void HeapRegion::hr_clear(bool clear_space) { - assert(_humongous_start_region == NULL, - "we should have already filtered out humongous regions"); - + set_top(bottom()); clear_young_index_in_cset(); clear_index_in_opt_cset(); uninstall_surv_rate_group(); diff --git a/src/hotspot/share/gc/g1/heapRegion.inline.hpp b/src/hotspot/share/gc/g1/heapRegion.inline.hpp index 3bf5a156b5a..4b87d0c9284 100644 --- a/src/hotspot/share/gc/g1/heapRegion.inline.hpp +++ b/src/hotspot/share/gc/g1/heapRegion.inline.hpp @@ -181,8 +181,6 @@ inline size_t HeapRegion::block_size(const HeapWord* p, HeapWord* const pb) cons } inline void HeapRegion::reset_compacted_after_full_gc(HeapWord* new_top) { - assert(!is_pinned(), "must be"); - set_top(new_top); // After a compaction the mark bitmap in a non-pinned regions is invalid. // But all objects are live, we get this by setting TAMS to bottom. diff --git a/src/hotspot/share/utilities/growableArray.hpp b/src/hotspot/share/utilities/growableArray.hpp index 4cc77b40708..be3a0f935d6 100644 --- a/src/hotspot/share/utilities/growableArray.hpp +++ b/src/hotspot/share/utilities/growableArray.hpp @@ -254,10 +254,18 @@ public: // Remove all elements up to the index (exclusive). The order is preserved. void remove_till(int idx) { - for (int i = 0, j = idx; j < length(); i++, j++) { + remove_range(0, idx); + } + + // Remove all elements in the range [start - end). The order is preserved. + void remove_range(int start, int end) { + assert(0 <= start, "illegal index"); + assert(start < end && end <= _len, "erase called with invalid range"); + + for (int i = start, j = end; j < length(); i++, j++) { at_put(i, at(j)); } - trunc_to(length() - idx); + trunc_to(length() - (end - start)); } // The order is changed. diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocHumongousFragment.java b/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java similarity index 94% rename from test/hotspot/jtreg/gc/shenandoah/TestAllocHumongousFragment.java rename to test/hotspot/jtreg/gc/TestAllocHumongousFragment.java index 4c75af3fce5..88b838b4bde 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocHumongousFragment.java +++ b/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java @@ -1,4 +1,5 @@ /* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -164,6 +165,18 @@ * TestAllocHumongousFragment */ + /* + * @test id=g1 + * @summary Make sure G1 can recover from humongous allocation fragmentation + * @key randomness + * @requires vm.gc.G1 + * @library /test/lib + * + * @run main/othervm -Xlog:gc+region=trace -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:VerifyGCType=full -XX:+VerifyDuringGC -XX:+VerifyAfterGC + * TestAllocHumongousFragment + */ + import java.util.*; import jdk.test.lib.Utils;