diff --git a/src/hotspot/share/gc/g1/g1FullCollector.cpp b/src/hotspot/share/gc/g1/g1FullCollector.cpp index cae06c67d6c..2a2d1da1cc5 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.cpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -327,7 +327,7 @@ void G1FullCollector::phase2_prepare_compaction() { // Try to avoid OOM immediately after Full GC in case there are no free regions // left after determining the result locations (i.e. this phase). Prepare to // maximally compact the tail regions of the compaction queues serially. - if (!has_free_compaction_targets) { + if (scope()->do_maximal_compaction() || !has_free_compaction_targets) { phase2c_prepare_serial_compaction(); } } @@ -348,37 +348,67 @@ bool G1FullCollector::phase2b_forward_oops() { return task.has_free_compaction_targets(); } -void G1FullCollector::phase2c_prepare_serial_compaction() { - GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare serial compaction", scope()->timer()); - // At this point we know that after parallel compaction there will be no - // completely free regions. That means that the last region of - // all compaction queues still have data in them. We try to compact - // these regions in serial to avoid a premature OOM when the mutator wants - // to allocate the first eden region after gc. +uint G1FullCollector::truncate_parallel_cps() { + uint lowest_current = (uint)-1; for (uint i = 0; i < workers(); i++) { G1FullGCCompactionPoint* cp = compaction_point(i); if (cp->has_regions()) { - serial_compaction_point()->add(cp->remove_last()); + lowest_current = MIN2(lowest_current, cp->current_region()->hrm_index()); } } - // Update the forwarding information for the regions in the serial - // compaction point. - G1FullGCCompactionPoint* cp = serial_compaction_point(); - for (GrowableArrayIterator it = cp->regions()->begin(); it != cp->regions()->end(); ++it) { - HeapRegion* current = *it; - if (!cp->is_initialized()) { - // Initialize the compaction point. Nothing more is needed for the first heap region - // since it is already prepared for compaction. - cp->initialize(current); - } else { - assert(!current->is_humongous(), "Should be no humongous regions in compaction queue"); - G1SerialRePrepareClosure re_prepare(cp, current); + if (lowest_current == (uint)-1) { + // worker compaction points are empty + return lowest_current; + } + + for (uint i = 0; i < workers(); i++) { + G1FullGCCompactionPoint* cp = compaction_point(i); + if (cp->has_regions()) { + cp->remove_at_or_above(lowest_current); + } + } + return lowest_current; +} + +void G1FullCollector::phase2c_prepare_serial_compaction() { + GCTraceTime(Debug, gc, phases) debug("Phase 2: Prepare serial compaction", scope()->timer()); + // At this point, we know that after parallel compaction there will be regions that + // are partially compacted into. Thus, the last compaction region of all + // compaction queues still have space in them. We try to re-compact these regions + // in serial to avoid a premature OOM when the mutator wants to allocate the first + // eden region after gc. + + // For maximum compaction, we need to re-prepare all objects above the lowest + // region among the current regions for all thread compaction points. It may + // happen that due to the uneven distribution of objects to parallel threads, holes + // have been created as threads compact to different target regions between the + // lowest and the highest region in the tails of the compaction points. + + uint start_serial = truncate_parallel_cps(); + if (start_serial >= _heap->max_reserved_regions()) { + return; + } + + G1FullGCCompactionPoint* serial_cp = serial_compaction_point(); + assert(!serial_cp->is_initialized(), "sanity!"); + + HeapRegion* start_hr = _heap->region_at(start_serial); + serial_cp->add(start_hr); + serial_cp->initialize(start_hr); + + HeapWord* dense_prefix_top = compaction_top(start_hr); + G1SerialRePrepareClosure re_prepare(serial_cp, dense_prefix_top); + + for (uint i = start_serial + 1; i < _heap->max_reserved_regions(); i++) { + if (is_compaction_target(i)) { + HeapRegion* current = _heap->region_at(i); set_compaction_top(current, current->bottom()); + serial_cp->add(current); current->apply_to_marked_objects(mark_bitmap(), &re_prepare); } } - cp->update(); + serial_cp->update(); } void G1FullCollector::phase3_adjust_pointers() { diff --git a/src/hotspot/share/gc/g1/g1FullCollector.hpp b/src/hotspot/share/gc/g1/g1FullCollector.hpp index b163a8eca8d..2fe4451f69c 100644 --- a/src/hotspot/share/gc/g1/g1FullCollector.hpp +++ b/src/hotspot/share/gc/g1/g1FullCollector.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -137,6 +137,8 @@ public: inline void set_compaction_top(HeapRegion* r, HeapWord* value); inline HeapWord* compaction_top(HeapRegion* r) const; + uint truncate_parallel_cps(); + private: void phase1_mark_live_objects(); void phase2_prepare_compaction(); diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp index e919cbfd49a..3142fd1da59 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -116,6 +116,17 @@ void G1FullGCCompactionPoint::add(HeapRegion* hr) { _compaction_regions->append(hr); } -HeapRegion* G1FullGCCompactionPoint::remove_last() { - return _compaction_regions->pop(); +void G1FullGCCompactionPoint::remove_at_or_above(uint bottom) { + HeapRegion* cur = current_region(); + assert(cur->hrm_index() >= bottom, "Sanity!"); + + int start_index = 0; + for (HeapRegion* r : *_compaction_regions) { + if (r->hrm_index() < bottom) { + start_index++; + } + } + + assert(start_index >= 0, "Should have at least one region"); + _compaction_regions->trunc_to(start_index); } diff --git a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp index 8cdb4a8dbf3..4b9be9a12f2 100644 --- a/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCCompactionPoint.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -55,7 +55,7 @@ public: void forward(oop object, size_t size); void add(HeapRegion* hr); - HeapRegion* remove_last(); + void remove_at_or_above(uint bottom); HeapRegion* current_region(); GrowableArray* regions(); diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp index a7b9b2c4630..5a54331f4eb 100644 --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -119,12 +119,12 @@ private: // serial compaction. class G1SerialRePrepareClosure : public StackObj { G1FullGCCompactionPoint* _cp; - HeapRegion* _current; + HeapWord* _dense_prefix_top; public: - G1SerialRePrepareClosure(G1FullGCCompactionPoint* hrcp, HeapRegion* hr) : + G1SerialRePrepareClosure(G1FullGCCompactionPoint* hrcp, HeapWord* dense_prefix_top) : _cp(hrcp), - _current(hr) { } + _dense_prefix_top(dense_prefix_top) { } inline size_t apply(oop obj); }; diff --git a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp index ab8ad98b05b..f48842e0127 100644 --- a/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCPrepareTask.inline.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -110,10 +110,12 @@ inline bool G1DetermineCompactionQueueClosure::do_heap_region(HeapRegion* hr) { } inline size_t G1SerialRePrepareClosure::apply(oop obj) { - // We only re-prepare objects forwarded within the current region, so - // skip objects that are already forwarded to another region. - if (obj->is_forwarded() && !_current->is_in(obj->forwardee())) { - return obj->size(); + if (obj->is_forwarded()) { + // We skip objects compiled into the first region or + // into regions not part of the serial compaction point. + if (cast_from_oop(obj->forwardee()) < _dense_prefix_top) { + return obj->size(); + } } // Get size and forward. diff --git a/src/hotspot/share/gc/g1/g1FullGCScope.cpp b/src/hotspot/share/gc/g1/g1FullGCScope.cpp index a453aebc2fa..25abb4b05ea 100644 --- a/src/hotspot/share/gc/g1/g1FullGCScope.cpp +++ b/src/hotspot/share/gc/g1/g1FullGCScope.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -43,6 +43,7 @@ G1FullGCScope::G1FullGCScope(G1MonitoringSupport* monitoring_support, G1FullGCTracer* tracer) : _rm(), _explicit_gc(explicit_gc), + _do_maximal_compaction(do_maximal_compaction), _g1h(G1CollectedHeap::heap()), _svc_marker(SvcGCMarker::FULL), _timer(), diff --git a/src/hotspot/share/gc/g1/g1FullGCScope.hpp b/src/hotspot/share/gc/g1/g1FullGCScope.hpp index 3863be8358b..5579fdf1b54 100644 --- a/src/hotspot/share/gc/g1/g1FullGCScope.hpp +++ b/src/hotspot/share/gc/g1/g1FullGCScope.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -48,6 +48,7 @@ public: class G1FullGCScope : public StackObj { ResourceMark _rm; bool _explicit_gc; + bool _do_maximal_compaction; G1CollectedHeap* _g1h; SvcGCMarker _svc_marker; STWGCTimer _timer; @@ -68,6 +69,7 @@ public: bool is_explicit_gc(); bool should_clear_soft_refs(); + bool do_maximal_compaction() { return _do_maximal_compaction; } STWGCTimer* timer(); G1FullGCTracer* tracer();