diff --git a/src/hotspot/share/gc/z/zBarrierSet.cpp b/src/hotspot/share/gc/z/zBarrierSet.cpp index 643eba1947e..15b694b2ecc 100644 --- a/src/hotspot/share/gc/z/zBarrierSet.cpp +++ b/src/hotspot/share/gc/z/zBarrierSet.cpp @@ -217,11 +217,10 @@ static void deoptimize_allocation(JavaThread* thread) { void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) { const ZPage* const page = ZHeap::heap()->page(to_zaddress(new_obj)); - const ZPageAge age = page->age(); - if (age == ZPageAge::old) { + if (!page->allows_raw_null()) { // We promised C2 that its allocations would end up in young gen. This object - // breaks that promise. Take a few steps in the interpreter instead, which has - // no such assumptions about where an object resides. + // is too old to guarantee that. Take a few steps in the interpreter instead, + // which does not elide barriers based on the age of an object. deoptimize_allocation(thread); } } diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp index 2b632ef29a9..a8c646bf895 100644 --- a/src/hotspot/share/gc/z/zGeneration.cpp +++ b/src/hotspot/share/gc/z/zGeneration.cpp @@ -190,7 +190,8 @@ void ZGeneration::flip_age_pages(const ZRelocationSetSelector* selector) { ZRendezvousHandshakeClosure cl; Handshake::execute(&cl); - _relocate.barrier_flip_promoted_pages(_relocation_set.flip_promoted_pages()); + _relocate.barrier_promoted_pages(_relocation_set.flip_promoted_pages(), + _relocation_set.relocate_promoted_pages()); } static double fragmentation_limit(ZGenerationId generation) { diff --git a/src/hotspot/share/gc/z/zPage.cpp b/src/hotspot/share/gc/z/zPage.cpp index 9f4654a655f..b6e0d162ef0 100644 --- a/src/hotspot/share/gc/z/zPage.cpp +++ b/src/hotspot/share/gc/z/zPage.cpp @@ -41,7 +41,8 @@ ZPage::ZPage(ZPageType type, ZPageAge age, const ZVirtualMemory& vmem, ZMultiPar _top(to_zoffset_end(start())), _livemap(object_max_count()), _remembered_set(), - _multi_partition_tracker(multi_partition_tracker) { + _multi_partition_tracker(multi_partition_tracker), + _relocate_promoted(false) { assert(!_virtual.is_null(), "Should not be null"); assert((_type == ZPageType::small && size() == ZPageSizeSmall) || (_type == ZPageType::medium && ZPageSizeMediumMin <= size() && size() <= ZPageSizeMediumMax) || @@ -70,6 +71,14 @@ ZPage* ZPage::clone_for_promotion() const { return page; } +bool ZPage::allows_raw_null() const { + return is_young() && !AtomicAccess::load(&_relocate_promoted); +} + +void ZPage::set_is_relocate_promoted() { + AtomicAccess::store(&_relocate_promoted, true); +} + ZGeneration* ZPage::generation() { return ZGeneration::generation(_generation_id); } diff --git a/src/hotspot/share/gc/z/zPage.hpp b/src/hotspot/share/gc/z/zPage.hpp index 96900a37680..0fc134e170a 100644 --- a/src/hotspot/share/gc/z/zPage.hpp +++ b/src/hotspot/share/gc/z/zPage.hpp @@ -52,6 +52,7 @@ private: ZLiveMap _livemap; ZRememberedSet _remembered_set; ZMultiPartitionTracker* const _multi_partition_tracker; + volatile bool _relocate_promoted; const char* type_to_string() const; @@ -103,6 +104,9 @@ public: ZPageAge age() const; + bool allows_raw_null() const; + void set_is_relocate_promoted(); + uint32_t seqnum() const; bool is_allocating() const; bool is_relocatable() const; diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp index 24c4bdeac16..da07f67d859 100644 --- a/src/hotspot/share/gc/z/zRelocate.cpp +++ b/src/hotspot/share/gc/z/zRelocate.cpp @@ -1366,27 +1366,35 @@ public: class ZPromoteBarrierTask : public ZTask { private: - ZArrayParallelIterator _iter; + ZArrayParallelIterator _flip_promoted_iter; + ZArrayParallelIterator _relocate_promoted_iter; public: - ZPromoteBarrierTask(const ZArray* pages) + ZPromoteBarrierTask(const ZArray* flip_promoted_pages, + const ZArray* relocate_promoted_pages) : ZTask("ZPromoteBarrierTask"), - _iter(pages) {} + _flip_promoted_iter(flip_promoted_pages), + _relocate_promoted_iter(relocate_promoted_pages) {} virtual void work() { SuspendibleThreadSetJoiner sts_joiner; - for (ZPage* page; _iter.next(&page);) { - // When promoting an object (and before relocate start), we must ensure that all - // contained zpointers are store good. The marking code ensures that for non-null - // pointers, but null pointers are ignored. This code ensures that even null pointers - // are made store good, for the promoted objects. - page->object_iterate([&](oop obj) { - ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field); - }); + auto promote_barriers = [&](ZArrayParallelIterator* iter) { + for (ZPage* page; iter->next(&page);) { + // When promoting an object (and before relocate start), we must ensure that all + // contained zpointers are store good. The marking code ensures that for non-null + // pointers, but null pointers are ignored. This code ensures that even null pointers + // are made store good, for the promoted objects. + page->object_iterate([&](oop obj) { + ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field); + }); - SuspendibleThreadSet::yield(); - } + SuspendibleThreadSet::yield(); + } + }; + + promote_barriers(&_flip_promoted_iter); + promote_barriers(&_relocate_promoted_iter); } }; @@ -1395,8 +1403,9 @@ void ZRelocate::flip_age_pages(const ZArray* pages) { workers()->run(&flip_age_task); } -void ZRelocate::barrier_flip_promoted_pages(const ZArray* pages) { - ZPromoteBarrierTask promote_barrier_task(pages); +void ZRelocate::barrier_promoted_pages(const ZArray* flip_promoted_pages, + const ZArray* relocate_promoted_pages) { + ZPromoteBarrierTask promote_barrier_task(flip_promoted_pages, relocate_promoted_pages); workers()->run(&promote_barrier_task); } diff --git a/src/hotspot/share/gc/z/zRelocate.hpp b/src/hotspot/share/gc/z/zRelocate.hpp index 50111f24ee5..038efba83eb 100644 --- a/src/hotspot/share/gc/z/zRelocate.hpp +++ b/src/hotspot/share/gc/z/zRelocate.hpp @@ -119,7 +119,8 @@ public: void relocate(ZRelocationSet* relocation_set); void flip_age_pages(const ZArray* pages); - void barrier_flip_promoted_pages(const ZArray* pages); + void barrier_promoted_pages(const ZArray* flip_promoted_pages, + const ZArray* relocate_promoted_pages); void synchronize(); void desynchronize(); diff --git a/src/hotspot/share/gc/z/zRelocationSet.cpp b/src/hotspot/share/gc/z/zRelocationSet.cpp index fb865934690..95ca6e56c45 100644 --- a/src/hotspot/share/gc/z/zRelocationSet.cpp +++ b/src/hotspot/share/gc/z/zRelocationSet.cpp @@ -38,6 +38,7 @@ class ZRelocationSetInstallTask : public ZTask { private: + ZRelocationSet* _relocation_set; ZForwardingAllocator* const _allocator; ZForwarding** _forwardings; const size_t _nforwardings; @@ -54,16 +55,6 @@ private: page->log_msg(" (relocation selected)"); _forwardings[index] = forwarding; - - if (forwarding->is_promotion()) { - // Before promoting an object (and before relocate start), we must ensure that all - // contained zpointers are store good. The marking code ensures that for non-null - // pointers, but null pointers are ignored. This code ensures that even null pointers - // are made store good, for the promoted objects. - page->object_iterate([&](oop obj) { - ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field); - }); - } } void install_small(ZForwarding* forwarding, size_t index) { @@ -78,10 +69,18 @@ private: return ZRelocate::compute_to_age(page->age()); } + void track_if_promoted(ZPage* page, ZForwarding* forwarding, ZArray& relocate_promoted) { + if (forwarding->is_promotion()) { + page->set_is_relocate_promoted(); + relocate_promoted.append(page); + } + } + public: - ZRelocationSetInstallTask(ZForwardingAllocator* allocator, const ZRelocationSetSelector* selector) + ZRelocationSetInstallTask(ZRelocationSet* relocation_set, const ZRelocationSetSelector* selector) : ZTask("ZRelocationSetInstallTask"), - _allocator(allocator), + _relocation_set(relocation_set), + _allocator(&relocation_set->_allocator), _forwardings(nullptr), _nforwardings((size_t)selector->selected_small()->length() + (size_t)selector->selected_medium()->length()), _small(selector->selected_small()), @@ -108,11 +107,14 @@ public: // Join the STS to block out VMThreads while running promote_barrier_on_young_oop_field SuspendibleThreadSetJoiner sts_joiner; + ZArray relocate_promoted; + // Allocate and install forwardings for small pages for (size_t page_index; _small_iter.next_index(&page_index);) { ZPage* page = _small->at(int(page_index)); ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page, to_age(page)); install_small(forwarding, (size_t)_medium->length() + page_index); + track_if_promoted(page, forwarding, relocate_promoted); SuspendibleThreadSet::yield(); } @@ -122,9 +124,12 @@ public: ZPage* page = _medium->at(int(page_index)); ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page, to_age(page)); install_medium(forwarding, page_index); + track_if_promoted(page, forwarding, relocate_promoted); SuspendibleThreadSet::yield(); } + + _relocation_set->register_relocate_promoted(relocate_promoted); } ZForwarding** forwardings() const { @@ -143,6 +148,7 @@ ZRelocationSet::ZRelocationSet(ZGeneration* generation) _nforwardings(0), _promotion_lock(), _flip_promoted_pages(), + _relocate_promoted_pages(), _in_place_relocate_promoted_pages() {} ZWorkers* ZRelocationSet::workers() const { @@ -157,9 +163,13 @@ ZArray* ZRelocationSet::flip_promoted_pages() { return &_flip_promoted_pages; } +ZArray* ZRelocationSet::relocate_promoted_pages() { + return &_relocate_promoted_pages; +} + void ZRelocationSet::install(const ZRelocationSetSelector* selector) { // Install relocation set - ZRelocationSetInstallTask task(&_allocator, selector); + ZRelocationSetInstallTask task(this, selector); workers()->run(&task); _forwardings = task.forwardings(); @@ -189,6 +199,7 @@ void ZRelocationSet::reset(ZPageAllocator* page_allocator) { destroy_and_clear(page_allocator, &_in_place_relocate_promoted_pages); destroy_and_clear(page_allocator, &_flip_promoted_pages); + _relocate_promoted_pages.clear(); } void ZRelocationSet::register_flip_promoted(const ZArray& pages) { @@ -199,6 +210,18 @@ void ZRelocationSet::register_flip_promoted(const ZArray& pages) { } } +void ZRelocationSet::register_relocate_promoted(const ZArray& pages) { + if (pages.is_empty()) { + return; + } + + ZLocker locker(&_promotion_lock); + for (ZPage* const page : pages) { + assert(!_relocate_promoted_pages.contains(page), "no duplicates allowed"); + _relocate_promoted_pages.append(page); + } +} + void ZRelocationSet::register_in_place_relocate_promoted(ZPage* page) { ZLocker locker(&_promotion_lock); assert(!_in_place_relocate_promoted_pages.contains(page), "no duplicates allowed"); diff --git a/src/hotspot/share/gc/z/zRelocationSet.hpp b/src/hotspot/share/gc/z/zRelocationSet.hpp index ee1a9447617..5f54df79805 100644 --- a/src/hotspot/share/gc/z/zRelocationSet.hpp +++ b/src/hotspot/share/gc/z/zRelocationSet.hpp @@ -37,6 +37,7 @@ class ZWorkers; class ZRelocationSet { template friend class ZRelocationSetIteratorImpl; + friend class ZRelocationSetInstallTask; private: ZGeneration* _generation; @@ -45,6 +46,7 @@ private: size_t _nforwardings; ZLock _promotion_lock; ZArray _flip_promoted_pages; + ZArray _relocate_promoted_pages; ZArray _in_place_relocate_promoted_pages; ZWorkers* workers() const; @@ -58,8 +60,10 @@ public: void reset(ZPageAllocator* page_allocator); ZGeneration* generation() const; ZArray* flip_promoted_pages(); + ZArray* relocate_promoted_pages(); void register_flip_promoted(const ZArray& pages); + void register_relocate_promoted(const ZArray& pages); void register_in_place_relocate_promoted(ZPage* page); };