diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 72129ff9e14..8a7ce7ea831 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -37,13 +37,13 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/shenandoahPrefetch.inline.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "memory/iterator.inline.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" -#include "runtime/prefetch.inline.hpp" #include "utilities/devirtualizer.inline.hpp" #include "utilities/powerOfTwo.hpp" @@ -365,6 +365,7 @@ inline void ShenandoahMark::mark_ref(ShenandoahObjToScanQueue* q, marked = mark_context->mark_strong(obj, /* was_upgraded = */ skip_live); } if (marked) { + ShenandoahPrefetch::prefetch(obj); bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak)); assert(pushed, "overflow queue should always succeed pushing"); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPrefetch.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPrefetch.inline.hpp new file mode 100644 index 00000000000..35aa297e629 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahPrefetch.inline.hpp @@ -0,0 +1,77 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHPREFETCH_INLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHPREFETCH_INLINE_HPP + +// No shenandoahPrefetch.hpp + +#include "memory/allStatic.hpp" +#include "runtime/prefetch.inline.hpp" + +// Utility to centralize prefetching decisions. +// +// Prefetching needs to strike the balance between the latency savings +// from upcoming accesses and the excess memory throughput for accesses +// that are prefetched but are never used. +// +// A common access pattern for the object in hot GC code is: +// [mark word] // sometimes, for forwarding pointer accesses +// [klass word] // very often, to discover object type +// ... +// [oop field N] // often, to traverse the heap or fix references +// +// Prefetches work on cache line granularity, so we can pick and choose +// good static offsets at which to prefetch. It also frees us from +// polling mark/klass word offsets at runtime. +// +// It stands to reason that prefetching at zero is most beneficial. +// Since it is almost guaranteed to be used by future accesses, there is +// little downside. For objects that are fully within the cache line, +// that zero-prefetch also picks up oop fields nicely. +// +// Experiments suggest it is also important to handle the case when +// object crosses the cache line. In this case, zero-prefetch is likely +// to miss the oop fields cache line. In extreme case, it can prefetch only +// the mark word, leaving klass word unprefetched. We can prefetch +// the full next cache line to deal with this case, but it is wasteful, +// especially on platforms with very large cache lines. +// +// Therefore, the second prefetch is done at some small offset to balance +// the crossing case. If second prefetch hits the same cache line as the +// first one, there is little downside. This also works automagically with +// platforms with larger cache line sizes, as both prefetches would converge. +// If prefetch hits another cache line, it likely means the object crosses +// the cache line, and that the second prefetch is profitable. +// +class ShenandoahPrefetch : AllStatic { +public: + static void prefetch(oop obj) { + void* addr = obj->base_addr(); + Prefetch::read(addr, 0); + Prefetch::read(addr, 32); + } +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHPREFETCH_INLINE_HPP