diff --git a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp index 600512c456b..bef2fc5607e 100644 --- a/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp +++ b/src/hotspot/share/gc/shared/c2/barrierSetC2.hpp @@ -278,6 +278,9 @@ public: virtual bool optimize_loops(PhaseIdealLoop* phase, LoopOptsMode mode, VectorSet& visited, Node_Stack& nstack, Node_List& worklist) const { return false; } virtual bool strip_mined_loops_expanded(LoopOptsMode mode) const { return false; } virtual bool is_gc_specific_loop_opts_pass(LoopOptsMode mode) const { return false; } + // Estimated size of the node barrier in number of C2 Ideal nodes. + // This is used to guide heuristics in C2, e.g. whether to unroll a loop. + virtual uint estimated_barrier_size(const Node* node) const { return 0; } enum CompilePhase { BeforeOptimize, diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp index 70b9bd6eaa7..4c876500f70 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -320,6 +320,20 @@ void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) { ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast(this)); } +uint ZBarrierSetC2::estimated_barrier_size(const Node* node) const { + uint8_t barrier_data = MemNode::barrier_data(node); + assert(barrier_data != 0, "should be a barrier node"); + uint uncolor_or_color_size = node->is_Load() ? 1 : 2; + if ((barrier_data & ZBarrierElided) != 0) { + return uncolor_or_color_size; + } + // A compare and branch corresponds to approximately four fast-path Ideal + // nodes (Cmp, Bool, If, If projection). The slow path (If projection and + // runtime call) is excluded since the corresponding code is laid out + // separately and does not directly affect performance. + return uncolor_or_color_size + 4; +} + void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const { return new (comp_arena) ZBarrierSetC2State(comp_arena); } diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp index 7af70c64096..e4b3be4dadc 100644 --- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp +++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp @@ -128,6 +128,7 @@ protected: const Type* val_type) const; public: + virtual uint estimated_barrier_size(const Node* node) const; virtual void* create_barrier_state(Arena* comp_arena) const; virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type, diff --git a/src/hotspot/share/opto/loopTransform.cpp b/src/hotspot/share/opto/loopTransform.cpp index 3d3e7bdaa7c..a23285fe389 100644 --- a/src/hotspot/share/opto/loopTransform.cpp +++ b/src/hotspot/share/opto/loopTransform.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,6 +24,8 @@ #include "precompiled.hpp" #include "compiler/compileLog.hpp" +#include "gc/shared/barrierSet.hpp" +#include "gc/shared/c2/barrierSetC2.hpp" #include "memory/allocation.inline.hpp" #include "opto/addnode.hpp" #include "opto/callnode.hpp" @@ -996,9 +998,12 @@ bool IdealLoopTree::policy_unroll(PhaseIdealLoop *phase) { uint body_size = _body.size(); // Key test to unroll loop in CRC32 java code int xors_in_loop = 0; - // Also count ModL, DivL and MulL which expand mightly + // Also count ModL, DivL, MulL, and other nodes that expand mightly for (uint k = 0; k < _body.size(); k++) { Node* n = _body.at(k); + if (MemNode::barrier_data(n) != 0) { + body_size += BarrierSet::barrier_set()->barrier_set_c2()->estimated_barrier_size(n); + } switch (n->Opcode()) { case Op_XorI: xors_in_loop++; break; // CRC32 java code case Op_ModL: body_size += 30; break; diff --git a/src/hotspot/share/opto/memnode.cpp b/src/hotspot/share/opto/memnode.cpp index 50eaac1c8d0..010a13a07ba 100644 --- a/src/hotspot/share/opto/memnode.cpp +++ b/src/hotspot/share/opto/memnode.cpp @@ -839,6 +839,15 @@ const TypePtr* MemNode::calculate_adr_type(const Type* t, const TypePtr* cross_c } } +uint8_t MemNode::barrier_data(const Node* n) { + if (n->is_LoadStore()) { + return n->as_LoadStore()->barrier_data(); + } else if (n->is_Mem()) { + return n->as_Mem()->barrier_data(); + } + return 0; +} + //============================================================================= // Should LoadNode::Ideal() attempt to remove control edges? bool LoadNode::can_remove_control() const { diff --git a/src/hotspot/share/opto/memnode.hpp b/src/hotspot/share/opto/memnode.hpp index c7fa14cb96d..e0f5a437413 100644 --- a/src/hotspot/share/opto/memnode.hpp +++ b/src/hotspot/share/opto/memnode.hpp @@ -126,6 +126,9 @@ public: #endif } + // Return the barrier data of n, if available, or 0 otherwise. + static uint8_t barrier_data(const Node* n); + // Map a load or store opcode to its corresponding store opcode. // (Return -1 if unknown.) virtual int store_Opcode() const { return -1; } diff --git a/test/hotspot/jtreg/compiler/gcbarriers/TestZGCUnrolling.java b/test/hotspot/jtreg/compiler/gcbarriers/TestZGCUnrolling.java new file mode 100644 index 00000000000..618b03e4cfb --- /dev/null +++ b/test/hotspot/jtreg/compiler/gcbarriers/TestZGCUnrolling.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package compiler.gcbarriers; + +import compiler.lib.ir_framework.*; +import java.lang.invoke.VarHandle; +import java.lang.invoke.MethodHandles; + +/** + * @test + * @summary Test that the expanded size of ZGC barriers is taken into account in + * C2's loop unrolling heuristics so that over-unrolling is avoided. + * The tests use volatile memory accesses to prevent C2 from simply + * optimizing them away. + * @library /test/lib / + * @requires vm.gc.ZGenerational + * @run driver compiler.gcbarriers.TestZGCUnrolling + */ + +public class TestZGCUnrolling { + + static class Outer { + Object f; + } + + static final VarHandle fVarHandle; + static { + MethodHandles.Lookup l = MethodHandles.lookup(); + try { + fVarHandle = l.findVarHandle(Outer.class, "f", Object.class); + } catch (Exception e) { + throw new Error(e); + } + } + + public static void main(String[] args) { + TestFramework.runWithFlags("-XX:+UseZGC", "-XX:+ZGenerational", + "-XX:LoopUnrollLimit=24"); + } + + @Test + @IR(counts = {IRNode.STORE_P, "1"}) + public static void testNoUnrolling(Outer o, Object o1) { + for (int i = 0; i < 64; i++) { + fVarHandle.setVolatile(o, o1); + } + } + + @Run(test = {"testNoUnrolling"}) + void run() { + testNoUnrolling(new Outer(), new Object()); + } +}