From f232f552af2df3ef5190438a4bb22f0d7ee42dd2 Mon Sep 17 00:00:00 2001 From: Michael Reeves Date: Tue, 30 Jun 2026 04:39:30 +0000 Subject: [PATCH] 8379327: 128-bit multiplication uses two multiply instructions on x86_64 Reviewed-by: dlong, sviswanathan --- src/hotspot/cpu/aarch64/aarch64.ad | 14 +-- src/hotspot/cpu/arm/arm.ad | 16 +-- src/hotspot/cpu/ppc/ppc.ad | 16 +-- src/hotspot/cpu/riscv/riscv.ad | 14 +-- src/hotspot/cpu/s390/s390.ad | 16 +-- src/hotspot/cpu/x86/x86.ad | 44 +++++-- src/hotspot/share/opto/classes.hpp | 2 + src/hotspot/share/opto/compile.cpp | 30 ++++- src/hotspot/share/opto/compile.hpp | 1 + src/hotspot/share/opto/divnode.cpp | 54 ++++---- src/hotspot/share/opto/divnode.hpp | 28 +---- src/hotspot/share/opto/matcher.hpp | 16 +-- src/hotspot/share/opto/mulnode.cpp | 32 +++++ src/hotspot/share/opto/mulnode.hpp | 29 ++++- src/hotspot/share/opto/multnode.hpp | 28 +++++ src/hotspot/share/opto/node.cpp | 25 +++- src/hotspot/share/opto/node.hpp | 2 +- .../c2/TestMultiplyHighLowFusion.java | 117 ++++++++++++++++++ .../library/Operations.java | 4 + .../vm/compiler/MultiplyHighLowFusion.java | 116 +++++++++++++++++ 20 files changed, 490 insertions(+), 114 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/c2/TestMultiplyHighLowFusion.java create mode 100644 test/micro/org/openjdk/bench/vm/compiler/MultiplyHighLowFusion.java diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 49f3419dfb6..05e4321b663 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2512,25 +2512,25 @@ uint Matcher::float_pressure_limit() return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE; } -const RegMask& Matcher::divI_proj_mask() { +const RegMask& Matcher::firstI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODI projection of divmodI. -const RegMask& Matcher::modI_proj_mask() { +// Register for the second projection of an int pair +const RegMask& Matcher::secondI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for DIVL projection of divmodL. -const RegMask& Matcher::divL_proj_mask() { +// Register for the first projection of a long pair +const RegMask& Matcher::firstL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODL projection of divmodL. -const RegMask& Matcher::modL_proj_mask() { +// Register for the second projection of a long pair +const RegMask& Matcher::secondL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } diff --git a/src/hotspot/cpu/arm/arm.ad b/src/hotspot/cpu/arm/arm.ad index 45ae283e05a..7ae3381600e 100644 --- a/src/hotspot/cpu/arm/arm.ad +++ b/src/hotspot/cpu/arm/arm.ad @@ -1112,26 +1112,26 @@ uint Matcher::float_pressure_limit() return (FLOATPRESSURE == -1) ? 30 : FLOATPRESSURE; } -// Register for DIVI projection of divmodI -const RegMask& Matcher::divI_proj_mask() { +// Register for the first projection of an int pair +const RegMask& Matcher::firstI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODI projection of divmodI -const RegMask& Matcher::modI_proj_mask() { +// Register for the second projection of an int pair +const RegMask& Matcher::secondI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for DIVL projection of divmodL -const RegMask& Matcher::divL_proj_mask() { +// Register for the first projection of a long pair +const RegMask& Matcher::firstL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODL projection of divmodL -const RegMask& Matcher::modL_proj_mask() { +// Register for the second projection of a long pair +const RegMask& Matcher::secondL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } diff --git a/src/hotspot/cpu/ppc/ppc.ad b/src/hotspot/cpu/ppc/ppc.ad index 3cdc820b5f9..e7464feb4ab 100644 --- a/src/hotspot/cpu/ppc/ppc.ad +++ b/src/hotspot/cpu/ppc/ppc.ad @@ -2351,26 +2351,26 @@ uint Matcher::float_pressure_limit() return (FLOATPRESSURE == -1) ? 28 : FLOATPRESSURE; } -// Register for DIVI projection of divmodI. -const RegMask& Matcher::divI_proj_mask() { +// Register for the first projection of an int pair +const RegMask& Matcher::firstI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODI projection of divmodI. -const RegMask& Matcher::modI_proj_mask() { +// Register for the second projection of an int pair +const RegMask& Matcher::secondI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for DIVL projection of divmodL. -const RegMask& Matcher::divL_proj_mask() { +// Register for the first projection of a long pair +const RegMask& Matcher::firstL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODL projection of divmodL. -const RegMask& Matcher::modL_proj_mask() { +// Register for the second projection of a long pair +const RegMask& Matcher::secondL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } diff --git a/src/hotspot/cpu/riscv/riscv.ad b/src/hotspot/cpu/riscv/riscv.ad index 0c077dc84a3..7bfff4b2086 100644 --- a/src/hotspot/cpu/riscv/riscv.ad +++ b/src/hotspot/cpu/riscv/riscv.ad @@ -2100,25 +2100,25 @@ uint Matcher::float_pressure_limit() return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE; } -const RegMask& Matcher::divI_proj_mask() { +const RegMask& Matcher::firstI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODI projection of divmodI. -const RegMask& Matcher::modI_proj_mask() { +// Register for the second projection of an int pair +const RegMask& Matcher::secondI_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for DIVL projection of divmodL. -const RegMask& Matcher::divL_proj_mask() { +// Register for the first projection of a long pair +const RegMask& Matcher::firstL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } -// Register for MODL projection of divmodL. -const RegMask& Matcher::modL_proj_mask() { +// Register for the second projection of a long pair +const RegMask& Matcher::secondL_proj_mask() { ShouldNotReachHere(); return RegMask::EMPTY; } diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 2208a197ac9..c0e51bd2bfd 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -1929,23 +1929,23 @@ uint Matcher::float_pressure_limit() return (FLOATPRESSURE == -1) ? 15 : FLOATPRESSURE; } -// Register for DIVI projection of divmodI -const RegMask& Matcher::divI_proj_mask() { +// Register for the first projection of an int pair +const RegMask& Matcher::firstI_proj_mask() { return _Z_RARG4_INT_REG_mask; } -// Register for MODI projection of divmodI -const RegMask& Matcher::modI_proj_mask() { +// Register for the second projection of an int pair +const RegMask& Matcher::secondI_proj_mask() { return _Z_RARG3_INT_REG_mask; } -// Register for DIVL projection of divmodL -const RegMask& Matcher::divL_proj_mask() { +// Register for the first projection of a long pair +const RegMask& Matcher::firstL_proj_mask() { return _Z_RARG4_LONG_REG_mask; } -// Register for MODL projection of divmodL -const RegMask& Matcher::modL_proj_mask() { +// Register for the second projection of a long pair +const RegMask& Matcher::secondL_proj_mask() { return _Z_RARG3_LONG_REG_mask; } diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index df035f39f58..3f953dbe725 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2764,23 +2764,23 @@ uint Matcher::float_pressure_limit() return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE; } -// Register for DIVI projection of divmodI -const RegMask& Matcher::divI_proj_mask() { +// Register for the first projection of an int pair +const RegMask& Matcher::firstI_proj_mask() { return INT_RAX_REG_mask(); } -// Register for MODI projection of divmodI -const RegMask& Matcher::modI_proj_mask() { +// Register for the second projection of an int pair +const RegMask& Matcher::secondI_proj_mask() { return INT_RDX_REG_mask(); } -// Register for DIVL projection of divmodL -const RegMask& Matcher::divL_proj_mask() { +// Register for the first projection of a long pair +const RegMask& Matcher::firstL_proj_mask() { return LONG_RAX_REG_mask(); } -// Register for MODL projection of divmodL -const RegMask& Matcher::modL_proj_mask() { +// Register for the second projection of a long pair +const RegMask& Matcher::secondL_proj_mask() { return LONG_RDX_REG_mask(); } @@ -11379,6 +11379,34 @@ instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr) ins_pipe(ialu_reg_mem_alu0); %} +instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr) +%{ + match(MulHiLoL src rax); + match(MulHiLoL rax src); + effect(KILL cr); + + ins_cost(300); + format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %} + ins_encode %{ + __ imulq($src$$Register); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + +instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr) +%{ + match(UMulHiLoL src rax); + match(UMulHiLoL rax src); + effect(KILL cr); + + ins_cost(300); + format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %} + ins_encode %{ + __ mulq($src$$Register); + %} + ins_pipe(ialu_reg_reg_alu0); +%} + instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr) %{ match(Set dst (MulHiL src rax)); diff --git a/src/hotspot/share/opto/classes.hpp b/src/hotspot/share/opto/classes.hpp index 7033dad211c..4d06e20875a 100644 --- a/src/hotspot/share/opto/classes.hpp +++ b/src/hotspot/share/opto/classes.hpp @@ -268,6 +268,8 @@ macro(MulD) macro(MulF) macro(MulHiL) macro(UMulHiL) +macro(MulHiLoL) +macro(UMulHiLoL) macro(MulI) macro(MulL) macro(Multi) diff --git a/src/hotspot/share/opto/compile.cpp b/src/hotspot/share/opto/compile.cpp index a273bb6053e..1f51cdc1d39 100644 --- a/src/hotspot/share/opto/compile.cpp +++ b/src/hotspot/share/opto/compile.cpp @@ -3276,8 +3276,8 @@ void Compile::handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned) { // DivMod node so the dependency is not lost. divmod->add_prec_from(n); divmod->add_prec_from(d); - d->subsume_by(divmod->div_proj(), this); - n->subsume_by(divmod->mod_proj(), this); + d->subsume_by(divmod->first_proj(), this); + n->subsume_by(divmod->second_proj(), this); } else { // Replace "a % b" with "a - ((a / b) * b)" Node* mult = MulNode::make(d, d->in(2), bt); @@ -3286,6 +3286,24 @@ void Compile::handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned) { } } +void Compile::handle_mulhi_mul_op(Node* n, bool is_unsigned) { + const int fused_opcode = is_unsigned ? Op_UMulHiLoL : Op_MulHiLoL; + if (!Matcher::has_match_rule(fused_opcode)) { + return; + } + + Node* mul = n->find_similar(Op_MulL, true); + + if (mul == nullptr) { + return; + } + + MulHiLoLNode* mul_hi_lo = is_unsigned ? static_cast(UMulHiLoLNode::make(n)) + : MulHiLoLNode::make(n); + mul->subsume_by(mul_hi_lo->first_proj(), this); + n->subsume_by(mul_hi_lo->second_proj(), this); +} + void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes) { switch( nop ) { case Op_Opaque1: // Remove Opaque Nodes before matching @@ -3721,6 +3739,14 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f handle_div_mod_op(n, T_LONG, true); break; + case Op_MulHiL: + handle_mulhi_mul_op(n, false); + break; + + case Op_UMulHiL: + handle_mulhi_mul_op(n, true); + break; + case Op_LoadVector: case Op_StoreVector: #ifdef ASSERT diff --git a/src/hotspot/share/opto/compile.hpp b/src/hotspot/share/opto/compile.hpp index 3c2e1c64119..ab36f59a28f 100644 --- a/src/hotspot/share/opto/compile.hpp +++ b/src/hotspot/share/opto/compile.hpp @@ -1257,6 +1257,7 @@ public: void final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes); void final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes); void handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned); + void handle_mulhi_mul_op(Node* n, bool is_unsigned); // Logic cone optimization. void optimize_logic_cones(PhaseIterGVN &igvn); diff --git a/src/hotspot/share/opto/divnode.cpp b/src/hotspot/share/opto/divnode.cpp index b398ec27b80..1687ff2cade 100644 --- a/src/hotspot/share/opto/divnode.cpp +++ b/src/hotspot/share/opto/divnode.cpp @@ -1614,12 +1614,6 @@ const Type* ModFloatingNode::Value(PhaseGVN* phase) const { //============================================================================= -DivModNode::DivModNode( Node *c, Node *dividend, Node *divisor ) : MultiNode(3) { - init_req(0, c); - init_req(1, dividend); - init_req(2, divisor); -} - DivModNode* DivModNode::make(Node* div_or_mod, BasicType bt, bool is_unsigned) { assert(bt == T_INT || bt == T_LONG, "only int or long input pattern accepted"); @@ -1645,8 +1639,8 @@ DivModINode* DivModINode::make(Node* div_or_mod) { "only div or mod input pattern accepted"); DivModINode* divmod = new DivModINode(n->in(0), n->in(1), n->in(2)); - Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num); - Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num); + Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num); + Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num); return divmod; } @@ -1657,8 +1651,8 @@ DivModLNode* DivModLNode::make(Node* div_or_mod) { "only div or mod input pattern accepted"); DivModLNode* divmod = new DivModLNode(n->in(0), n->in(1), n->in(2)); - Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num); - Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num); + Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num); + Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num); return divmod; } @@ -1667,11 +1661,11 @@ DivModLNode* DivModLNode::make(Node* div_or_mod) { Node *DivModINode::match( const ProjNode *proj, const Matcher *match ) { uint ideal_reg = proj->ideal_reg(); RegMask rm; - if (proj->_con == div_proj_num) { - rm.assignFrom(match->divI_proj_mask()); + if (proj->_con == first_proj_num) { + rm.assignFrom(match->firstI_proj_mask()); } else { - assert(proj->_con == mod_proj_num, "must be div or mod projection"); - rm.assignFrom(match->modI_proj_mask()); + assert(proj->_con == second_proj_num, "must be div or mod projection"); + rm.assignFrom(match->secondI_proj_mask()); } return new MachProjNode(this, proj->_con, rm, ideal_reg); } @@ -1682,11 +1676,11 @@ Node *DivModINode::match( const ProjNode *proj, const Matcher *match ) { Node *DivModLNode::match( const ProjNode *proj, const Matcher *match ) { uint ideal_reg = proj->ideal_reg(); RegMask rm; - if (proj->_con == div_proj_num) { - rm.assignFrom(match->divL_proj_mask()); + if (proj->_con == first_proj_num) { + rm.assignFrom(match->firstL_proj_mask()); } else { - assert(proj->_con == mod_proj_num, "must be div or mod projection"); - rm.assignFrom(match->modL_proj_mask()); + assert(proj->_con == second_proj_num, "must be div or mod projection"); + rm.assignFrom(match->secondL_proj_mask()); } return new MachProjNode(this, proj->_con, rm, ideal_reg); } @@ -1698,8 +1692,8 @@ UDivModINode* UDivModINode::make(Node* div_or_mod) { "only div or mod input pattern accepted"); UDivModINode* divmod = new UDivModINode(n->in(0), n->in(1), n->in(2)); - Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num); - Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num); + Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num); + Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num); return divmod; } @@ -1710,8 +1704,8 @@ UDivModLNode* UDivModLNode::make(Node* div_or_mod) { "only div or mod input pattern accepted"); UDivModLNode* divmod = new UDivModLNode(n->in(0), n->in(1), n->in(2)); - Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num); - Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num); + Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num); + Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num); return divmod; } @@ -1720,11 +1714,11 @@ UDivModLNode* UDivModLNode::make(Node* div_or_mod) { Node* UDivModINode::match( const ProjNode *proj, const Matcher *match ) { uint ideal_reg = proj->ideal_reg(); RegMask rm; - if (proj->_con == div_proj_num) { - rm.assignFrom(match->divI_proj_mask()); + if (proj->_con == first_proj_num) { + rm.assignFrom(match->firstI_proj_mask()); } else { - assert(proj->_con == mod_proj_num, "must be div or mod projection"); - rm.assignFrom(match->modI_proj_mask()); + assert(proj->_con == second_proj_num, "must be div or mod projection"); + rm.assignFrom(match->secondI_proj_mask()); } return new MachProjNode(this, proj->_con, rm, ideal_reg); } @@ -1735,11 +1729,11 @@ Node* UDivModINode::match( const ProjNode *proj, const Matcher *match ) { Node* UDivModLNode::match( const ProjNode *proj, const Matcher *match ) { uint ideal_reg = proj->ideal_reg(); RegMask rm; - if (proj->_con == div_proj_num) { - rm.assignFrom(match->divL_proj_mask()); + if (proj->_con == first_proj_num) { + rm.assignFrom(match->firstL_proj_mask()); } else { - assert(proj->_con == mod_proj_num, "must be div or mod projection"); - rm.assignFrom(match->modL_proj_mask()); + assert(proj->_con == second_proj_num, "must be div or mod projection"); + rm.assignFrom(match->secondL_proj_mask()); } return new MachProjNode(this, proj->_con, rm, ideal_reg); } diff --git a/src/hotspot/share/opto/divnode.hpp b/src/hotspot/share/opto/divnode.hpp index 2598429716f..366e3fb882d 100644 --- a/src/hotspot/share/opto/divnode.hpp +++ b/src/hotspot/share/opto/divnode.hpp @@ -239,36 +239,20 @@ public: //------------------------------DivModNode--------------------------------------- // Division with remainder result. -class DivModNode : public MultiNode { +class DivModNode : public BinaryMultiNode { protected: - DivModNode( Node *c, Node *dividend, Node *divisor ); + DivModNode(Node* ctrl, Node* dividend, Node* divisor) : BinaryMultiNode(ctrl, dividend, divisor) {} public: - enum { - div_proj_num = 0, // quotient - mod_proj_num = 1 // remainder - }; virtual int Opcode() const; - virtual Node* Identity(PhaseGVN* phase) { return this; } - virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { return nullptr; } - virtual const Type* Value(PhaseGVN* phase) const { return bottom_type(); } - virtual uint hash() const { return Node::hash(); } - virtual bool is_CFG() const { return false; } - virtual uint ideal_reg() const { return NotAMachineReg; } static DivModNode* make(Node* div_or_mod, BasicType bt, bool is_unsigned); - - ProjNode* div_proj() { return proj_out_or_null(div_proj_num); } - ProjNode* mod_proj() { return proj_out_or_null(mod_proj_num); } - -private: - virtual bool depends_only_on_test() const { return false; } }; //------------------------------DivModINode--------------------------------------- // Integer division with remainder result. class DivModINode : public DivModNode { public: - DivModINode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {} + DivModINode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {} virtual int Opcode() const; virtual const Type *bottom_type() const { return TypeTuple::INT_PAIR; } virtual Node *match( const ProjNode *proj, const Matcher *m ); @@ -281,7 +265,7 @@ public: // Long division with remainder result. class DivModLNode : public DivModNode { public: - DivModLNode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {} + DivModLNode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {} virtual int Opcode() const; virtual const Type *bottom_type() const { return TypeTuple::LONG_PAIR; } virtual Node *match( const ProjNode *proj, const Matcher *m ); @@ -295,7 +279,7 @@ public: // Unsigend integer division with remainder result. class UDivModINode : public DivModNode { public: - UDivModINode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {} + UDivModINode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {} virtual int Opcode() const; virtual const Type *bottom_type() const { return TypeTuple::INT_PAIR; } virtual Node *match( const ProjNode *proj, const Matcher *m ); @@ -308,7 +292,7 @@ public: // Unsigned long division with remainder result. class UDivModLNode : public DivModNode { public: - UDivModLNode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {} + UDivModLNode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {} virtual int Opcode() const; virtual const Type *bottom_type() const { return TypeTuple::LONG_PAIR; } virtual Node *match( const ProjNode *proj, const Matcher *m ); diff --git a/src/hotspot/share/opto/matcher.hpp b/src/hotspot/share/opto/matcher.hpp index 31f4a782247..2453a7ece4e 100644 --- a/src/hotspot/share/opto/matcher.hpp +++ b/src/hotspot/share/opto/matcher.hpp @@ -418,15 +418,15 @@ public: static OptoReg::Name inline_cache_reg(); static int inline_cache_reg_encode(); - // Register for DIVI projection of divmodI - static const RegMask& divI_proj_mask(); - // Register for MODI projection of divmodI - static const RegMask& modI_proj_mask(); + // Register for the first projection of an int pair + static const RegMask& firstI_proj_mask(); + // Register for the second projection of an int pair + static const RegMask& secondI_proj_mask(); - // Register for DIVL projection of divmodL - static const RegMask& divL_proj_mask(); - // Register for MODL projection of divmodL - static const RegMask& modL_proj_mask(); + // Register for the first projection of a long pair + static const RegMask& firstL_proj_mask(); + // Register for the second projection of a long pair + static const RegMask& secondL_proj_mask(); // Java-Interpreter calling convention // (what you use when calling between compiled-Java and Interpreted-Java diff --git a/src/hotspot/share/opto/mulnode.cpp b/src/hotspot/share/opto/mulnode.cpp index e48acd23b87..eb24e31eee2 100644 --- a/src/hotspot/share/opto/mulnode.cpp +++ b/src/hotspot/share/opto/mulnode.cpp @@ -26,6 +26,8 @@ #include "opto/addnode.hpp" #include "opto/connode.hpp" #include "opto/convertnode.hpp" +#include "opto/machnode.hpp" +#include "opto/matcher.hpp" #include "opto/memnode.hpp" #include "opto/mulnode.hpp" #include "opto/phaseX.hpp" @@ -606,6 +608,36 @@ const Type* UMulHiLNode::Value(PhaseGVN* phase) const { return MulHiValue(t1, t2, bot); } +MulHiLoLNode* MulHiLoLNode::make(Node* mul_hi) { + assert(mul_hi->Opcode() == Op_MulHiL, "expected MulHiL"); + + MulHiLoLNode* mul_hi_lo = new MulHiLoLNode(mul_hi->in(0), mul_hi->in(1), mul_hi->in(2)); + [[maybe_unused]] Node* lo_proj = new ProjNode(mul_hi_lo, MulHiLoLNode::first_proj_num); + [[maybe_unused]] Node* hi_proj = new ProjNode(mul_hi_lo, MulHiLoLNode::second_proj_num); + return mul_hi_lo; +} + +UMulHiLoLNode* UMulHiLoLNode::make(Node* umul_hi) { + assert(umul_hi->Opcode() == Op_UMulHiL, "expected UMulHiL"); + + UMulHiLoLNode* umul_hi_lo = new UMulHiLoLNode(umul_hi->in(0), umul_hi->in(1), umul_hi->in(2)); + [[maybe_unused]] Node* lo_proj = new ProjNode(umul_hi_lo, MulHiLoLNode::first_proj_num); + [[maybe_unused]] Node* hi_proj = new ProjNode(umul_hi_lo, MulHiLoLNode::second_proj_num); + return umul_hi_lo; +} + +Node* MulHiLoLNode::match(const ProjNode* proj, const Matcher* match) { + uint ideal_reg = proj->ideal_reg(); + RegMask rm; + if (proj->_con == first_proj_num) { + rm.assignFrom(match->firstL_proj_mask()); + } else { + assert(proj->_con == second_proj_num, "must be lo or hi projection"); + rm.assignFrom(match->secondL_proj_mask()); + } + return new MachProjNode(this, proj->_con, rm, ideal_reg); +} + // A common routine used by UMulHiLNode and MulHiLNode const Type* MulHiValue(const Type *t1, const Type *t2, const Type *bot) { // Either input is TOP ==> the result is TOP diff --git a/src/hotspot/share/opto/mulnode.hpp b/src/hotspot/share/opto/mulnode.hpp index 1e19e8ec5cd..f26137dfe49 100644 --- a/src/hotspot/share/opto/mulnode.hpp +++ b/src/hotspot/share/opto/mulnode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -25,6 +25,7 @@ #ifndef SHARE_OPTO_MULNODE_HPP #define SHARE_OPTO_MULNODE_HPP +#include "opto/multnode.hpp" #include "opto/node.hpp" #include "opto/opcodes.hpp" #include "opto/type.hpp" @@ -32,6 +33,7 @@ // Portions of code courtesy of Clifford Click class PhaseTransform; +class Matcher; //------------------------------MulNode---------------------------------------- // Classic MULTIPLY functionality. This covers all the usual 'multiply' @@ -205,6 +207,31 @@ public: friend const Type* MulHiValue(const Type *t1, const Type *t2, const Type *bot); }; +//------------------------------MulHiLoLNode----------------------------------- +// Lower and upper 64-bit results of a signed 64x64->128 multiply. +class MulHiLoLNode : public BinaryMultiNode { +protected: + MulHiLoLNode(Node* ctrl, Node* in1, Node* in2) : BinaryMultiNode(ctrl, in1, in2) {} + +public: + virtual int Opcode() const; + virtual const Type* bottom_type() const { return TypeTuple::LONG_PAIR; } + + virtual Node* match(const ProjNode* proj, const Matcher* m); + + static MulHiLoLNode* make(Node* mul_hi); +}; + +//------------------------------UMulHiLoLNode---------------------------------- +// Lower and upper 64-bit results of an unsigned 64x64->128 multiply. +class UMulHiLoLNode : public MulHiLoLNode { +public: + UMulHiLoLNode(Node* ctrl, Node* in1, Node* in2) : MulHiLoLNode(ctrl, in1, in2) {} + virtual int Opcode() const; + + static UMulHiLoLNode* make(Node* umul_hi); +}; + //------------------------------AndINode--------------------------------------- // Logically AND 2 integers. Included with the MUL nodes because it inherits // all the behavior of multiplication on a ring. diff --git a/src/hotspot/share/opto/multnode.hpp b/src/hotspot/share/opto/multnode.hpp index b63d418b742..6a69eafb7ed 100644 --- a/src/hotspot/share/opto/multnode.hpp +++ b/src/hotspot/share/opto/multnode.hpp @@ -149,6 +149,34 @@ public: ProjNode* find_first(uint which_proj, bool is_io_use) const; }; +class BinaryMultiNode : public MultiNode { +protected: + BinaryMultiNode(Node* ctrl, Node* in1, Node* in2) : MultiNode(3) { + init_req(0, ctrl); + init_req(1, in1); + init_req(2, in2); + } + +public: + enum { + first_proj_num = 0, + second_proj_num = 1 + }; + + virtual Node* Identity(PhaseGVN* phase) { return this; } + virtual Node* Ideal(PhaseGVN* phase, bool can_reshape) { return nullptr; } + virtual const Type* Value(PhaseGVN* phase) const { return bottom_type(); } + virtual uint hash() const { return Node::hash(); } + virtual bool is_CFG() const { return false; } + virtual uint ideal_reg() const { return NotAMachineReg; } + + ProjNode* first_proj() const { return proj_out_or_null(first_proj_num); } + ProjNode* second_proj() const { return proj_out_or_null(second_proj_num); } + +private: + virtual bool depends_only_on_test() const { return false; } +}; + //------------------------------ProjNode--------------------------------------- // This class defines a Projection node. Projections project a single element // out of a tuple (or Signature) type. Only MultiNodes produce TypeTuple diff --git a/src/hotspot/share/opto/node.cpp b/src/hotspot/share/opto/node.cpp index 997ce92fe1c..2f7cc6d1c1d 100644 --- a/src/hotspot/share/opto/node.cpp +++ b/src/hotspot/share/opto/node.cpp @@ -2882,7 +2882,7 @@ bool Node::is_iteratively_computed() { //--------------------------find_similar------------------------------ // Return a node with opcode "opc" and same inputs as "this" if one can // be found; Otherwise return null; -Node* Node::find_similar(int opc) { +Node* Node::find_similar(int opc, bool is_commutative) { if (req() >= 2) { Node* def = in(1); if (def && def->outcnt() >= 2) { @@ -2890,9 +2890,26 @@ Node* Node::find_similar(int opc) { Node* use = def->fast_out(i); if (use != this && use->Opcode() == opc && - use->req() == req() && - has_same_inputs_as(use)) { - return use; + use->req() == req()) { + bool same = false; + if (!is_commutative || req() < 3) { + same = use->has_same_inputs_as(this); + } else { + if (use->in(0) == in(0) && + ((use->in(1) == in(1) && use->in(2) == in(2)) || + (use->in(1) == in(2) && use->in(2) == in(1)))) { + same = true; + for (uint j = 3; j < req(); j++) { + if (use->in(j) != in(j)) { + same = false; + break; + } + } + } + } + if (same) { + return use; + } } } } diff --git a/src/hotspot/share/opto/node.hpp b/src/hotspot/share/opto/node.hpp index 1ef4b5a51b6..443f4bfbe8a 100644 --- a/src/hotspot/share/opto/node.hpp +++ b/src/hotspot/share/opto/node.hpp @@ -1315,7 +1315,7 @@ public: // Return a node with opcode "opc" and same inputs as "this" if one can // be found; Otherwise return null; - Node* find_similar(int opc); + Node* find_similar(int opc, bool is_commutative = false); bool has_same_inputs_as(const Node* other) const; // Return the unique control out if only one. Null if none or more than one. diff --git a/test/hotspot/jtreg/compiler/c2/TestMultiplyHighLowFusion.java b/test/hotspot/jtreg/compiler/c2/TestMultiplyHighLowFusion.java new file mode 100644 index 00000000000..31dd52bd3f6 --- /dev/null +++ b/test/hotspot/jtreg/compiler/c2/TestMultiplyHighLowFusion.java @@ -0,0 +1,117 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8379327 + * @summary Verify correctness for combined low/high 64-bit multiplication patterns. + * @library /test/lib / + * @run driver ${test.main.class} + */ + +package compiler.c2; + +import compiler.lib.generators.Generator; +import compiler.lib.generators.Generators; +import compiler.lib.ir_framework.*; +import java.math.BigInteger; + +public class TestMultiplyHighLowFusion { + private static final BigInteger MASK_64 = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE); + private static final Generator LONG_GEN = Generators.G.longs(); + + public static void main(String[] args) { + TestFramework.run(); + } + + @Test + @IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bMulHiLoL\\b", "1"}) + public static long doMath(long a, long b) { + long low = a * b; + long high = Math.multiplyHigh(a, b); + return low + high; + } + + @Test + @IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bMulHiLoL\\b", "1"}) + public static long doMathSwapped(long a, long b) { + long low = b * a; + long high = Math.multiplyHigh(b, a); + return low + high; + } + + @Test + @IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bUMulHiLoL\\b", "1"}) + public static long doUnsignedMath(long a, long b) { + long low = a * b; + long high = Math.unsignedMultiplyHigh(a, b); + return low + high; + } + + @Test + @IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bUMulHiLoL\\b", "1"}) + public static long doUnsignedMathSwapped(long a, long b) { + long low = b * a; + long high = Math.unsignedMultiplyHigh(b, a); + return low + high; + } + + @Run(test = {"doMath", "doMathSwapped", "doUnsignedMath", "doUnsignedMathSwapped"}) + public void runTests() { + verifyPair(LONG_GEN.next(), LONG_GEN.next()); + } + + private void verifyPair(long a, long b) { + long expectedSigned = expectedSigned(a, b); + long expectedUnsigned = expectedUnsigned(a, b); + + if (doMath(a, b) != expectedSigned) { + throw new RuntimeException("Signed mismatch for a=" + a + ", b=" + b); + } + if (doMathSwapped(a, b) != expectedSigned) { + throw new RuntimeException("Signed swapped mismatch for a=" + a + ", b=" + b); + } + if (doUnsignedMath(a, b) != expectedUnsigned) { + throw new RuntimeException("Unsigned mismatch for a=" + a + ", b=" + b); + } + if (doUnsignedMathSwapped(a, b) != expectedUnsigned) { + throw new RuntimeException("Unsigned swapped mismatch for a=" + a + ", b=" + b); + } + } + + private static long expectedSigned(long a, long b) { + BigInteger product = BigInteger.valueOf(a).multiply(BigInteger.valueOf(b)); + long low = product.longValue(); + long high = product.shiftRight(64).longValue(); + return low + high; + } + + private static long expectedUnsigned(long a, long b) { + BigInteger ua = BigInteger.valueOf(a).and(MASK_64); + BigInteger ub = BigInteger.valueOf(b).and(MASK_64); + BigInteger product = ua.multiply(ub); + long low = product.longValue(); + long high = product.shiftRight(64).longValue(); + return low + high; + } +} diff --git a/test/hotspot/jtreg/compiler/lib/template_framework/library/Operations.java b/test/hotspot/jtreg/compiler/lib/template_framework/library/Operations.java index becda83a029..3dffa096525 100644 --- a/test/hotspot/jtreg/compiler/lib/template_framework/library/Operations.java +++ b/test/hotspot/jtreg/compiler/lib/template_framework/library/Operations.java @@ -264,6 +264,10 @@ public final class Operations { ops.add(Expression.make(BOOLEANS, "Boolean.logicalOr(", BOOLEANS, ", ", BOOLEANS, ")")); ops.add(Expression.make(BOOLEANS, "Boolean.logicalXor(", BOOLEANS, ", ", BOOLEANS, ")")); + // ------------ Math ------------- + ops.add(Expression.make(LONGS, "Math.multiplyHigh(", LONGS, ", ", LONGS, ")")); + ops.add(Expression.make(LONGS, "Math.unsignedMultiplyHigh(", LONGS, ", ", LONGS, ")")); + // TODO: Math and other classes. // Note: Math.copySign is non-deterministic because of NaN having encoding with sign bit set and unset. diff --git a/test/micro/org/openjdk/bench/vm/compiler/MultiplyHighLowFusion.java b/test/micro/org/openjdk/bench/vm/compiler/MultiplyHighLowFusion.java new file mode 100644 index 00000000000..c4f4e522409 --- /dev/null +++ b/test/micro/org/openjdk/bench/vm/compiler/MultiplyHighLowFusion.java @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.bench.vm.compiler; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +import java.util.Random; +import java.util.concurrent.TimeUnit; + +/** + * Benchmarks patterns that may fuse low/high 64-bit multiply operations. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@State(Scope.Thread) +@Warmup(iterations = 4, time = 2, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 4, time = 2, timeUnit = TimeUnit.SECONDS) +@Fork(value = 3) +public class MultiplyHighLowFusion { + + @Param("1024") + private int arraySize; + + private long[] lhs; + private long[] rhs; + + @Setup + public void setup() { + Random random = new Random(0x5EED); + lhs = new long[arraySize]; + rhs = new long[arraySize]; + for (int i = 0; i < arraySize; i++) { + lhs[i] = random.nextLong(); + rhs[i] = random.nextLong(); + } + } + + @Benchmark + public long signedLowOnly() { + long sum = 0; + for (int i = 0; i < arraySize; i++) { + sum += lhs[i] * rhs[i]; + } + return sum; + } + + @Benchmark + public long signedHighOnly() { + long sum = 0; + for (int i = 0; i < arraySize; i++) { + sum += Math.multiplyHigh(lhs[i], rhs[i]); + } + return sum; + } + + @Benchmark + public long signedLowPlusHigh() { + long sum = 0; + for (int i = 0; i < arraySize; i++) { + long a = lhs[i]; + long b = rhs[i]; + sum += (a * b) + Math.multiplyHigh(a, b); + } + return sum; + } + + @Benchmark + public long unsignedHighOnly() { + long sum = 0; + for (int i = 0; i < arraySize; i++) { + sum += Math.unsignedMultiplyHigh(lhs[i], rhs[i]); + } + return sum; + } + + @Benchmark + public long unsignedLowPlusHigh() { + long sum = 0; + for (int i = 0; i < arraySize; i++) { + long a = lhs[i]; + long b = rhs[i]; + sum += (a * b) + Math.unsignedMultiplyHigh(a, b); + } + return sum; + } +}