infernap12/jdk
1
0
Fork 0
mirror of https://github.com/openjdk/jdk.git synced 2026-07-02 07:10:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

8379327: 128-bit multiplication uses two multiply instructions on x86_64

Browse Source 
Reviewed-by: dlong, sviswanathan
This commit is contained in:
Michael Reeves 2026-06-30 04:39:30 +00:00 committed by Dean Long
parent e4cd944590
commit f232f552af
20 changed files with 490 additions and 114 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/hotspot/cpu/aarch64/aarch64.ad
View File

@ -2512,25 +2512,25 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
}
const RegMask& Matcher::divI_proj_mask() {
const RegMask& Matcher::firstI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODI projection of divmodI.
const RegMask& Matcher::modI_proj_mask() {
// Register for the second projection of an int pair
const RegMask& Matcher::secondI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for DIVL projection of divmodL.
const RegMask& Matcher::divL_proj_mask() {
// Register for the first projection of a long pair
const RegMask& Matcher::firstL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODL projection of divmodL.
const RegMask& Matcher::modL_proj_mask() {
// Register for the second projection of a long pair
const RegMask& Matcher::secondL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}

16
src/hotspot/cpu/arm/arm.ad
View File

@ -1112,26 +1112,26 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 30 : FLOATPRESSURE;
}
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
// Register for the first projection of an int pair
const RegMask& Matcher::firstI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODI projection of divmodI
const RegMask& Matcher::modI_proj_mask() {
// Register for the second projection of an int pair
const RegMask& Matcher::secondI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for DIVL projection of divmodL
const RegMask& Matcher::divL_proj_mask() {
// Register for the first projection of a long pair
const RegMask& Matcher::firstL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODL projection of divmodL
const RegMask& Matcher::modL_proj_mask() {
// Register for the second projection of a long pair
const RegMask& Matcher::secondL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}

16
src/hotspot/cpu/ppc/ppc.ad
View File

@ -2351,26 +2351,26 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 28 : FLOATPRESSURE;
}
// Register for DIVI projection of divmodI.
const RegMask& Matcher::divI_proj_mask() {
// Register for the first projection of an int pair
const RegMask& Matcher::firstI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODI projection of divmodI.
const RegMask& Matcher::modI_proj_mask() {
// Register for the second projection of an int pair
const RegMask& Matcher::secondI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for DIVL projection of divmodL.
const RegMask& Matcher::divL_proj_mask() {
// Register for the first projection of a long pair
const RegMask& Matcher::firstL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODL projection of divmodL.
const RegMask& Matcher::modL_proj_mask() {
// Register for the second projection of a long pair
const RegMask& Matcher::secondL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}

14
src/hotspot/cpu/riscv/riscv.ad
View File

@ -2100,25 +2100,25 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
}
const RegMask& Matcher::divI_proj_mask() {
const RegMask& Matcher::firstI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODI projection of divmodI.
const RegMask& Matcher::modI_proj_mask() {
// Register for the second projection of an int pair
const RegMask& Matcher::secondI_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for DIVL projection of divmodL.
const RegMask& Matcher::divL_proj_mask() {
// Register for the first projection of a long pair
const RegMask& Matcher::firstL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}
// Register for MODL projection of divmodL.
const RegMask& Matcher::modL_proj_mask() {
// Register for the second projection of a long pair
const RegMask& Matcher::secondL_proj_mask() {
ShouldNotReachHere();
return RegMask::EMPTY;
}

16
src/hotspot/cpu/s390/s390.ad
View File

@ -1929,23 +1929,23 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? 15 : FLOATPRESSURE;
}
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
// Register for the first projection of an int pair
const RegMask& Matcher::firstI_proj_mask() {
return _Z_RARG4_INT_REG_mask;
}
// Register for MODI projection of divmodI
const RegMask& Matcher::modI_proj_mask() {
// Register for the second projection of an int pair
const RegMask& Matcher::secondI_proj_mask() {
return _Z_RARG3_INT_REG_mask;
}
// Register for DIVL projection of divmodL
const RegMask& Matcher::divL_proj_mask() {
// Register for the first projection of a long pair
const RegMask& Matcher::firstL_proj_mask() {
return _Z_RARG4_LONG_REG_mask;
}
// Register for MODL projection of divmodL
const RegMask& Matcher::modL_proj_mask() {
// Register for the second projection of a long pair
const RegMask& Matcher::secondL_proj_mask() {
return _Z_RARG3_LONG_REG_mask;
}

44
src/hotspot/cpu/x86/x86.ad
View File

@ -2764,23 +2764,23 @@ uint Matcher::float_pressure_limit()
return (FLOATPRESSURE == -1) ? default_float_pressure_threshold : FLOATPRESSURE;
}
// Register for DIVI projection of divmodI
const RegMask& Matcher::divI_proj_mask() {
// Register for the first projection of an int pair
const RegMask& Matcher::firstI_proj_mask() {
return INT_RAX_REG_mask();
}
// Register for MODI projection of divmodI
const RegMask& Matcher::modI_proj_mask() {
// Register for the second projection of an int pair
const RegMask& Matcher::secondI_proj_mask() {
return INT_RDX_REG_mask();
}
// Register for DIVL projection of divmodL
const RegMask& Matcher::divL_proj_mask() {
// Register for the first projection of a long pair
const RegMask& Matcher::firstL_proj_mask() {
return LONG_RAX_REG_mask();
}
// Register for MODL projection of divmodL
const RegMask& Matcher::modL_proj_mask() {
// Register for the second projection of a long pair
const RegMask& Matcher::secondL_proj_mask() {
return LONG_RDX_REG_mask();
}
@ -11379,6 +11379,34 @@ instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
ins_pipe(ialu_reg_mem_alu0);
%}
instruct mulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
%{
match(MulHiLoL src rax);
match(MulHiLoL rax src);
effect(KILL cr);
ins_cost(300);
format %{ "imulq RDX:RAX, RAX, $src\t# mulhilo" %}
ins_encode %{
__ imulq($src$$Register);
%}
ins_pipe(ialu_reg_reg_alu0);
%}
instruct umulHiLoL_rReg(rax_RegL rax, rdx_RegL rdx, rRegL src, rFlagsReg cr)
%{
match(UMulHiLoL src rax);
match(UMulHiLoL rax src);
effect(KILL cr);
ins_cost(300);
format %{ "mulq RDX:RAX, RAX, $src\t# umulhilo" %}
ins_encode %{
__ mulq($src$$Register);
%}
ins_pipe(ialu_reg_reg_alu0);
%}
instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
%{
match(Set dst (MulHiL src rax));

2
src/hotspot/share/opto/classes.hpp
View File

@ -268,6 +268,8 @@ macro(MulD)
macro(MulF)
macro(MulHiL)
macro(UMulHiL)
macro(MulHiLoL)
macro(UMulHiLoL)
macro(MulI)
macro(MulL)
macro(Multi)

30
src/hotspot/share/opto/compile.cpp
View File

@ -3276,8 +3276,8 @@ void Compile::handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned) {
// DivMod node so the dependency is not lost.
divmod->add_prec_from(n);
divmod->add_prec_from(d);
d->subsume_by(divmod->div_proj(), this);
n->subsume_by(divmod->mod_proj(), this);
d->subsume_by(divmod->first_proj(), this);
n->subsume_by(divmod->second_proj(), this);
} else {
// Replace "a % b" with "a - ((a / b) * b)"
Node* mult = MulNode::make(d, d->in(2), bt);
@ -3286,6 +3286,24 @@ void Compile::handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned) {
}
}
void Compile::handle_mulhi_mul_op(Node* n, bool is_unsigned) {
const int fused_opcode = is_unsigned ? Op_UMulHiLoL : Op_MulHiLoL;
if (!Matcher::has_match_rule(fused_opcode)) {
return;
}
Node* mul = n->find_similar(Op_MulL, true);
if (mul == nullptr) {
return;
}
MulHiLoLNode* mul_hi_lo = is_unsigned ? static_cast<MulHiLoLNode*>(UMulHiLoLNode::make(n))
: MulHiLoLNode::make(n);
mul->subsume_by(mul_hi_lo->first_proj(), this);
n->subsume_by(mul_hi_lo->second_proj(), this);
}
void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes) {
switch( nop ) {
case Op_Opaque1: // Remove Opaque Nodes before matching
@ -3721,6 +3739,14 @@ void Compile::final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& f
handle_div_mod_op(n, T_LONG, true);
break;
case Op_MulHiL:
handle_mulhi_mul_op(n, false);
break;
case Op_UMulHiL:
handle_mulhi_mul_op(n, true);
break;
case Op_LoadVector:
case Op_StoreVector:
#ifdef ASSERT

1
src/hotspot/share/opto/compile.hpp
View File

@ -1257,6 +1257,7 @@ public:
void final_graph_reshaping_main_switch(Node* n, Final_Reshape_Counts& frc, uint nop, Unique_Node_List& dead_nodes);
void final_graph_reshaping_walk(Node_Stack& nstack, Node* root, Final_Reshape_Counts& frc, Unique_Node_List& dead_nodes);
void handle_div_mod_op(Node* n, BasicType bt, bool is_unsigned);
void handle_mulhi_mul_op(Node* n, bool is_unsigned);
// Logic cone optimization.
void optimize_logic_cones(PhaseIterGVN &igvn);

54
src/hotspot/share/opto/divnode.cpp
View File

@ -1614,12 +1614,6 @@ const Type* ModFloatingNode::Value(PhaseGVN* phase) const {
//=============================================================================
DivModNode::DivModNode( Node *c, Node *dividend, Node *divisor ) : MultiNode(3) {
init_req(0, c);
init_req(1, dividend);
init_req(2, divisor);
}
DivModNode* DivModNode::make(Node* div_or_mod, BasicType bt, bool is_unsigned) {
assert(bt == T_INT || bt == T_LONG, "only int or long input pattern accepted");
@ -1645,8 +1639,8 @@ DivModINode* DivModINode::make(Node* div_or_mod) {
"only div or mod input pattern accepted");
DivModINode* divmod = new DivModINode(n->in(0), n->in(1), n->in(2));
Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num);
Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num);
return divmod;
}
@ -1657,8 +1651,8 @@ DivModLNode* DivModLNode::make(Node* div_or_mod) {
"only div or mod input pattern accepted");
DivModLNode* divmod = new DivModLNode(n->in(0), n->in(1), n->in(2));
Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num);
Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num);
return divmod;
}
@ -1667,11 +1661,11 @@ DivModLNode* DivModLNode::make(Node* div_or_mod) {
Node *DivModINode::match( const ProjNode *proj, const Matcher *match ) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == div_proj_num) {
rm.assignFrom(match->divI_proj_mask());
if (proj->_con == first_proj_num) {
rm.assignFrom(match->firstI_proj_mask());
} else {
assert(proj->_con == mod_proj_num, "must be div or mod projection");
rm.assignFrom(match->modI_proj_mask());
assert(proj->_con == second_proj_num, "must be div or mod projection");
rm.assignFrom(match->secondI_proj_mask());
}
return new MachProjNode(this, proj->_con, rm, ideal_reg);
}
@ -1682,11 +1676,11 @@ Node *DivModINode::match( const ProjNode *proj, const Matcher *match ) {
Node *DivModLNode::match( const ProjNode *proj, const Matcher *match ) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == div_proj_num) {
rm.assignFrom(match->divL_proj_mask());
if (proj->_con == first_proj_num) {
rm.assignFrom(match->firstL_proj_mask());
} else {
assert(proj->_con == mod_proj_num, "must be div or mod projection");
rm.assignFrom(match->modL_proj_mask());
assert(proj->_con == second_proj_num, "must be div or mod projection");
rm.assignFrom(match->secondL_proj_mask());
}
return new MachProjNode(this, proj->_con, rm, ideal_reg);
}
@ -1698,8 +1692,8 @@ UDivModINode* UDivModINode::make(Node* div_or_mod) {
"only div or mod input pattern accepted");
UDivModINode* divmod = new UDivModINode(n->in(0), n->in(1), n->in(2));
Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num);
Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num);
return divmod;
}
@ -1710,8 +1704,8 @@ UDivModLNode* UDivModLNode::make(Node* div_or_mod) {
"only div or mod input pattern accepted");
UDivModLNode* divmod = new UDivModLNode(n->in(0), n->in(1), n->in(2));
Node* dproj = new ProjNode(divmod, DivModNode::div_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::mod_proj_num);
Node* dproj = new ProjNode(divmod, DivModNode::first_proj_num);
Node* mproj = new ProjNode(divmod, DivModNode::second_proj_num);
return divmod;
}
@ -1720,11 +1714,11 @@ UDivModLNode* UDivModLNode::make(Node* div_or_mod) {
Node* UDivModINode::match( const ProjNode *proj, const Matcher *match ) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == div_proj_num) {
rm.assignFrom(match->divI_proj_mask());
if (proj->_con == first_proj_num) {
rm.assignFrom(match->firstI_proj_mask());
} else {
assert(proj->_con == mod_proj_num, "must be div or mod projection");
rm.assignFrom(match->modI_proj_mask());
assert(proj->_con == second_proj_num, "must be div or mod projection");
rm.assignFrom(match->secondI_proj_mask());
}
return new MachProjNode(this, proj->_con, rm, ideal_reg);
}
@ -1735,11 +1729,11 @@ Node* UDivModINode::match( const ProjNode *proj, const Matcher *match ) {
Node* UDivModLNode::match( const ProjNode *proj, const Matcher *match ) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == div_proj_num) {
rm.assignFrom(match->divL_proj_mask());
if (proj->_con == first_proj_num) {
rm.assignFrom(match->firstL_proj_mask());
} else {
assert(proj->_con == mod_proj_num, "must be div or mod projection");
rm.assignFrom(match->modL_proj_mask());
assert(proj->_con == second_proj_num, "must be div or mod projection");
rm.assignFrom(match->secondL_proj_mask());
}
return new MachProjNode(this, proj->_con, rm, ideal_reg);
}

28
src/hotspot/share/opto/divnode.hpp
View File

@ -239,36 +239,20 @@ public:
//------------------------------DivModNode---------------------------------------
// Division with remainder result.
class DivModNode : public MultiNode {
class DivModNode : public BinaryMultiNode {
protected:
DivModNode( Node *c, Node *dividend, Node *divisor );
DivModNode(Node* ctrl, Node* dividend, Node* divisor) : BinaryMultiNode(ctrl, dividend, divisor) {}
public:
enum {
div_proj_num = 0, // quotient
mod_proj_num = 1 // remainder
};
virtual int Opcode() const;
virtual Node* Identity(PhaseGVN* phase) { return this; }
virtual Node *Ideal(PhaseGVN *phase, bool can_reshape) { return nullptr; }
virtual const Type* Value(PhaseGVN* phase) const { return bottom_type(); }
virtual uint hash() const { return Node::hash(); }
virtual bool is_CFG() const { return false; }
virtual uint ideal_reg() const { return NotAMachineReg; }
static DivModNode* make(Node* div_or_mod, BasicType bt, bool is_unsigned);
ProjNode* div_proj() { return proj_out_or_null(div_proj_num); }
ProjNode* mod_proj() { return proj_out_or_null(mod_proj_num); }
private:
virtual bool depends_only_on_test() const { return false; }
};
//------------------------------DivModINode---------------------------------------
// Integer division with remainder result.
class DivModINode : public DivModNode {
public:
DivModINode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {}
DivModINode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::INT_PAIR; }
virtual Node *match( const ProjNode *proj, const Matcher *m );
@ -281,7 +265,7 @@ public:
// Long division with remainder result.
class DivModLNode : public DivModNode {
public:
DivModLNode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {}
DivModLNode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::LONG_PAIR; }
virtual Node *match( const ProjNode *proj, const Matcher *m );
@ -295,7 +279,7 @@ public:
// Unsigend integer division with remainder result.
class UDivModINode : public DivModNode {
public:
UDivModINode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {}
UDivModINode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::INT_PAIR; }
virtual Node *match( const ProjNode *proj, const Matcher *m );
@ -308,7 +292,7 @@ public:
// Unsigned long division with remainder result.
class UDivModLNode : public DivModNode {
public:
UDivModLNode( Node *c, Node *dividend, Node *divisor ) : DivModNode(c, dividend, divisor) {}
UDivModLNode(Node* ctrl, Node* dividend, Node* divisor) : DivModNode(ctrl, dividend, divisor) {}
virtual int Opcode() const;
virtual const Type *bottom_type() const { return TypeTuple::LONG_PAIR; }
virtual Node *match( const ProjNode *proj, const Matcher *m );

16
src/hotspot/share/opto/matcher.hpp
View File

@ -418,15 +418,15 @@ public:
static OptoReg::Name inline_cache_reg();
static int inline_cache_reg_encode();
// Register for DIVI projection of divmodI
static const RegMask& divI_proj_mask();
// Register for MODI projection of divmodI
static const RegMask& modI_proj_mask();
// Register for the first projection of an int pair
static const RegMask& firstI_proj_mask();
// Register for the second projection of an int pair
static const RegMask& secondI_proj_mask();
// Register for DIVL projection of divmodL
static const RegMask& divL_proj_mask();
// Register for MODL projection of divmodL
static const RegMask& modL_proj_mask();
// Register for the first projection of a long pair
static const RegMask& firstL_proj_mask();
// Register for the second projection of a long pair
static const RegMask& secondL_proj_mask();
// Java-Interpreter calling convention
// (what you use when calling between compiled-Java and Interpreted-Java

32
src/hotspot/share/opto/mulnode.cpp
View File

@ -26,6 +26,8 @@
#include "opto/addnode.hpp"
#include "opto/connode.hpp"
#include "opto/convertnode.hpp"
#include "opto/machnode.hpp"
#include "opto/matcher.hpp"
#include "opto/memnode.hpp"
#include "opto/mulnode.hpp"
#include "opto/phaseX.hpp"
@ -606,6 +608,36 @@ const Type* UMulHiLNode::Value(PhaseGVN* phase) const {
return MulHiValue(t1, t2, bot);
}
MulHiLoLNode* MulHiLoLNode::make(Node* mul_hi) {
assert(mul_hi->Opcode() == Op_MulHiL, "expected MulHiL");
MulHiLoLNode* mul_hi_lo = new MulHiLoLNode(mul_hi->in(0), mul_hi->in(1), mul_hi->in(2));
[[maybe_unused]] Node* lo_proj = new ProjNode(mul_hi_lo, MulHiLoLNode::first_proj_num);
[[maybe_unused]] Node* hi_proj = new ProjNode(mul_hi_lo, MulHiLoLNode::second_proj_num);
return mul_hi_lo;
}
UMulHiLoLNode* UMulHiLoLNode::make(Node* umul_hi) {
assert(umul_hi->Opcode() == Op_UMulHiL, "expected UMulHiL");
UMulHiLoLNode* umul_hi_lo = new UMulHiLoLNode(umul_hi->in(0), umul_hi->in(1), umul_hi->in(2));
[[maybe_unused]] Node* lo_proj = new ProjNode(umul_hi_lo, MulHiLoLNode::first_proj_num);
[[maybe_unused]] Node* hi_proj = new ProjNode(umul_hi_lo, MulHiLoLNode::second_proj_num);
return umul_hi_lo;
}
Node* MulHiLoLNode::match(const ProjNode* proj, const Matcher* match) {
uint ideal_reg = proj->ideal_reg();
RegMask rm;
if (proj->_con == first_proj_num) {
rm.assignFrom(match->firstL_proj_mask());
} else {
assert(proj->_con == second_proj_num, "must be lo or hi projection");
rm.assignFrom(match->secondL_proj_mask());
}
return new MachProjNode(this, proj->_con, rm, ideal_reg);
}
// A common routine used by UMulHiLNode and MulHiLNode
const Type* MulHiValue(const Type *t1, const Type *t2, const Type *bot) {
// Either input is TOP ==> the result is TOP

29
src/hotspot/share/opto/mulnode.hpp
View File

@ -1,5 +1,5 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1997, 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@ -25,6 +25,7 @@
#ifndef SHARE_OPTO_MULNODE_HPP
#define SHARE_OPTO_MULNODE_HPP
#include "opto/multnode.hpp"
#include "opto/node.hpp"
#include "opto/opcodes.hpp"
#include "opto/type.hpp"
@ -32,6 +33,7 @@
// Portions of code courtesy of Clifford Click
class PhaseTransform;
class Matcher;
//------------------------------MulNode----------------------------------------
// Classic MULTIPLY functionality. This covers all the usual 'multiply'
@ -205,6 +207,31 @@ public:
friend const Type* MulHiValue(const Type *t1, const Type *t2, const Type *bot);
};
//------------------------------MulHiLoLNode-----------------------------------
// Lower and upper 64-bit results of a signed 64x64->128 multiply.
class MulHiLoLNode : public BinaryMultiNode {
protected:
MulHiLoLNode(Node* ctrl, Node* in1, Node* in2) : BinaryMultiNode(ctrl, in1, in2) {}
public:
virtual int Opcode() const;
virtual const Type* bottom_type() const { return TypeTuple::LONG_PAIR; }
virtual Node* match(const ProjNode* proj, const Matcher* m);
static MulHiLoLNode* make(Node* mul_hi);
};
//------------------------------UMulHiLoLNode----------------------------------
// Lower and upper 64-bit results of an unsigned 64x64->128 multiply.
class UMulHiLoLNode : public MulHiLoLNode {
public:
UMulHiLoLNode(Node* ctrl, Node* in1, Node* in2) : MulHiLoLNode(ctrl, in1, in2) {}
virtual int Opcode() const;
static UMulHiLoLNode* make(Node* umul_hi);
};
//------------------------------AndINode---------------------------------------
// Logically AND 2 integers. Included with the MUL nodes because it inherits
// all the behavior of multiplication on a ring.

28
src/hotspot/share/opto/multnode.hpp
View File

@ -149,6 +149,34 @@ public:
ProjNode* find_first(uint which_proj, bool is_io_use) const;
};
class BinaryMultiNode : public MultiNode {
protected:
BinaryMultiNode(Node* ctrl, Node* in1, Node* in2) : MultiNode(3) {
init_req(0, ctrl);
init_req(1, in1);
init_req(2, in2);
}
public:
enum {
first_proj_num = 0,
second_proj_num = 1
};
virtual Node* Identity(PhaseGVN* phase) { return this; }
virtual Node* Ideal(PhaseGVN* phase, bool can_reshape) { return nullptr; }
virtual const Type* Value(PhaseGVN* phase) const { return bottom_type(); }
virtual uint hash() const { return Node::hash(); }
virtual bool is_CFG() const { return false; }
virtual uint ideal_reg() const { return NotAMachineReg; }
ProjNode* first_proj() const { return proj_out_or_null(first_proj_num); }
ProjNode* second_proj() const { return proj_out_or_null(second_proj_num); }
private:
virtual bool depends_only_on_test() const { return false; }
};
//------------------------------ProjNode---------------------------------------
// This class defines a Projection node. Projections project a single element
// out of a tuple (or Signature) type. Only MultiNodes produce TypeTuple

25
src/hotspot/share/opto/node.cpp
View File

@ -2882,7 +2882,7 @@ bool Node::is_iteratively_computed() {
//--------------------------find_similar------------------------------
// Return a node with opcode "opc" and same inputs as "this" if one can
// be found; Otherwise return null;
Node* Node::find_similar(int opc) {
Node* Node::find_similar(int opc, bool is_commutative) {
if (req() >= 2) {
Node* def = in(1);
if (def && def->outcnt() >= 2) {
@ -2890,9 +2890,26 @@ Node* Node::find_similar(int opc) {
Node* use = def->fast_out(i);
if (use != this &&
use->Opcode() == opc &&
use->req() == req() &&
has_same_inputs_as(use)) {
return use;
use->req() == req()) {
bool same = false;
if (!is_commutative || req() < 3) {
same = use->has_same_inputs_as(this);
} else {
if (use->in(0) == in(0) &&
((use->in(1) == in(1) && use->in(2) == in(2)) ||
(use->in(1) == in(2) && use->in(2) == in(1)))) {
same = true;
for (uint j = 3; j < req(); j++) {
if (use->in(j) != in(j)) {
same = false;
break;
}
}
}
}
if (same) {
return use;
}
}
}
}

2
src/hotspot/share/opto/node.hpp
View File

@ -1315,7 +1315,7 @@ public:
// Return a node with opcode "opc" and same inputs as "this" if one can
// be found; Otherwise return null;
Node* find_similar(int opc);
Node* find_similar(int opc, bool is_commutative = false);
bool has_same_inputs_as(const Node* other) const;
// Return the unique control out if only one. Null if none or more than one.

117
test/hotspot/jtreg/compiler/c2/TestMultiplyHighLowFusion.java Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
/*
* @test
* @bug 8379327
* @summary Verify correctness for combined low/high 64-bit multiplication patterns.
* @library /test/lib /
* @run driver ${test.main.class}
*/
package compiler.c2;
import compiler.lib.generators.Generator;
import compiler.lib.generators.Generators;
import compiler.lib.ir_framework.*;
import java.math.BigInteger;
public class TestMultiplyHighLowFusion {
private static final BigInteger MASK_64 = BigInteger.ONE.shiftLeft(64).subtract(BigInteger.ONE);
private static final Generator<Long> LONG_GEN = Generators.G.longs();
public static void main(String[] args) {
TestFramework.run();
}
@Test
@IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bMulHiLoL\\b", "1"})
public static long doMath(long a, long b) {
long low = a * b;
long high = Math.multiplyHigh(a, b);
return low + high;
}
@Test
@IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bMulHiLoL\\b", "1"})
public static long doMathSwapped(long a, long b) {
long low = b * a;
long high = Math.multiplyHigh(b, a);
return low + high;
}
@Test
@IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bUMulHiLoL\\b", "1"})
public static long doUnsignedMath(long a, long b) {
long low = a * b;
long high = Math.unsignedMultiplyHigh(a, b);
return low + high;
}
@Test
@IR(applyIfPlatform = {"x64", "true"}, phase = CompilePhase.PRINT_IDEAL, counts = {"\\bUMulHiLoL\\b", "1"})
public static long doUnsignedMathSwapped(long a, long b) {
long low = b * a;
long high = Math.unsignedMultiplyHigh(b, a);
return low + high;
}
@Run(test = {"doMath", "doMathSwapped", "doUnsignedMath", "doUnsignedMathSwapped"})
public void runTests() {
verifyPair(LONG_GEN.next(), LONG_GEN.next());
}
private void verifyPair(long a, long b) {
long expectedSigned = expectedSigned(a, b);
long expectedUnsigned = expectedUnsigned(a, b);
if (doMath(a, b) != expectedSigned) {
throw new RuntimeException("Signed mismatch for a=" + a + ", b=" + b);
}
if (doMathSwapped(a, b) != expectedSigned) {
throw new RuntimeException("Signed swapped mismatch for a=" + a + ", b=" + b);
}
if (doUnsignedMath(a, b) != expectedUnsigned) {
throw new RuntimeException("Unsigned mismatch for a=" + a + ", b=" + b);
}
if (doUnsignedMathSwapped(a, b) != expectedUnsigned) {
throw new RuntimeException("Unsigned swapped mismatch for a=" + a + ", b=" + b);
}
}
private static long expectedSigned(long a, long b) {
BigInteger product = BigInteger.valueOf(a).multiply(BigInteger.valueOf(b));
long low = product.longValue();
long high = product.shiftRight(64).longValue();
return low + high;
}
private static long expectedUnsigned(long a, long b) {
BigInteger ua = BigInteger.valueOf(a).and(MASK_64);
BigInteger ub = BigInteger.valueOf(b).and(MASK_64);
BigInteger product = ua.multiply(ub);
long low = product.longValue();
long high = product.shiftRight(64).longValue();
return low + high;
}
}

4
test/hotspot/jtreg/compiler/lib/template_framework/library/Operations.java
View File

@ -264,6 +264,10 @@ public final class Operations {
ops.add(Expression.make(BOOLEANS, "Boolean.logicalOr(", BOOLEANS, ", ", BOOLEANS, ")"));
ops.add(Expression.make(BOOLEANS, "Boolean.logicalXor(", BOOLEANS, ", ", BOOLEANS, ")"));
// ------------ Math -------------
ops.add(Expression.make(LONGS, "Math.multiplyHigh(", LONGS, ", ", LONGS, ")"));
ops.add(Expression.make(LONGS, "Math.unsignedMultiplyHigh(", LONGS, ", ", LONGS, ")"));
// TODO: Math and other classes.
// Note: Math.copySign is non-deterministic because of NaN having encoding with sign bit set and unset.

116
test/micro/org/openjdk/bench/vm/compiler/MultiplyHighLowFusion.java Normal file
View File

@ -0,0 +1,116 @@
/*
* Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.bench.vm.compiler;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.BenchmarkMode;
import org.openjdk.jmh.annotations.Fork;
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
import org.openjdk.jmh.annotations.Warmup;
import java.util.Random;
import java.util.concurrent.TimeUnit;
/**
* Benchmarks patterns that may fuse low/high 64-bit multiply operations.
*/
@BenchmarkMode(Mode.AverageTime)
@OutputTimeUnit(TimeUnit.NANOSECONDS)
@State(Scope.Thread)
@Warmup(iterations = 4, time = 2, timeUnit = TimeUnit.SECONDS)
@Measurement(iterations = 4, time = 2, timeUnit = TimeUnit.SECONDS)
@Fork(value = 3)
public class MultiplyHighLowFusion {
@Param("1024")
private int arraySize;
private long[] lhs;
private long[] rhs;
@Setup
public void setup() {
Random random = new Random(0x5EED);
lhs = new long[arraySize];
rhs = new long[arraySize];
for (int i = 0; i < arraySize; i++) {
lhs[i] = random.nextLong();
rhs[i] = random.nextLong();
}
}
@Benchmark
public long signedLowOnly() {
long sum = 0;
for (int i = 0; i < arraySize; i++) {
sum += lhs[i] * rhs[i];
}
return sum;
}
@Benchmark
public long signedHighOnly() {
long sum = 0;
for (int i = 0; i < arraySize; i++) {
sum += Math.multiplyHigh(lhs[i], rhs[i]);
}
return sum;
}
@Benchmark
public long signedLowPlusHigh() {
long sum = 0;
for (int i = 0; i < arraySize; i++) {
long a = lhs[i];
long b = rhs[i];
sum += (a * b) + Math.multiplyHigh(a, b);
}
return sum;
}
@Benchmark
public long unsignedHighOnly() {
long sum = 0;
for (int i = 0; i < arraySize; i++) {
sum += Math.unsignedMultiplyHigh(lhs[i], rhs[i]);
}
return sum;
}
@Benchmark
public long unsignedLowPlusHigh() {
long sum = 0;
for (int i = 0; i < arraySize; i++) {
long a = lhs[i];
long b = rhs[i];
sum += (a * b) + Math.unsignedMultiplyHigh(a, b);
}
return sum;
}
}