diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp index 008d52dcb1c..50ed4750d47 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.cpp @@ -1760,25 +1760,36 @@ void C2_MacroAssembler::load_vector_mask(KRegister dst, XMMRegister src, XMMRegi } } -void C2_MacroAssembler::load_vector(XMMRegister dst, Address src, int vlen_in_bytes) { - switch (vlen_in_bytes) { - case 4: movdl(dst, src); break; - case 8: movq(dst, src); break; - case 16: movdqu(dst, src); break; - case 32: vmovdqu(dst, src); break; - case 64: evmovdqul(dst, src, Assembler::AVX_512bit); break; - default: ShouldNotReachHere(); +void C2_MacroAssembler::load_vector(BasicType bt, XMMRegister dst, Address src, int vlen_in_bytes) { + if (is_integral_type(bt)) { + switch (vlen_in_bytes) { + case 4: movdl(dst, src); break; + case 8: movq(dst, src); break; + case 16: movdqu(dst, src); break; + case 32: vmovdqu(dst, src); break; + case 64: evmovdqul(dst, src, Assembler::AVX_512bit); break; + default: ShouldNotReachHere(); + } + } else { + switch (vlen_in_bytes) { + case 4: movflt(dst, src); break; + case 8: movdbl(dst, src); break; + case 16: movups(dst, src); break; + case 32: vmovups(dst, src, Assembler::AVX_256bit); break; + case 64: vmovups(dst, src, Assembler::AVX_512bit); break; + default: ShouldNotReachHere(); + } } } -void C2_MacroAssembler::load_vector(XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch) { +void C2_MacroAssembler::load_vector(BasicType bt, XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch) { assert(rscratch != noreg || always_reachable(src), "missing"); if (reachable(src)) { - load_vector(dst, as_Address(src), vlen_in_bytes); + load_vector(bt, dst, as_Address(src), vlen_in_bytes); } else { lea(rscratch, src); - load_vector(dst, Address(rscratch, 0), vlen_in_bytes); + load_vector(bt, dst, Address(rscratch, 0), vlen_in_bytes); } } @@ -1807,10 +1818,7 @@ void C2_MacroAssembler::load_constant_vector(BasicType bt, XMMRegister dst, Inte } else if (VM_Version::supports_sse3()) { movddup(dst, src); } else { - movq(dst, src); - if (vlen == 16) { - punpcklqdq(dst, dst); - } + load_vector(bt, dst, src, vlen); } } @@ -1821,7 +1829,7 @@ void C2_MacroAssembler::load_iota_indices(XMMRegister dst, int vlen_in_bytes, Ba offset += 128; } ExternalAddress addr(StubRoutines::x86::vector_iota_indices() + offset); - load_vector(dst, addr, vlen_in_bytes); + load_vector(T_BYTE, dst, addr, vlen_in_bytes); } // Reductions for vectors of bytes, shorts, ints, longs, floats, and doubles. @@ -3514,11 +3522,11 @@ void C2_MacroAssembler::arrays_hashcode_elload(Register dst, Address src, BasicT } void C2_MacroAssembler::arrays_hashcode_elvload(XMMRegister dst, Address src, BasicType eltype) { - load_vector(dst, src, arrays_hashcode_elsize(eltype) * 8); + load_vector(eltype, dst, src, arrays_hashcode_elsize(eltype) * 8); } void C2_MacroAssembler::arrays_hashcode_elvload(XMMRegister dst, AddressLiteral src, BasicType eltype) { - load_vector(dst, src, arrays_hashcode_elsize(eltype) * 8); + load_vector(eltype, dst, src, arrays_hashcode_elsize(eltype) * 8); } void C2_MacroAssembler::arrays_hashcode_elvcast(XMMRegister dst, BasicType eltype) { diff --git a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp index 3a36fd75e3f..6e49cdefa6c 100644 --- a/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/c2_MacroAssembler_x86.hpp @@ -139,8 +139,8 @@ public: void evpcmp(BasicType typ, KRegister kdmask, KRegister ksmask, XMMRegister src1, AddressLiteral src2, int comparison, int vector_len, Register rscratch = noreg); void evpblend(BasicType typ, XMMRegister dst, KRegister kmask, XMMRegister src1, XMMRegister src2, bool merge, int vector_len); - void load_vector(XMMRegister dst, Address src, int vlen_in_bytes); - void load_vector(XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = noreg); + void load_vector(BasicType bt, XMMRegister dst, Address src, int vlen_in_bytes); + void load_vector(BasicType bt, XMMRegister dst, AddressLiteral src, int vlen_in_bytes, Register rscratch = noreg); void load_vector_mask(XMMRegister dst, XMMRegister src, int vlen_in_bytes, BasicType elem_bt, bool is_legacy); void load_vector_mask(KRegister dst, XMMRegister src, XMMRegister xtmp, bool novlbwdq, int vlen_enc); diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index 09023562dd6..49efbc03a4d 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -2762,20 +2762,40 @@ void vec_spill_helper(C2_MacroAssembler *masm, bool is_load, } template -static inline GrowableArray* vreplicate_imm(BasicType bt, T con, int len) { - GrowableArray* val = new GrowableArray(len); - jvalue ele; - switch (bt) { - case T_BYTE: ele.b = con; break; - case T_SHORT: ele.s = con; break; - case T_INT: ele.i = con; break; - case T_LONG: ele.j = con; break; - case T_FLOAT: ele.f = con; break; - case T_DOUBLE: ele.d = con; break; - default: ShouldNotReachHere(); - } +static inline GrowableArray* vreplicate_imm(BasicType bt, T con, int len) { + int size = type2aelembytes(bt) * len; + GrowableArray* val = new GrowableArray(size, size, 0); for (int i = 0; i < len; i++) { - val->append(ele); + int offset = i * type2aelembytes(bt); + switch (bt) { + case T_BYTE: val->at(i) = con; break; + case T_SHORT: { + jshort c = con; + memcpy(val->adr_at(offset), &c, sizeof(jshort)); + break; + } + case T_INT: { + jint c = con; + memcpy(val->adr_at(offset), &c, sizeof(jint)); + break; + } + case T_LONG: { + jlong c = con; + memcpy(val->adr_at(offset), &c, sizeof(jlong)); + break; + } + case T_FLOAT: { + jfloat c = con; + memcpy(val->adr_at(offset), &c, sizeof(jfloat)); + break; + } + case T_DOUBLE: { + jdouble c = con; + memcpy(val->adr_at(offset), &c, sizeof(jdouble)); + break; + } + default: assert(false, "%s", type2name(bt)); + } } return val; } @@ -4060,7 +4080,8 @@ instruct loadV(vec dst, memory mem) %{ ins_cost(125); format %{ "load_vector $dst,$mem" %} ins_encode %{ - __ load_vector($dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this)); + BasicType bt = Matcher::vector_element_basic_type(this); + __ load_vector(bt, $dst$$XMMRegister, $mem$$Address, Matcher::vector_length_in_bytes(this)); %} ins_pipe( pipe_slow ); %} @@ -4543,10 +4564,9 @@ instruct ReplI_imm(vec dst, immI con) %{ match(Set dst (Replicate con)); format %{ "replicateI $dst,$con" %} ins_encode %{ - InternalAddress addr = $constantaddress(Matcher::vector_element_basic_type(this), - vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant, - (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 8) / - type2aelembytes(Matcher::vector_element_basic_type(this)))); + InternalAddress addr = $constantaddress(vreplicate_imm(Matcher::vector_element_basic_type(this), $con$$constant, + (VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 4 : 8) : 16) / + type2aelembytes(Matcher::vector_element_basic_type(this)))); BasicType bt = Matcher::vector_element_basic_type(this); int vlen = Matcher::vector_length_in_bytes(this); __ load_constant_vector(bt, $dst$$XMMRegister, addr, vlen); @@ -4684,7 +4704,7 @@ instruct ReplL_imm(vec dst, immL con) %{ match(Set dst (Replicate con)); format %{ "replicateL $dst,$con" %} ins_encode %{ - InternalAddress addr = $constantaddress(T_LONG, vreplicate_imm(T_LONG, $con$$constant, 1)); + InternalAddress addr = $constantaddress(vreplicate_imm(T_LONG, $con$$constant, VM_Version::supports_sse3() ? 1 : 2)); int vlen = Matcher::vector_length_in_bytes(this); __ load_constant_vector(T_LONG, $dst$$XMMRegister, addr, vlen); %} @@ -4766,8 +4786,8 @@ instruct ReplF_imm(vec dst, immF con) %{ match(Set dst (Replicate con)); format %{ "replicateF $dst,$con" %} ins_encode %{ - InternalAddress addr = $constantaddress(T_FLOAT, vreplicate_imm(T_FLOAT, $con$$constant, - VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 2)); + InternalAddress addr = $constantaddress(vreplicate_imm(T_FLOAT, $con$$constant, + VM_Version::supports_sse3() ? (VM_Version::supports_avx() ? 1 : 2) : 4)); int vlen = Matcher::vector_length_in_bytes(this); __ load_constant_vector(T_FLOAT, $dst$$XMMRegister, addr, vlen); %} @@ -4843,7 +4863,7 @@ instruct ReplD_imm(vec dst, immD con) %{ match(Set dst (Replicate con)); format %{ "replicateD $dst,$con" %} ins_encode %{ - InternalAddress addr = $constantaddress(T_DOUBLE, vreplicate_imm(T_DOUBLE, $con$$constant, 1)); + InternalAddress addr = $constantaddress(vreplicate_imm(T_DOUBLE, $con$$constant, VM_Version::supports_sse3() ? 1 : 2)); int vlen = Matcher::vector_length_in_bytes(this); __ load_constant_vector(T_DOUBLE, $dst$$XMMRegister, addr, vlen); %} @@ -9755,7 +9775,7 @@ instruct vreverse_reg_gfni(vec dst, vec src, vec xtmp) %{ ins_encode %{ int vec_enc = vector_length_encoding(this); BasicType bt = Matcher::vector_element_basic_type(this); - InternalAddress addr = $constantaddress(T_LONG, vreplicate_imm(T_LONG, 0x8040201008040201L, 1)); + InternalAddress addr = $constantaddress(jlong(0x8040201008040201)); __ vector_reverse_bit_gfni(bt, $dst$$XMMRegister, $src$$XMMRegister, addr, vec_enc, $xtmp$$XMMRegister); %} diff --git a/src/hotspot/share/asm/assembler.hpp b/src/hotspot/share/asm/assembler.hpp index d79a3fbf48c..9abd3eb7171 100644 --- a/src/hotspot/share/asm/assembler.hpp +++ b/src/hotspot/share/asm/assembler.hpp @@ -480,26 +480,12 @@ class AbstractAssembler : public ResourceObj { } return ptr; } - address array_constant(BasicType bt, GrowableArray* c, int alignment) { + address array_constant(const GrowableArray* c, int alignment) { CodeSection* c1 = _code_section; - int len = c->length(); - int size = type2aelembytes(bt) * len; - address ptr = start_a_const(size, alignment); + address ptr = start_a_const(c->length(), alignment); if (ptr != nullptr) { - for (int i = 0; i < len; i++) { - jvalue e = c->at(i); - switch(bt) { - case T_BOOLEAN: emit_int8(e.z); break; - case T_BYTE: emit_int8(e.b); break; - case T_CHAR: emit_int16(e.c); break; - case T_SHORT: emit_int16(e.s); break; - case T_INT: emit_int32(e.i); break; - case T_LONG: emit_int64(e.j); break; - case T_FLOAT: emit_float(e.f); break; - case T_DOUBLE: emit_double(e.d); break; - default: - ShouldNotReachHere(); - } + for (int i = 0; i < c->length(); i++) { + emit_int8(c->at(i)); } end_a_const(c1); } diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp index b25e07bc36d..4aaf8b6daa2 100644 --- a/src/hotspot/share/asm/codeBuffer.cpp +++ b/src/hotspot/share/asm/codeBuffer.cpp @@ -889,6 +889,7 @@ void CodeBuffer::expand(CodeSection* which_cs, csize_t amount) { // Create a new (temporary) code buffer to hold all the new data CodeBuffer cb(name(), new_total_cap, 0); + cb.set_const_section_alignment(_const_section_alignment); if (cb.blob() == nullptr) { // Failed to allocate in code cache. free_blob(); diff --git a/src/hotspot/share/opto/constantTable.cpp b/src/hotspot/share/opto/constantTable.cpp index 345af51839d..bcfc048746f 100644 --- a/src/hotspot/share/opto/constantTable.cpp +++ b/src/hotspot/share/opto/constantTable.cpp @@ -41,26 +41,13 @@ bool ConstantTable::Constant::operator==(const Constant& other) { return false; } for (int i = 0; i < get_array()->length(); i++) { - jvalue ele1 = get_array()->at(i); - jvalue ele2 = other.get_array()->at(i); - bool is_eq; - switch (type()) { - case T_BOOLEAN: is_eq = ele1.z == ele2.z; break; - case T_BYTE: is_eq = ele1.b == ele2.b; break; - case T_CHAR: is_eq = ele1.c == ele2.c; break; - case T_SHORT: is_eq = ele1.s == ele2.s; break; - case T_INT: is_eq = ele1.i == ele2.i; break; - case T_LONG: is_eq = ele1.j == ele2.j; break; - case T_FLOAT: is_eq = jint_cast(ele1.f) == jint_cast(ele2.f); break; - case T_DOUBLE: is_eq = jlong_cast(ele1.d) == jlong_cast(ele2.d); break; - default: ShouldNotReachHere(); is_eq = false; - } - if (!is_eq) { + if (get_array()->at(i) != other.get_array()->at(i)) { return false; } } return true; } + // For floating point values we compare the bit pattern. switch (type()) { case T_INT: return (_v._value.i == other._v._value.i); @@ -75,16 +62,45 @@ bool ConstantTable::Constant::operator==(const Constant& other) { } } +int ConstantTable::alignment() const { + int res = 1; + for (int i = 0; i < _constants.length(); i++) { + const Constant& c = _constants.at(i); + res = MAX2(res, c.alignment()); + } + return res; +} + int ConstantTable::qsort_comparator(Constant* a, Constant* b) { - // sort descending - if (a->freq() > b->freq()) return -1; - if (a->freq() < b->freq()) return 1; - return 0; + // put the ones with large alignments first + if (a->alignment() > 8 && b->alignment() > 8) { + // sort them by alignment + if (a->alignment() > b->alignment()) { + return -1; + } else if (a->alignment() < b->alignment()) { + return 1; + } else { + return 0; + } + } else if (a->alignment() > 8) { + return -1; + } else if (b->alignment() > 8) { + return 1; + } else { + // for constants with small alignments, sort them by frequency + if (a->freq() > b->freq()) { + return -1; + } else if (a->freq() < b->freq()) { + return 1; + } else { + return 0; + } + } } static int constant_size(ConstantTable::Constant* con) { if (con->is_array()) { - return type2aelembytes(con->type()) * con->get_array()->length(); + return con->get_array()->length(); } switch (con->type()) { case T_INT: return sizeof(jint ); @@ -149,7 +165,7 @@ bool ConstantTable::emit(C2_MacroAssembler* masm) const { Constant con = _constants.at(i); address constant_addr = nullptr; if (con.is_array()) { - constant_addr = masm->array_constant(con.type(), con.get_array(), con.alignment()); + constant_addr = masm->array_constant(con.get_array(), con.alignment()); } else { switch (con.type()) { case T_INT: constant_addr = masm->int_constant( con.get_jint() ); break; @@ -251,16 +267,14 @@ ConstantTable::Constant ConstantTable::add(Metadata* metadata) { return con; } -ConstantTable::Constant ConstantTable::add(MachConstantNode* n, BasicType bt, - GrowableArray* array, int alignment) { - Constant con(bt, array, alignment); +ConstantTable::Constant ConstantTable::add(MachConstantNode* n, GrowableArray* array, int alignment) { + Constant con(array, alignment); add(con); return con; } -ConstantTable::Constant ConstantTable::add(MachConstantNode* n, BasicType bt, - GrowableArray* array) { - return add(n, bt, array, array->length() * type2aelembytes(bt)); +ConstantTable::Constant ConstantTable::add(MachConstantNode* n, GrowableArray* array) { + return add(n, array, array->length()); } ConstantTable::Constant ConstantTable::add(MachConstantNode* n, MachOper* oper) { diff --git a/src/hotspot/share/opto/constantTable.hpp b/src/hotspot/share/opto/constantTable.hpp index 74d310ec8d5..d1b36942c18 100644 --- a/src/hotspot/share/opto/constantTable.hpp +++ b/src/hotspot/share/opto/constantTable.hpp @@ -43,7 +43,7 @@ public: union { jvalue _value; Metadata* _metadata; - GrowableArray* _array; + GrowableArray* _array; } _v; int _offset; // offset of this constant (in bytes) relative to the constant table base. float _freq; @@ -72,18 +72,17 @@ public: { _v._metadata = metadata; } - Constant(BasicType type, GrowableArray* array, int alignment, bool can_be_reused = true) : - _type(type), + Constant(GrowableArray* array, int alignment) : + _type(T_BYTE), _is_array(true), _alignment(alignment), _offset(-1), _freq(0.0f), - _can_be_reused(can_be_reused) + _can_be_reused(true) { - assert(is_java_primitive(type), "not applicable for %s", type2name(type)); assert(is_power_of_2(alignment), "invalid alignment %d", alignment); - _v._array = new GrowableArray(array->length()); - for (jvalue ele : *array) { + _v._array = new GrowableArray(array->length()); + for (jbyte ele : *array) { _v._array->append(ele); } } @@ -102,7 +101,7 @@ public: Metadata* get_metadata() const { return _v._metadata; } - GrowableArray* get_array() const { return _v._array; } + const GrowableArray* get_array() const { return _v._array; } int offset() const { return _offset; } void set_offset(int offset) { _offset = offset; } @@ -135,6 +134,10 @@ public: int size() const { assert(_size != -1, "not calculated yet"); return _size; } + // The minimum alignment requirement of the constant table, must be a power of 2. The constant + // section of the nmethod must satisfy this value. + int alignment() const; + int calculate_table_base_offset() const; // AD specific void set_table_base_offset(int x) { assert(_table_base_offset == -1 || x == _table_base_offset, "can't change"); _table_base_offset = x; } int table_base_offset() const { assert(_table_base_offset != -1, "not set yet"); return _table_base_offset; } @@ -150,8 +153,8 @@ public: void add(Constant& con); Constant add(MachConstantNode* n, BasicType type, jvalue value); Constant add(Metadata* metadata); - Constant add(MachConstantNode* n, BasicType bt, GrowableArray* array); - Constant add(MachConstantNode* n, BasicType bt, GrowableArray* array, int alignment); + Constant add(MachConstantNode* n, GrowableArray* array); + Constant add(MachConstantNode* n, GrowableArray* array, int alignment); Constant add(MachConstantNode* n, MachOper* oper); Constant add(MachConstantNode* n, jint i) { jvalue value; value.i = i; diff --git a/src/hotspot/share/opto/output.cpp b/src/hotspot/share/opto/output.cpp index ee91dc4323a..260aa7e635d 100644 --- a/src/hotspot/share/opto/output.cpp +++ b/src/hotspot/share/opto/output.cpp @@ -1352,7 +1352,7 @@ void PhaseOutput::estimate_buffer_size(int& const_req) { // Calculate the offsets of the constants and the size of the // constant table (including the padding to the next section). constant_table().calculate_offsets_and_size(); - const_req = constant_table().size() + add_size; + const_req = constant_table().alignment() + constant_table().size() + add_size; } // Initialize the space for the BufferBlob used to find and verify @@ -1392,6 +1392,7 @@ CodeBuffer* PhaseOutput::init_buffer() { total_req += deopt_handler_req; // deopt MH handler CodeBuffer* cb = code_buffer(); + cb->set_const_section_alignment(constant_table().alignment()); cb->initialize(total_req, _buf_sizes._reloc); // Have we run out of code space?