*nodes, PhaseRegAlloc *ra_) {
- ShouldNotReachHere();
-}
-
-void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
- // Empty encoding
-}
-
-uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
- return 0;
-}
-
-#ifndef PRODUCT
-void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
- st->print("# MachConstantBaseNode (empty encoding)");
-}
-#endif
-
-
-//=============================================================================
-#ifndef PRODUCT
-void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
- Compile* C = ra_->C;
-
- int framesize = C->output()->frame_size_in_bytes();
- int bangsize = C->output()->bang_size_in_bytes();
- assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
- // Remove wordSize for return addr which is already pushed.
- framesize -= wordSize;
-
- if (C->output()->need_stack_bang(bangsize)) {
- framesize -= wordSize;
- st->print("# stack bang (%d bytes)", bangsize);
- st->print("\n\t");
- st->print("PUSH EBP\t# Save EBP");
- if (PreserveFramePointer) {
- st->print("\n\t");
- st->print("MOV EBP, ESP\t# Save the caller's SP into EBP");
- }
- if (framesize) {
- st->print("\n\t");
- st->print("SUB ESP, #%d\t# Create frame",framesize);
- }
- } else {
- st->print("SUB ESP, #%d\t# Create frame",framesize);
- st->print("\n\t");
- framesize -= wordSize;
- st->print("MOV [ESP + #%d], EBP\t# Save EBP",framesize);
- if (PreserveFramePointer) {
- st->print("\n\t");
- st->print("MOV EBP, ESP\t# Save the caller's SP into EBP");
- if (framesize > 0) {
- st->print("\n\t");
- st->print("ADD EBP, #%d", framesize);
- }
- }
- }
-
- if (VerifyStackAtCalls) {
- st->print("\n\t");
- framesize -= wordSize;
- st->print("MOV [ESP + #%d], 0xBADB100D\t# Majik cookie for stack depth check",framesize);
- }
-
- if( C->in_24_bit_fp_mode() ) {
- st->print("\n\t");
- st->print("FLDCW \t# load 24 bit fpu control word");
- }
- if (UseSSE >= 2 && VerifyFPU) {
- st->print("\n\t");
- st->print("# verify FPU stack (must be clean on entry)");
- }
-
-#ifdef ASSERT
- if (VerifyStackAtCalls) {
- st->print("\n\t");
- st->print("# stack alignment check");
- }
-#endif
- st->cr();
-}
-#endif
-
-
-void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
- Compile* C = ra_->C;
-
- int framesize = C->output()->frame_size_in_bytes();
- int bangsize = C->output()->bang_size_in_bytes();
-
- __ verified_entry(framesize, C->output()->need_stack_bang(bangsize)?bangsize:0, C->in_24_bit_fp_mode(), C->stub_function() != nullptr);
-
- C->output()->set_frame_complete(__ offset());
-
- if (C->has_mach_constant_base_node()) {
- // NOTE: We set the table base offset here because users might be
- // emitted before MachConstantBaseNode.
- ConstantTable& constant_table = C->output()->constant_table();
- constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
- }
-}
-
-uint MachPrologNode::size(PhaseRegAlloc *ra_) const {
- return MachNode::size(ra_); // too many variables; just compute it the hard way
-}
-
-int MachPrologNode::reloc() const {
- return 0; // a large enough number
-}
-
-//=============================================================================
-#ifndef PRODUCT
-void MachEpilogNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
- Compile *C = ra_->C;
- int framesize = C->output()->frame_size_in_bytes();
- assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
- // Remove two words for return addr and rbp,
- framesize -= 2*wordSize;
-
- if (C->max_vector_size() > 16) {
- st->print("VZEROUPPER");
- st->cr(); st->print("\t");
- }
- if (C->in_24_bit_fp_mode()) {
- st->print("FLDCW standard control word");
- st->cr(); st->print("\t");
- }
- if (framesize) {
- st->print("ADD ESP,%d\t# Destroy frame",framesize);
- st->cr(); st->print("\t");
- }
- st->print_cr("POPL EBP"); st->print("\t");
- if (do_polling() && C->is_method_compilation()) {
- st->print("CMPL rsp, poll_offset[thread] \n\t"
- "JA #safepoint_stub\t"
- "# Safepoint: poll for GC");
- }
-}
-#endif
-
-void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
- Compile *C = ra_->C;
-
- if (C->max_vector_size() > 16) {
- // Clear upper bits of YMM registers when current compiled code uses
- // wide vectors to avoid AVX <-> SSE transition penalty during call.
- __ vzeroupper();
- }
- // If method set FPU control word, restore to standard control word
- if (C->in_24_bit_fp_mode()) {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
- }
-
- int framesize = C->output()->frame_size_in_bytes();
- assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
- // Remove two words for return addr and rbp,
- framesize -= 2*wordSize;
-
- // Note that VerifyStackAtCalls' Majik cookie does not change the frame size popped here
-
- if (framesize >= 128) {
- emit_opcode(masm, 0x81); // add SP, #framesize
- emit_rm(masm, 0x3, 0x00, ESP_enc);
- emit_d32(masm, framesize);
- } else if (framesize) {
- emit_opcode(masm, 0x83); // add SP, #framesize
- emit_rm(masm, 0x3, 0x00, ESP_enc);
- emit_d8(masm, framesize);
- }
-
- emit_opcode(masm, 0x58 | EBP_enc);
-
- if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
- __ reserved_stack_check();
- }
-
- if (do_polling() && C->is_method_compilation()) {
- Register thread = as_Register(EBX_enc);
- __ get_thread(thread);
- Label dummy_label;
- Label* code_stub = &dummy_label;
- if (!C->output()->in_scratch_emit_size()) {
- C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
- C->output()->add_stub(stub);
- code_stub = &stub->entry();
- }
- __ set_inst_mark();
- __ relocate(relocInfo::poll_return_type);
- __ clear_inst_mark();
- __ safepoint_poll(*code_stub, thread, true /* at_return */, true /* in_nmethod */);
- }
-}
-
-uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
- return MachNode::size(ra_); // too many variables; just compute it
- // the hard way
-}
-
-int MachEpilogNode::reloc() const {
- return 0; // a large enough number
-}
-
-const Pipeline * MachEpilogNode::pipeline() const {
- return MachNode::pipeline_class();
-}
-
-//=============================================================================
-
-enum RC { rc_bad, rc_int, rc_kreg, rc_float, rc_xmm, rc_stack };
-static enum RC rc_class( OptoReg::Name reg ) {
-
- if( !OptoReg::is_valid(reg) ) return rc_bad;
- if (OptoReg::is_stack(reg)) return rc_stack;
-
- VMReg r = OptoReg::as_VMReg(reg);
- if (r->is_Register()) return rc_int;
- if (r->is_FloatRegister()) {
- assert(UseSSE < 2, "shouldn't be used in SSE2+ mode");
- return rc_float;
- }
- if (r->is_KRegister()) return rc_kreg;
- assert(r->is_XMMRegister(), "must be");
- return rc_xmm;
-}
-
-static int impl_helper( C2_MacroAssembler *masm, bool do_size, bool is_load, int offset, int reg,
- int opcode, const char *op_str, int size, outputStream* st ) {
- if( masm ) {
- masm->set_inst_mark();
- emit_opcode (masm, opcode );
- encode_RegMem(masm, Matcher::_regEncode[reg], ESP_enc, 0x4, 0, offset, relocInfo::none);
- masm->clear_inst_mark();
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- if( opcode == 0x8B || opcode == 0x89 ) { // MOV
- if( is_load ) st->print("%s %s,[ESP + #%d]",op_str,Matcher::regName[reg],offset);
- else st->print("%s [ESP + #%d],%s",op_str,offset,Matcher::regName[reg]);
- } else { // FLD, FST, PUSH, POP
- st->print("%s [ESP + #%d]",op_str,offset);
- }
-#endif
- }
- int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
- return size+3+offset_size;
-}
-
-// Helper for XMM registers. Extra opcode bits, limited syntax.
-static int impl_x_helper( C2_MacroAssembler *masm, bool do_size, bool is_load,
- int offset, int reg_lo, int reg_hi, int size, outputStream* st ) {
- int in_size_in_bits = Assembler::EVEX_32bit;
- int evex_encoding = 0;
- if (reg_lo+1 == reg_hi) {
- in_size_in_bits = Assembler::EVEX_64bit;
- evex_encoding = Assembler::VEX_W;
- }
- if (masm) {
- // EVEX spills remain EVEX: Compressed displacemement is better than AVX on spill mem operations,
- // it maps more cases to single byte displacement
- __ set_managed();
- if (reg_lo+1 == reg_hi) { // double move?
- if (is_load) {
- __ movdbl(as_XMMRegister(Matcher::_regEncode[reg_lo]), Address(rsp, offset));
- } else {
- __ movdbl(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[reg_lo]));
- }
- } else {
- if (is_load) {
- __ movflt(as_XMMRegister(Matcher::_regEncode[reg_lo]), Address(rsp, offset));
- } else {
- __ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[reg_lo]));
- }
- }
-#ifndef PRODUCT
- } else if (!do_size) {
- if (size != 0) st->print("\n\t");
- if (reg_lo+1 == reg_hi) { // double move?
- if (is_load) st->print("%s %s,[ESP + #%d]",
- UseXmmLoadAndClearUpper ? "MOVSD " : "MOVLPD",
- Matcher::regName[reg_lo], offset);
- else st->print("MOVSD [ESP + #%d],%s",
- offset, Matcher::regName[reg_lo]);
- } else {
- if (is_load) st->print("MOVSS %s,[ESP + #%d]",
- Matcher::regName[reg_lo], offset);
- else st->print("MOVSS [ESP + #%d],%s",
- offset, Matcher::regName[reg_lo]);
- }
-#endif
- }
- bool is_single_byte = false;
- if ((UseAVX > 2) && (offset != 0)) {
- is_single_byte = Assembler::query_compressed_disp_byte(offset, true, 0, Assembler::EVEX_T1S, in_size_in_bits, evex_encoding);
- }
- int offset_size = 0;
- if (UseAVX > 2 ) {
- offset_size = (offset == 0) ? 0 : ((is_single_byte) ? 1 : 4);
- } else {
- offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
- }
- size += (UseAVX > 2) ? 2 : 0; // Need an additional two bytes for EVEX
- // VEX_2bytes prefix is used if UseAVX > 0, so it takes the same 2 bytes as SIMD prefix.
- return size+5+offset_size;
-}
-
-
-static int impl_movx_helper( C2_MacroAssembler *masm, bool do_size, int src_lo, int dst_lo,
- int src_hi, int dst_hi, int size, outputStream* st ) {
- if (masm) {
- // EVEX spills remain EVEX: logic complex between full EVEX, partial and AVX, manage EVEX spill code one way.
- __ set_managed();
- if (src_lo+1 == src_hi && dst_lo+1 == dst_hi) { // double move?
- __ movdbl(as_XMMRegister(Matcher::_regEncode[dst_lo]),
- as_XMMRegister(Matcher::_regEncode[src_lo]));
- } else {
- __ movflt(as_XMMRegister(Matcher::_regEncode[dst_lo]),
- as_XMMRegister(Matcher::_regEncode[src_lo]));
- }
-#ifndef PRODUCT
- } else if (!do_size) {
- if (size != 0) st->print("\n\t");
- if (UseXmmRegToRegMoveAll) {//Use movaps,movapd to move between xmm registers
- if (src_lo+1 == src_hi && dst_lo+1 == dst_hi) { // double move?
- st->print("MOVAPD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
- } else {
- st->print("MOVAPS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
- }
- } else {
- if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double move?
- st->print("MOVSD %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
- } else {
- st->print("MOVSS %s,%s",Matcher::regName[dst_lo],Matcher::regName[src_lo]);
- }
- }
-#endif
- }
- // VEX_2bytes prefix is used if UseAVX > 0, and it takes the same 2 bytes as SIMD prefix.
- // Only MOVAPS SSE prefix uses 1 byte. EVEX uses an additional 2 bytes.
- int sz = (UseAVX > 2) ? 6 : 4;
- if (!(src_lo+1 == src_hi && dst_lo+1 == dst_hi) &&
- UseXmmRegToRegMoveAll && (UseAVX == 0)) sz = 3;
- return size + sz;
-}
-
-static int impl_movgpr2x_helper( C2_MacroAssembler *masm, bool do_size, int src_lo, int dst_lo,
- int src_hi, int dst_hi, int size, outputStream* st ) {
- // 32-bit
- if (masm) {
- // EVEX spills remain EVEX: logic complex between full EVEX, partial and AVX, manage EVEX spill code one way.
- __ set_managed();
- __ movdl(as_XMMRegister(Matcher::_regEncode[dst_lo]),
- as_Register(Matcher::_regEncode[src_lo]));
-#ifndef PRODUCT
- } else if (!do_size) {
- st->print("movdl %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
-#endif
- }
- return (UseAVX> 2) ? 6 : 4;
-}
-
-
-static int impl_movx2gpr_helper( C2_MacroAssembler *masm, bool do_size, int src_lo, int dst_lo,
- int src_hi, int dst_hi, int size, outputStream* st ) {
- // 32-bit
- if (masm) {
- // EVEX spills remain EVEX: logic complex between full EVEX, partial and AVX, manage EVEX spill code one way.
- __ set_managed();
- __ movdl(as_Register(Matcher::_regEncode[dst_lo]),
- as_XMMRegister(Matcher::_regEncode[src_lo]));
-#ifndef PRODUCT
- } else if (!do_size) {
- st->print("movdl %s, %s\t# spill", Matcher::regName[dst_lo], Matcher::regName[src_lo]);
-#endif
- }
- return (UseAVX> 2) ? 6 : 4;
-}
-
-static int impl_mov_helper( C2_MacroAssembler *masm, bool do_size, int src, int dst, int size, outputStream* st ) {
- if( masm ) {
- emit_opcode(masm, 0x8B );
- emit_rm (masm, 0x3, Matcher::_regEncode[dst], Matcher::_regEncode[src] );
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- st->print("MOV %s,%s",Matcher::regName[dst],Matcher::regName[src]);
-#endif
- }
- return size+2;
-}
-
-static int impl_fp_store_helper( C2_MacroAssembler *masm, bool do_size, int src_lo, int src_hi, int dst_lo, int dst_hi,
- int offset, int size, outputStream* st ) {
- if( src_lo != FPR1L_num ) { // Move value to top of FP stack, if not already there
- if( masm ) {
- emit_opcode( masm, 0xD9 ); // FLD (i.e., push it)
- emit_d8( masm, 0xC0-1+Matcher::_regEncode[src_lo] );
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- st->print("FLD %s",Matcher::regName[src_lo]);
-#endif
- }
- size += 2;
- }
-
- int st_op = (src_lo != FPR1L_num) ? EBX_num /*store & pop*/ : EDX_num /*store no pop*/;
- const char *op_str;
- int op;
- if( src_lo+1 == src_hi && dst_lo+1 == dst_hi ) { // double store?
- op_str = (src_lo != FPR1L_num) ? "FSTP_D" : "FST_D ";
- op = 0xDD;
- } else { // 32-bit store
- op_str = (src_lo != FPR1L_num) ? "FSTP_S" : "FST_S ";
- op = 0xD9;
- assert( !OptoReg::is_valid(src_hi) && !OptoReg::is_valid(dst_hi), "no non-adjacent float-stores" );
- }
-
- return impl_helper(masm,do_size,false,offset,st_op,op,op_str,size, st);
-}
-
-// Next two methods are shared by 32- and 64-bit VM. They are defined in x86.ad.
-static void vec_mov_helper(C2_MacroAssembler *masm, int src_lo, int dst_lo,
- int src_hi, int dst_hi, uint ireg, outputStream* st);
-
-void vec_spill_helper(C2_MacroAssembler *masm, bool is_load,
- int stack_offset, int reg, uint ireg, outputStream* st);
-
-static void vec_stack_to_stack_helper(C2_MacroAssembler *masm, int src_offset,
- int dst_offset, uint ireg, outputStream* st) {
- if (masm) {
- switch (ireg) {
- case Op_VecS:
- __ pushl(Address(rsp, src_offset));
- __ popl (Address(rsp, dst_offset));
- break;
- case Op_VecD:
- __ pushl(Address(rsp, src_offset));
- __ popl (Address(rsp, dst_offset));
- __ pushl(Address(rsp, src_offset+4));
- __ popl (Address(rsp, dst_offset+4));
- break;
- case Op_VecX:
- __ movdqu(Address(rsp, -16), xmm0);
- __ movdqu(xmm0, Address(rsp, src_offset));
- __ movdqu(Address(rsp, dst_offset), xmm0);
- __ movdqu(xmm0, Address(rsp, -16));
- break;
- case Op_VecY:
- __ vmovdqu(Address(rsp, -32), xmm0);
- __ vmovdqu(xmm0, Address(rsp, src_offset));
- __ vmovdqu(Address(rsp, dst_offset), xmm0);
- __ vmovdqu(xmm0, Address(rsp, -32));
- break;
- case Op_VecZ:
- __ evmovdquq(Address(rsp, -64), xmm0, 2);
- __ evmovdquq(xmm0, Address(rsp, src_offset), 2);
- __ evmovdquq(Address(rsp, dst_offset), xmm0, 2);
- __ evmovdquq(xmm0, Address(rsp, -64), 2);
- break;
- default:
- ShouldNotReachHere();
- }
-#ifndef PRODUCT
- } else {
- switch (ireg) {
- case Op_VecS:
- st->print("pushl [rsp + #%d]\t# 32-bit mem-mem spill\n\t"
- "popl [rsp + #%d]",
- src_offset, dst_offset);
- break;
- case Op_VecD:
- st->print("pushl [rsp + #%d]\t# 64-bit mem-mem spill\n\t"
- "popq [rsp + #%d]\n\t"
- "pushl [rsp + #%d]\n\t"
- "popq [rsp + #%d]",
- src_offset, dst_offset, src_offset+4, dst_offset+4);
- break;
- case Op_VecX:
- st->print("movdqu [rsp - #16], xmm0\t# 128-bit mem-mem spill\n\t"
- "movdqu xmm0, [rsp + #%d]\n\t"
- "movdqu [rsp + #%d], xmm0\n\t"
- "movdqu xmm0, [rsp - #16]",
- src_offset, dst_offset);
- break;
- case Op_VecY:
- st->print("vmovdqu [rsp - #32], xmm0\t# 256-bit mem-mem spill\n\t"
- "vmovdqu xmm0, [rsp + #%d]\n\t"
- "vmovdqu [rsp + #%d], xmm0\n\t"
- "vmovdqu xmm0, [rsp - #32]",
- src_offset, dst_offset);
- break;
- case Op_VecZ:
- st->print("vmovdqu [rsp - #64], xmm0\t# 512-bit mem-mem spill\n\t"
- "vmovdqu xmm0, [rsp + #%d]\n\t"
- "vmovdqu [rsp + #%d], xmm0\n\t"
- "vmovdqu xmm0, [rsp - #64]",
- src_offset, dst_offset);
- break;
- default:
- ShouldNotReachHere();
- }
-#endif
- }
-}
-
-uint MachSpillCopyNode::implementation( C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream* st ) const {
- // Get registers to move
- OptoReg::Name src_second = ra_->get_reg_second(in(1));
- OptoReg::Name src_first = ra_->get_reg_first(in(1));
- OptoReg::Name dst_second = ra_->get_reg_second(this );
- OptoReg::Name dst_first = ra_->get_reg_first(this );
-
- enum RC src_second_rc = rc_class(src_second);
- enum RC src_first_rc = rc_class(src_first);
- enum RC dst_second_rc = rc_class(dst_second);
- enum RC dst_first_rc = rc_class(dst_first);
-
- assert( OptoReg::is_valid(src_first) && OptoReg::is_valid(dst_first), "must move at least 1 register" );
-
- // Generate spill code!
- int size = 0;
-
- if( src_first == dst_first && src_second == dst_second )
- return size; // Self copy, no move
-
- if (bottom_type()->isa_vect() != nullptr && bottom_type()->isa_vectmask() == nullptr) {
- uint ireg = ideal_reg();
- assert((src_first_rc != rc_int && dst_first_rc != rc_int), "sanity");
- assert((src_first_rc != rc_float && dst_first_rc != rc_float), "sanity");
- assert((ireg == Op_VecS || ireg == Op_VecD || ireg == Op_VecX || ireg == Op_VecY || ireg == Op_VecZ ), "sanity");
- if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
- // mem -> mem
- int src_offset = ra_->reg2offset(src_first);
- int dst_offset = ra_->reg2offset(dst_first);
- vec_stack_to_stack_helper(masm, src_offset, dst_offset, ireg, st);
- } else if (src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
- vec_mov_helper(masm, src_first, dst_first, src_second, dst_second, ireg, st);
- } else if (src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
- int stack_offset = ra_->reg2offset(dst_first);
- vec_spill_helper(masm, false, stack_offset, src_first, ireg, st);
- } else if (src_first_rc == rc_stack && dst_first_rc == rc_xmm ) {
- int stack_offset = ra_->reg2offset(src_first);
- vec_spill_helper(masm, true, stack_offset, dst_first, ireg, st);
- } else {
- ShouldNotReachHere();
- }
- return 0;
- }
-
- // --------------------------------------
- // Check for mem-mem move. push/pop to move.
- if( src_first_rc == rc_stack && dst_first_rc == rc_stack ) {
- if( src_second == dst_first ) { // overlapping stack copy ranges
- assert( src_second_rc == rc_stack && dst_second_rc == rc_stack, "we only expect a stk-stk copy here" );
- size = impl_helper(masm,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH ",size, st);
- size = impl_helper(masm,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP ",size, st);
- src_second_rc = dst_second_rc = rc_bad; // flag as already moved the second bits
- }
- // move low bits
- size = impl_helper(masm,do_size,true ,ra_->reg2offset(src_first),ESI_num,0xFF,"PUSH ",size, st);
- size = impl_helper(masm,do_size,false,ra_->reg2offset(dst_first),EAX_num,0x8F,"POP ",size, st);
- if( src_second_rc == rc_stack && dst_second_rc == rc_stack ) { // mov second bits
- size = impl_helper(masm,do_size,true ,ra_->reg2offset(src_second),ESI_num,0xFF,"PUSH ",size, st);
- size = impl_helper(masm,do_size,false,ra_->reg2offset(dst_second),EAX_num,0x8F,"POP ",size, st);
- }
- return size;
- }
-
- // --------------------------------------
- // Check for integer reg-reg copy
- if( src_first_rc == rc_int && dst_first_rc == rc_int )
- size = impl_mov_helper(masm,do_size,src_first,dst_first,size, st);
-
- // Check for integer store
- if( src_first_rc == rc_int && dst_first_rc == rc_stack )
- size = impl_helper(masm,do_size,false,ra_->reg2offset(dst_first),src_first,0x89,"MOV ",size, st);
-
- // Check for integer load
- if( src_first_rc == rc_stack && dst_first_rc == rc_int )
- size = impl_helper(masm,do_size,true ,ra_->reg2offset(src_first),dst_first,0x8B,"MOV ",size, st);
-
- // Check for integer reg-xmm reg copy
- if( src_first_rc == rc_int && dst_first_rc == rc_xmm ) {
- assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
- "no 64 bit integer-float reg moves" );
- return impl_movgpr2x_helper(masm,do_size,src_first,dst_first,src_second, dst_second, size, st);
- }
- // --------------------------------------
- // Check for float reg-reg copy
- if( src_first_rc == rc_float && dst_first_rc == rc_float ) {
- assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
- (src_first+1 == src_second && dst_first+1 == dst_second), "no non-adjacent float-moves" );
- if( masm ) {
-
- // Note the mucking with the register encode to compensate for the 0/1
- // indexing issue mentioned in a comment in the reg_def sections
- // for FPR registers many lines above here.
-
- if( src_first != FPR1L_num ) {
- emit_opcode (masm, 0xD9 ); // FLD ST(i)
- emit_d8 (masm, 0xC0+Matcher::_regEncode[src_first]-1 );
- emit_opcode (masm, 0xDD ); // FSTP ST(i)
- emit_d8 (masm, 0xD8+Matcher::_regEncode[dst_first] );
- } else {
- emit_opcode (masm, 0xDD ); // FST ST(i)
- emit_d8 (masm, 0xD0+Matcher::_regEncode[dst_first]-1 );
- }
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- if( src_first != FPR1L_num ) st->print("FLD %s\n\tFSTP %s",Matcher::regName[src_first],Matcher::regName[dst_first]);
- else st->print( "FST %s", Matcher::regName[dst_first]);
-#endif
- }
- return size + ((src_first != FPR1L_num) ? 2+2 : 2);
- }
-
- // Check for float store
- if( src_first_rc == rc_float && dst_first_rc == rc_stack ) {
- return impl_fp_store_helper(masm,do_size,src_first,src_second,dst_first,dst_second,ra_->reg2offset(dst_first),size, st);
- }
-
- // Check for float load
- if( dst_first_rc == rc_float && src_first_rc == rc_stack ) {
- int offset = ra_->reg2offset(src_first);
- const char *op_str;
- int op;
- if( src_first+1 == src_second && dst_first+1 == dst_second ) { // double load?
- op_str = "FLD_D";
- op = 0xDD;
- } else { // 32-bit load
- op_str = "FLD_S";
- op = 0xD9;
- assert( src_second_rc == rc_bad && dst_second_rc == rc_bad, "no non-adjacent float-loads" );
- }
- if( masm ) {
- masm->set_inst_mark();
- emit_opcode (masm, op );
- encode_RegMem(masm, 0x0, ESP_enc, 0x4, 0, offset, relocInfo::none);
- emit_opcode (masm, 0xDD ); // FSTP ST(i)
- emit_d8 (masm, 0xD8+Matcher::_regEncode[dst_first] );
- masm->clear_inst_mark();
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- st->print("%s ST,[ESP + #%d]\n\tFSTP %s",op_str, offset,Matcher::regName[dst_first]);
-#endif
- }
- int offset_size = (offset == 0) ? 0 : ((offset <= 127) ? 1 : 4);
- return size + 3+offset_size+2;
- }
-
- // Check for xmm reg-reg copy
- if( src_first_rc == rc_xmm && dst_first_rc == rc_xmm ) {
- assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad) ||
- (src_first+1 == src_second && dst_first+1 == dst_second),
- "no non-adjacent float-moves" );
- return impl_movx_helper(masm,do_size,src_first,dst_first,src_second, dst_second, size, st);
- }
-
- // Check for xmm reg-integer reg copy
- if( src_first_rc == rc_xmm && dst_first_rc == rc_int ) {
- assert( (src_second_rc == rc_bad && dst_second_rc == rc_bad),
- "no 64 bit float-integer reg moves" );
- return impl_movx2gpr_helper(masm,do_size,src_first,dst_first,src_second, dst_second, size, st);
- }
-
- // Check for xmm store
- if( src_first_rc == rc_xmm && dst_first_rc == rc_stack ) {
- return impl_x_helper(masm,do_size,false,ra_->reg2offset(dst_first), src_first, src_second, size, st);
- }
-
- // Check for float xmm load
- if( src_first_rc == rc_stack && dst_first_rc == rc_xmm ) {
- return impl_x_helper(masm,do_size,true ,ra_->reg2offset(src_first),dst_first, dst_second, size, st);
- }
-
- // Copy from float reg to xmm reg
- if( src_first_rc == rc_float && dst_first_rc == rc_xmm ) {
- // copy to the top of stack from floating point reg
- // and use LEA to preserve flags
- if( masm ) {
- emit_opcode(masm,0x8D); // LEA ESP,[ESP-8]
- emit_rm(masm, 0x1, ESP_enc, 0x04);
- emit_rm(masm, 0x0, 0x04, ESP_enc);
- emit_d8(masm,0xF8);
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- st->print("LEA ESP,[ESP-8]");
-#endif
- }
- size += 4;
-
- size = impl_fp_store_helper(masm,do_size,src_first,src_second,dst_first,dst_second,0,size, st);
-
- // Copy from the temp memory to the xmm reg.
- size = impl_x_helper(masm,do_size,true ,0,dst_first, dst_second, size, st);
-
- if( masm ) {
- emit_opcode(masm,0x8D); // LEA ESP,[ESP+8]
- emit_rm(masm, 0x1, ESP_enc, 0x04);
- emit_rm(masm, 0x0, 0x04, ESP_enc);
- emit_d8(masm,0x08);
-#ifndef PRODUCT
- } else if( !do_size ) {
- if( size != 0 ) st->print("\n\t");
- st->print("LEA ESP,[ESP+8]");
-#endif
- }
- size += 4;
- return size;
- }
-
- // AVX-512 opmask specific spilling.
- if (src_first_rc == rc_stack && dst_first_rc == rc_kreg) {
- assert((src_first & 1) == 0 && src_first + 1 == src_second, "invalid register pair");
- assert((dst_first & 1) == 0 && dst_first + 1 == dst_second, "invalid register pair");
- int offset = ra_->reg2offset(src_first);
- if (masm != nullptr) {
- __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
-#ifndef PRODUCT
- } else {
- st->print("KMOV %s, [ESP + %d]", Matcher::regName[dst_first], offset);
-#endif
- }
- return 0;
- }
-
- if (src_first_rc == rc_kreg && dst_first_rc == rc_stack) {
- assert((src_first & 1) == 0 && src_first + 1 == src_second, "invalid register pair");
- assert((dst_first & 1) == 0 && dst_first + 1 == dst_second, "invalid register pair");
- int offset = ra_->reg2offset(dst_first);
- if (masm != nullptr) {
- __ kmov(Address(rsp, offset), as_KRegister(Matcher::_regEncode[src_first]));
-#ifndef PRODUCT
- } else {
- st->print("KMOV [ESP + %d], %s", offset, Matcher::regName[src_first]);
-#endif
- }
- return 0;
- }
-
- if (src_first_rc == rc_kreg && dst_first_rc == rc_int) {
- Unimplemented();
- return 0;
- }
-
- if (src_first_rc == rc_int && dst_first_rc == rc_kreg) {
- Unimplemented();
- return 0;
- }
-
- if (src_first_rc == rc_kreg && dst_first_rc == rc_kreg) {
- assert((src_first & 1) == 0 && src_first + 1 == src_second, "invalid register pair");
- assert((dst_first & 1) == 0 && dst_first + 1 == dst_second, "invalid register pair");
- if (masm != nullptr) {
- __ kmov(as_KRegister(Matcher::_regEncode[dst_first]), as_KRegister(Matcher::_regEncode[src_first]));
-#ifndef PRODUCT
- } else {
- st->print("KMOV %s, %s", Matcher::regName[dst_first], Matcher::regName[src_first]);
-#endif
- }
- return 0;
- }
-
- assert( size > 0, "missed a case" );
-
- // --------------------------------------------------------------------
- // Check for second bits still needing moving.
- if( src_second == dst_second )
- return size; // Self copy; no move
- assert( src_second_rc != rc_bad && dst_second_rc != rc_bad, "src_second & dst_second cannot be Bad" );
-
- // Check for second word int-int move
- if( src_second_rc == rc_int && dst_second_rc == rc_int )
- return impl_mov_helper(masm,do_size,src_second,dst_second,size, st);
-
- // Check for second word integer store
- if( src_second_rc == rc_int && dst_second_rc == rc_stack )
- return impl_helper(masm,do_size,false,ra_->reg2offset(dst_second),src_second,0x89,"MOV ",size, st);
-
- // Check for second word integer load
- if( dst_second_rc == rc_int && src_second_rc == rc_stack )
- return impl_helper(masm,do_size,true ,ra_->reg2offset(src_second),dst_second,0x8B,"MOV ",size, st);
-
- Unimplemented();
- return 0; // Mute compiler
-}
-
-#ifndef PRODUCT
-void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream* st) const {
- implementation( nullptr, ra_, false, st );
-}
-#endif
-
-void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
- implementation( masm, ra_, false, nullptr );
-}
-
-uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
- return MachNode::size(ra_);
-}
-
-
-//=============================================================================
-#ifndef PRODUCT
-void BoxLockNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
- int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
- int reg = ra_->get_reg_first(this);
- st->print("LEA %s,[ESP + #%d]",Matcher::regName[reg],offset);
-}
-#endif
-
-void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
- int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
- int reg = ra_->get_encode(this);
- if( offset >= 128 ) {
- emit_opcode(masm, 0x8D); // LEA reg,[SP+offset]
- emit_rm(masm, 0x2, reg, 0x04);
- emit_rm(masm, 0x0, 0x04, ESP_enc);
- emit_d32(masm, offset);
- }
- else {
- emit_opcode(masm, 0x8D); // LEA reg,[SP+offset]
- emit_rm(masm, 0x1, reg, 0x04);
- emit_rm(masm, 0x0, 0x04, ESP_enc);
- emit_d8(masm, offset);
- }
-}
-
-uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
- int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
- if( offset >= 128 ) {
- return 7;
- }
- else {
- return 4;
- }
-}
-
-//=============================================================================
-#ifndef PRODUCT
-void MachUEPNode::format( PhaseRegAlloc *ra_, outputStream* st ) const {
- st->print_cr( "CMP EAX,[ECX+4]\t# Inline cache check");
- st->print_cr("\tJNE SharedRuntime::handle_ic_miss_stub");
- st->print_cr("\tNOP");
- st->print_cr("\tNOP");
- if( !OptoBreakpoint )
- st->print_cr("\tNOP");
-}
-#endif
-
-void MachUEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
- __ ic_check(CodeEntryAlignment);
-}
-
-uint MachUEPNode::size(PhaseRegAlloc *ra_) const {
- return MachNode::size(ra_); // too many variables; just compute it
- // the hard way
-}
-
-
-//=============================================================================
-
-// Vector calling convention not supported.
-bool Matcher::supports_vector_calling_convention() {
- return false;
-}
-
-OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
- Unimplemented();
- return OptoRegPair(0, 0);
-}
-
-// Is this branch offset short enough that a short branch can be used?
-//
-// NOTE: If the platform does not provide any short branch variants, then
-// this method should return false for offset 0.
-bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
- // The passed offset is relative to address of the branch.
- // On 86 a branch displacement is calculated relative to address
- // of a next instruction.
- offset -= br_size;
-
- // the short version of jmpConUCF2 contains multiple branches,
- // making the reach slightly less
- if (rule == jmpConUCF2_rule)
- return (-126 <= offset && offset <= 125);
- return (-128 <= offset && offset <= 127);
-}
-
-// Return whether or not this register is ever used as an argument. This
-// function is used on startup to build the trampoline stubs in generateOptoStub.
-// Registers not mentioned will be killed by the VM call in the trampoline, and
-// arguments in those registers not be available to the callee.
-bool Matcher::can_be_java_arg( int reg ) {
- if( reg == ECX_num || reg == EDX_num ) return true;
- if( (reg == XMM0_num || reg == XMM1_num ) && UseSSE>=1 ) return true;
- if( (reg == XMM0b_num || reg == XMM1b_num) && UseSSE>=2 ) return true;
- return false;
-}
-
-bool Matcher::is_spillable_arg( int reg ) {
- return can_be_java_arg(reg);
-}
-
-uint Matcher::int_pressure_limit()
-{
- return (INTPRESSURE == -1) ? 6 : INTPRESSURE;
-}
-
-uint Matcher::float_pressure_limit()
-{
- return (FLOATPRESSURE == -1) ? 6 : FLOATPRESSURE;
-}
-
-bool Matcher::use_asm_for_ldiv_by_con( jlong divisor ) {
- // Use hardware integer DIV instruction when
- // it is faster than a code which use multiply.
- // Only when constant divisor fits into 32 bit
- // (min_jint is excluded to get only correct
- // positive 32 bit values from negative).
- return VM_Version::has_fast_idiv() &&
- (divisor == (int)divisor && divisor != min_jint);
-}
-
-// Register for DIVI projection of divmodI
-RegMask Matcher::divI_proj_mask() {
- return EAX_REG_mask();
-}
-
-// Register for MODI projection of divmodI
-RegMask Matcher::modI_proj_mask() {
- return EDX_REG_mask();
-}
-
-// Register for DIVL projection of divmodL
-RegMask Matcher::divL_proj_mask() {
- ShouldNotReachHere();
- return RegMask();
-}
-
-// Register for MODL projection of divmodL
-RegMask Matcher::modL_proj_mask() {
- ShouldNotReachHere();
- return RegMask();
-}
-
-const RegMask Matcher::method_handle_invoke_SP_save_mask() {
- return NO_REG_mask();
-}
-
-// Returns true if the high 32 bits of the value is known to be zero.
-bool is_operand_hi32_zero(Node* n) {
- int opc = n->Opcode();
- if (opc == Op_AndL) {
- Node* o2 = n->in(2);
- if (o2->is_Con() && (o2->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
- return true;
- }
- }
- if (opc == Op_ConL && (n->get_long() & 0xFFFFFFFF00000000LL) == 0LL) {
- return true;
- }
- return false;
-}
-
-%}
-
-//----------ENCODING BLOCK-----------------------------------------------------
-// This block specifies the encoding classes used by the compiler to output
-// byte streams. Encoding classes generate functions which are called by
-// Machine Instruction Nodes in order to generate the bit encoding of the
-// instruction. Operands specify their base encoding interface with the
-// interface keyword. There are currently supported four interfaces,
-// REG_INTER, CONST_INTER, MEMORY_INTER, & COND_INTER. REG_INTER causes an
-// operand to generate a function which returns its register number when
-// queried. CONST_INTER causes an operand to generate a function which
-// returns the value of the constant when queried. MEMORY_INTER causes an
-// operand to generate four functions which return the Base Register, the
-// Index Register, the Scale Value, and the Offset Value of the operand when
-// queried. COND_INTER causes an operand to generate six functions which
-// return the encoding code (ie - encoding bits for the instruction)
-// associated with each basic boolean condition for a conditional instruction.
-// Instructions specify two basic values for encoding. They use the
-// ins_encode keyword to specify their encoding class (which must be one of
-// the class names specified in the encoding block), and they use the
-// opcode keyword to specify, in order, their primary, secondary, and
-// tertiary opcode. Only the opcode sections which a particular instruction
-// needs for encoding need to be specified.
-encode %{
- // Build emit functions for each basic byte or larger field in the intel
- // encoding scheme (opcode, rm, sib, immediate), and call them from C++
- // code in the enc_class source block. Emit functions will live in the
- // main source block for now. In future, we can generalize this by
- // adding a syntax that specifies the sizes of fields in an order,
- // so that the adlc can build the emit functions automagically
-
- // Set instruction mark in MacroAssembler. This is used only in
- // instructions that emit bytes directly to the CodeBuffer wraped
- // in the MacroAssembler. Should go away once all "instruct" are
- // patched to emit bytes only using methods in MacroAssembler.
- enc_class SetInstMark %{
- __ set_inst_mark();
- %}
-
- enc_class ClearInstMark %{
- __ clear_inst_mark();
- %}
-
- // Emit primary opcode
- enc_class OpcP %{
- emit_opcode(masm, $primary);
- %}
-
- // Emit secondary opcode
- enc_class OpcS %{
- emit_opcode(masm, $secondary);
- %}
-
- // Emit opcode directly
- enc_class Opcode(immI d8) %{
- emit_opcode(masm, $d8$$constant);
- %}
-
- enc_class SizePrefix %{
- emit_opcode(masm,0x66);
- %}
-
- enc_class RegReg (rRegI dst, rRegI src) %{ // RegReg(Many)
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class OpcRegReg (immI opcode, rRegI dst, rRegI src) %{ // OpcRegReg(Many)
- emit_opcode(masm,$opcode$$constant);
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class mov_r32_imm0( rRegI dst ) %{
- emit_opcode( masm, 0xB8 + $dst$$reg ); // 0xB8+ rd -- MOV r32 ,imm32
- emit_d32 ( masm, 0x0 ); // imm32==0x0
- %}
-
- enc_class cdq_enc %{
- // Full implementation of Java idiv and irem; checks for
- // special case as described in JVM spec., p.243 & p.271.
- //
- // normal case special case
- //
- // input : rax,: dividend min_int
- // reg: divisor -1
- //
- // output: rax,: quotient (= rax, idiv reg) min_int
- // rdx: remainder (= rax, irem reg) 0
- //
- // Code sequnce:
- //
- // 81 F8 00 00 00 80 cmp rax,80000000h
- // 0F 85 0B 00 00 00 jne normal_case
- // 33 D2 xor rdx,edx
- // 83 F9 FF cmp rcx,0FFh
- // 0F 84 03 00 00 00 je done
- // normal_case:
- // 99 cdq
- // F7 F9 idiv rax,ecx
- // done:
- //
- emit_opcode(masm,0x81); emit_d8(masm,0xF8);
- emit_opcode(masm,0x00); emit_d8(masm,0x00);
- emit_opcode(masm,0x00); emit_d8(masm,0x80); // cmp rax,80000000h
- emit_opcode(masm,0x0F); emit_d8(masm,0x85);
- emit_opcode(masm,0x0B); emit_d8(masm,0x00);
- emit_opcode(masm,0x00); emit_d8(masm,0x00); // jne normal_case
- emit_opcode(masm,0x33); emit_d8(masm,0xD2); // xor rdx,edx
- emit_opcode(masm,0x83); emit_d8(masm,0xF9); emit_d8(masm,0xFF); // cmp rcx,0FFh
- emit_opcode(masm,0x0F); emit_d8(masm,0x84);
- emit_opcode(masm,0x03); emit_d8(masm,0x00);
- emit_opcode(masm,0x00); emit_d8(masm,0x00); // je done
- // normal_case:
- emit_opcode(masm,0x99); // cdq
- // idiv (note: must be emitted by the user of this rule)
- // normal:
- %}
-
- // Dense encoding for older common ops
- enc_class Opc_plus(immI opcode, rRegI reg) %{
- emit_opcode(masm, $opcode$$constant + $reg$$reg);
- %}
-
-
- // Opcde enc_class for 8/32 bit immediate instructions with sign-extension
- enc_class OpcSE (immI imm) %{ // Emit primary opcode and set sign-extend bit
- // Check for 8-bit immediate, and set sign extend bit in opcode
- if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
- emit_opcode(masm, $primary | 0x02);
- }
- else { // If 32-bit immediate
- emit_opcode(masm, $primary);
- }
- %}
-
- enc_class OpcSErm (rRegI dst, immI imm) %{ // OpcSEr/m
- // Emit primary opcode and set sign-extend bit
- // Check for 8-bit immediate, and set sign extend bit in opcode
- if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
- emit_opcode(masm, $primary | 0x02); }
- else { // If 32-bit immediate
- emit_opcode(masm, $primary);
- }
- // Emit r/m byte with secondary opcode, after primary opcode.
- emit_rm(masm, 0x3, $secondary, $dst$$reg);
- %}
-
- enc_class Con8or32 (immI imm) %{ // Con8or32(storeImmI), 8 or 32 bits
- // Check for 8-bit immediate, and set sign extend bit in opcode
- if (($imm$$constant >= -128) && ($imm$$constant <= 127)) {
- $$$emit8$imm$$constant;
- }
- else { // If 32-bit immediate
- // Output immediate
- $$$emit32$imm$$constant;
- }
- %}
-
- enc_class Long_OpcSErm_Lo(eRegL dst, immL imm) %{
- // Emit primary opcode and set sign-extend bit
- // Check for 8-bit immediate, and set sign extend bit in opcode
- int con = (int)$imm$$constant; // Throw away top bits
- emit_opcode(masm, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
- // Emit r/m byte with secondary opcode, after primary opcode.
- emit_rm(masm, 0x3, $secondary, $dst$$reg);
- if ((con >= -128) && (con <= 127)) emit_d8 (masm,con);
- else emit_d32(masm,con);
- %}
-
- enc_class Long_OpcSErm_Hi(eRegL dst, immL imm) %{
- // Emit primary opcode and set sign-extend bit
- // Check for 8-bit immediate, and set sign extend bit in opcode
- int con = (int)($imm$$constant >> 32); // Throw away bottom bits
- emit_opcode(masm, ((con >= -128) && (con <= 127)) ? ($primary | 0x02) : $primary);
- // Emit r/m byte with tertiary opcode, after primary opcode.
- emit_rm(masm, 0x3, $tertiary, HIGH_FROM_LOW_ENC($dst$$reg));
- if ((con >= -128) && (con <= 127)) emit_d8 (masm,con);
- else emit_d32(masm,con);
- %}
-
- enc_class OpcSReg (rRegI dst) %{ // BSWAP
- emit_cc(masm, $secondary, $dst$$reg );
- %}
-
- enc_class bswap_long_bytes(eRegL dst) %{ // BSWAP
- int destlo = $dst$$reg;
- int desthi = HIGH_FROM_LOW_ENC(destlo);
- // bswap lo
- emit_opcode(masm, 0x0F);
- emit_cc(masm, 0xC8, destlo);
- // bswap hi
- emit_opcode(masm, 0x0F);
- emit_cc(masm, 0xC8, desthi);
- // xchg lo and hi
- emit_opcode(masm, 0x87);
- emit_rm(masm, 0x3, destlo, desthi);
- %}
-
- enc_class RegOpc (rRegI div) %{ // IDIV, IMOD, JMP indirect, ...
- emit_rm(masm, 0x3, $secondary, $div$$reg );
- %}
-
- enc_class enc_cmov(cmpOp cop ) %{ // CMOV
- $$$emit8$primary;
- emit_cc(masm, $secondary, $cop$$cmpcode);
- %}
-
- enc_class enc_cmov_dpr(cmpOp cop, regDPR src ) %{ // CMOV
- int op = 0xDA00 + $cop$$cmpcode + ($src$$reg-1);
- emit_d8(masm, op >> 8 );
- emit_d8(masm, op & 255);
- %}
-
- // emulate a CMOV with a conditional branch around a MOV
- enc_class enc_cmov_branch( cmpOp cop, immI brOffs ) %{ // CMOV
- // Invert sense of branch from sense of CMOV
- emit_cc( masm, 0x70, ($cop$$cmpcode^1) );
- emit_d8( masm, $brOffs$$constant );
- %}
-
- enc_class enc_PartialSubtypeCheck( ) %{
- Register Redi = as_Register(EDI_enc); // result register
- Register Reax = as_Register(EAX_enc); // super class
- Register Recx = as_Register(ECX_enc); // killed
- Register Resi = as_Register(ESI_enc); // sub class
- Label miss;
-
- // NB: Callers may assume that, when $result is a valid register,
- // check_klass_subtype_slow_path sets it to a nonzero value.
- __ check_klass_subtype_slow_path(Resi, Reax, Recx, Redi,
- nullptr, &miss,
- /*set_cond_codes:*/ true);
- if ($primary) {
- __ xorptr(Redi, Redi);
- }
- __ bind(miss);
- %}
-
- enc_class FFree_Float_Stack_All %{ // Free_Float_Stack_All
- int start = __ offset();
- if (UseSSE >= 2) {
- if (VerifyFPU) {
- __ verify_FPU(0, "must be empty in SSE2+ mode");
- }
- } else {
- // External c_calling_convention expects the FPU stack to be 'clean'.
- // Compiled code leaves it dirty. Do cleanup now.
- __ empty_FPU_stack();
- }
- if (sizeof_FFree_Float_Stack_All == -1) {
- sizeof_FFree_Float_Stack_All = __ offset() - start;
- } else {
- assert(__ offset() - start == sizeof_FFree_Float_Stack_All, "wrong size");
- }
- %}
-
- enc_class Verify_FPU_For_Leaf %{
- if( VerifyFPU ) {
- __ verify_FPU( -3, "Returning from Runtime Leaf call");
- }
- %}
-
- enc_class Java_To_Runtime (method meth) %{ // CALL Java_To_Runtime, Java_To_Runtime_Leaf
- // This is the instruction starting address for relocation info.
- __ set_inst_mark();
- $$$emit8$primary;
- // CALL directly to the runtime
- emit_d32_reloc(masm, ($meth$$method - (int)(__ pc()) - 4),
- runtime_call_Relocation::spec(), RELOC_IMM32 );
- __ clear_inst_mark();
- __ post_call_nop();
-
- if (UseSSE >= 2) {
- BasicType rt = tf()->return_type();
-
- if ((rt == T_FLOAT || rt == T_DOUBLE) && !return_value_is_used()) {
- // A C runtime call where the return value is unused. In SSE2+
- // mode the result needs to be removed from the FPU stack. It's
- // likely that this function call could be removed by the
- // optimizer if the C function is a pure function.
- __ ffree(0);
- } else if (rt == T_FLOAT) {
- __ lea(rsp, Address(rsp, -4));
- __ fstp_s(Address(rsp, 0));
- __ movflt(xmm0, Address(rsp, 0));
- __ lea(rsp, Address(rsp, 4));
- } else if (rt == T_DOUBLE) {
- __ lea(rsp, Address(rsp, -8));
- __ fstp_d(Address(rsp, 0));
- __ movdbl(xmm0, Address(rsp, 0));
- __ lea(rsp, Address(rsp, 8));
- }
- }
- %}
-
- enc_class pre_call_resets %{
- // If method sets FPU control word restore it here
- debug_only(int off0 = __ offset());
- if (ra_->C->in_24_bit_fp_mode()) {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
- }
- // Clear upper bits of YMM registers when current compiled code uses
- // wide vectors to avoid AVX <-> SSE transition penalty during call.
- __ vzeroupper();
- debug_only(int off1 = __ offset());
- assert(off1 - off0 == pre_call_resets_size(), "correct size prediction");
- %}
-
- enc_class post_call_FPU %{
- // If method sets FPU control word do it here also
- if (Compile::current()->in_24_bit_fp_mode()) {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
- }
- %}
-
- enc_class Java_Static_Call (method meth) %{ // JAVA STATIC CALL
- // CALL to fixup routine. Fixup routine uses ScopeDesc info to determine
- // who we intended to call.
- __ set_inst_mark();
- $$$emit8$primary;
-
- if (!_method) {
- emit_d32_reloc(masm, ($meth$$method - (int)(__ pc()) - 4),
- runtime_call_Relocation::spec(),
- RELOC_IMM32);
- __ clear_inst_mark();
- __ post_call_nop();
- } else {
- int method_index = resolved_method_index(masm);
- RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
- : static_call_Relocation::spec(method_index);
- emit_d32_reloc(masm, ($meth$$method - (int)(__ pc()) - 4),
- rspec, RELOC_DISP32);
- __ post_call_nop();
- address mark = __ inst_mark();
- if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
- // Calls of the same statically bound method can share
- // a stub to the interpreter.
- __ code()->shared_stub_to_interp_for(_method, __ code()->insts()->mark_off());
- __ clear_inst_mark();
- } else {
- // Emit stubs for static call.
- address stub = CompiledDirectCall::emit_to_interp_stub(masm, mark);
- __ clear_inst_mark();
- if (stub == nullptr) {
- ciEnv::current()->record_failure("CodeCache is full");
- return;
- }
- }
- }
- %}
-
- enc_class Java_Dynamic_Call (method meth) %{ // JAVA DYNAMIC CALL
- __ ic_call((address)$meth$$method, resolved_method_index(masm));
- __ post_call_nop();
- %}
-
- enc_class Java_Compiled_Call (method meth) %{ // JAVA COMPILED CALL
- int disp = in_bytes(Method::from_compiled_offset());
- assert( -128 <= disp && disp <= 127, "compiled_code_offset isn't small");
-
- // CALL *[EAX+in_bytes(Method::from_compiled_code_entry_point_offset())]
- __ set_inst_mark();
- $$$emit8$primary;
- emit_rm(masm, 0x01, $secondary, EAX_enc ); // R/M byte
- emit_d8(masm, disp); // Displacement
- __ clear_inst_mark();
- __ post_call_nop();
- %}
-
- enc_class RegOpcImm (rRegI dst, immI8 shift) %{ // SHL, SAR, SHR
- $$$emit8$primary;
- emit_rm(masm, 0x3, $secondary, $dst$$reg);
- $$$emit8$shift$$constant;
- %}
-
- enc_class LdImmI (rRegI dst, immI src) %{ // Load Immediate
- // Load immediate does not have a zero or sign extended version
- // for 8-bit immediates
- emit_opcode(masm, 0xB8 + $dst$$reg);
- $$$emit32$src$$constant;
- %}
-
- enc_class LdImmP (rRegI dst, immI src) %{ // Load Immediate
- // Load immediate does not have a zero or sign extended version
- // for 8-bit immediates
- emit_opcode(masm, $primary + $dst$$reg);
- $$$emit32$src$$constant;
- %}
-
- enc_class LdImmL_Lo( eRegL dst, immL src) %{ // Load Immediate
- // Load immediate does not have a zero or sign extended version
- // for 8-bit immediates
- int dst_enc = $dst$$reg;
- int src_con = $src$$constant & 0x0FFFFFFFFL;
- if (src_con == 0) {
- // xor dst, dst
- emit_opcode(masm, 0x33);
- emit_rm(masm, 0x3, dst_enc, dst_enc);
- } else {
- emit_opcode(masm, $primary + dst_enc);
- emit_d32(masm, src_con);
- }
- %}
-
- enc_class LdImmL_Hi( eRegL dst, immL src) %{ // Load Immediate
- // Load immediate does not have a zero or sign extended version
- // for 8-bit immediates
- int dst_enc = $dst$$reg + 2;
- int src_con = ((julong)($src$$constant)) >> 32;
- if (src_con == 0) {
- // xor dst, dst
- emit_opcode(masm, 0x33);
- emit_rm(masm, 0x3, dst_enc, dst_enc);
- } else {
- emit_opcode(masm, $primary + dst_enc);
- emit_d32(masm, src_con);
- }
- %}
-
-
- // Encode a reg-reg copy. If it is useless, then empty encoding.
- enc_class enc_Copy( rRegI dst, rRegI src ) %{
- encode_Copy( masm, $dst$$reg, $src$$reg );
- %}
-
- enc_class enc_CopyL_Lo( rRegI dst, eRegL src ) %{
- encode_Copy( masm, $dst$$reg, $src$$reg );
- %}
-
- enc_class RegReg (rRegI dst, rRegI src) %{ // RegReg(Many)
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class RegReg_Lo(eRegL dst, eRegL src) %{ // RegReg(Many)
- $$$emit8$primary;
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class RegReg_Hi(eRegL dst, eRegL src) %{ // RegReg(Many)
- $$$emit8$secondary;
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), HIGH_FROM_LOW_ENC($src$$reg));
- %}
-
- enc_class RegReg_Lo2(eRegL dst, eRegL src) %{ // RegReg(Many)
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class RegReg_Hi2(eRegL dst, eRegL src) %{ // RegReg(Many)
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), HIGH_FROM_LOW_ENC($src$$reg));
- %}
-
- enc_class RegReg_HiLo( eRegL src, rRegI dst ) %{
- emit_rm(masm, 0x3, $dst$$reg, HIGH_FROM_LOW_ENC($src$$reg));
- %}
-
- enc_class Con32 (immI src) %{ // Con32(storeImmI)
- // Output immediate
- $$$emit32$src$$constant;
- %}
-
- enc_class Con32FPR_as_bits(immFPR src) %{ // storeF_imm
- // Output Float immediate bits
- jfloat jf = $src$$constant;
- int jf_as_bits = jint_cast( jf );
- emit_d32(masm, jf_as_bits);
- %}
-
- enc_class Con32F_as_bits(immF src) %{ // storeX_imm
- // Output Float immediate bits
- jfloat jf = $src$$constant;
- int jf_as_bits = jint_cast( jf );
- emit_d32(masm, jf_as_bits);
- %}
-
- enc_class Con16 (immI src) %{ // Con16(storeImmI)
- // Output immediate
- $$$emit16$src$$constant;
- %}
-
- enc_class Con_d32(immI src) %{
- emit_d32(masm,$src$$constant);
- %}
-
- enc_class conmemref (eRegP t1) %{ // Con32(storeImmI)
- // Output immediate memory reference
- emit_rm(masm, 0x00, $t1$$reg, 0x05 );
- emit_d32(masm, 0x00);
- %}
-
- enc_class lock_prefix( ) %{
- emit_opcode(masm,0xF0); // [Lock]
- %}
-
- // Cmp-xchg long value.
- // Note: we need to swap rbx, and rcx before and after the
- // cmpxchg8 instruction because the instruction uses
- // rcx as the high order word of the new value to store but
- // our register encoding uses rbx,.
- enc_class enc_cmpxchg8(eSIRegP mem_ptr) %{
-
- // XCHG rbx,ecx
- emit_opcode(masm,0x87);
- emit_opcode(masm,0xD9);
- // [Lock]
- emit_opcode(masm,0xF0);
- // CMPXCHG8 [Eptr]
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xC7);
- emit_rm( masm, 0x0, 1, $mem_ptr$$reg );
- // XCHG rbx,ecx
- emit_opcode(masm,0x87);
- emit_opcode(masm,0xD9);
- %}
-
- enc_class enc_cmpxchg(eSIRegP mem_ptr) %{
- // [Lock]
- emit_opcode(masm,0xF0);
-
- // CMPXCHG [Eptr]
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xB1);
- emit_rm( masm, 0x0, 1, $mem_ptr$$reg );
- %}
-
- enc_class enc_cmpxchgb(eSIRegP mem_ptr) %{
- // [Lock]
- emit_opcode(masm,0xF0);
-
- // CMPXCHGB [Eptr]
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xB0);
- emit_rm( masm, 0x0, 1, $mem_ptr$$reg );
- %}
-
- enc_class enc_cmpxchgw(eSIRegP mem_ptr) %{
- // [Lock]
- emit_opcode(masm,0xF0);
-
- // 16-bit mode
- emit_opcode(masm, 0x66);
-
- // CMPXCHGW [Eptr]
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xB1);
- emit_rm( masm, 0x0, 1, $mem_ptr$$reg );
- %}
-
- enc_class enc_flags_ne_to_boolean( iRegI res ) %{
- int res_encoding = $res$$reg;
-
- // MOV res,0
- emit_opcode( masm, 0xB8 + res_encoding);
- emit_d32( masm, 0 );
- // JNE,s fail
- emit_opcode(masm,0x75);
- emit_d8(masm, 5 );
- // MOV res,1
- emit_opcode( masm, 0xB8 + res_encoding);
- emit_d32( masm, 1 );
- // fail:
- %}
-
- enc_class RegMem (rRegI ereg, memory mem) %{ // emit_reg_mem
- int reg_encoding = $ereg$$reg;
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp;
- relocInfo::relocType disp_reloc = $mem->disp_reloc();
- encode_RegMem(masm, reg_encoding, base, index, scale, displace, disp_reloc);
- %}
-
- enc_class RegMem_Hi(eRegL ereg, memory mem) %{ // emit_reg_mem
- int reg_encoding = HIGH_FROM_LOW_ENC($ereg$$reg); // Hi register of pair, computed from lo
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp + 4; // Offset is 4 further in memory
- assert( $mem->disp_reloc() == relocInfo::none, "Cannot add 4 to oop" );
- encode_RegMem(masm, reg_encoding, base, index, scale, displace, relocInfo::none);
- %}
-
- enc_class move_long_small_shift( eRegL dst, immI_1_31 cnt ) %{
- int r1, r2;
- if( $tertiary == 0xA4 ) { r1 = $dst$$reg; r2 = HIGH_FROM_LOW_ENC($dst$$reg); }
- else { r2 = $dst$$reg; r1 = HIGH_FROM_LOW_ENC($dst$$reg); }
- emit_opcode(masm,0x0F);
- emit_opcode(masm,$tertiary);
- emit_rm(masm, 0x3, r1, r2);
- emit_d8(masm,$cnt$$constant);
- emit_d8(masm,$primary);
- emit_rm(masm, 0x3, $secondary, r1);
- emit_d8(masm,$cnt$$constant);
- %}
-
- enc_class move_long_big_shift_sign( eRegL dst, immI_32_63 cnt ) %{
- emit_opcode( masm, 0x8B ); // Move
- emit_rm(masm, 0x3, $dst$$reg, HIGH_FROM_LOW_ENC($dst$$reg));
- if( $cnt$$constant > 32 ) { // Shift, if not by zero
- emit_d8(masm,$primary);
- emit_rm(masm, 0x3, $secondary, $dst$$reg);
- emit_d8(masm,$cnt$$constant-32);
- }
- emit_d8(masm,$primary);
- emit_rm(masm, 0x3, $secondary, HIGH_FROM_LOW_ENC($dst$$reg));
- emit_d8(masm,31);
- %}
-
- enc_class move_long_big_shift_clr( eRegL dst, immI_32_63 cnt ) %{
- int r1, r2;
- if( $secondary == 0x5 ) { r1 = $dst$$reg; r2 = HIGH_FROM_LOW_ENC($dst$$reg); }
- else { r2 = $dst$$reg; r1 = HIGH_FROM_LOW_ENC($dst$$reg); }
-
- emit_opcode( masm, 0x8B ); // Move r1,r2
- emit_rm(masm, 0x3, r1, r2);
- if( $cnt$$constant > 32 ) { // Shift, if not by zero
- emit_opcode(masm,$primary);
- emit_rm(masm, 0x3, $secondary, r1);
- emit_d8(masm,$cnt$$constant-32);
- }
- emit_opcode(masm,0x33); // XOR r2,r2
- emit_rm(masm, 0x3, r2, r2);
- %}
-
- // Clone of RegMem but accepts an extra parameter to access each
- // half of a double in memory; it never needs relocation info.
- enc_class Mov_MemD_half_to_Reg (immI opcode, memory mem, immI disp_for_half, rRegI rm_reg) %{
- emit_opcode(masm,$opcode$$constant);
- int reg_encoding = $rm_reg$$reg;
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp + $disp_for_half$$constant;
- relocInfo::relocType disp_reloc = relocInfo::none;
- encode_RegMem(masm, reg_encoding, base, index, scale, displace, disp_reloc);
- %}
-
- // !!!!! Special Custom Code used by MemMove, and stack access instructions !!!!!
- //
- // Clone of RegMem except the RM-byte's reg/opcode field is an ADLC-time constant
- // and it never needs relocation information.
- // Frequently used to move data between FPU's Stack Top and memory.
- enc_class RMopc_Mem_no_oop (immI rm_opcode, memory mem) %{
- int rm_byte_opcode = $rm_opcode$$constant;
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp;
- assert( $mem->disp_reloc() == relocInfo::none, "No oops here because no reloc info allowed" );
- encode_RegMem(masm, rm_byte_opcode, base, index, scale, displace, relocInfo::none);
- %}
-
- enc_class RMopc_Mem (immI rm_opcode, memory mem) %{
- int rm_byte_opcode = $rm_opcode$$constant;
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp;
- relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
- encode_RegMem(masm, rm_byte_opcode, base, index, scale, displace, disp_reloc);
- %}
-
- enc_class RegLea (rRegI dst, rRegI src0, immI src1 ) %{ // emit_reg_lea
- int reg_encoding = $dst$$reg;
- int base = $src0$$reg; // 0xFFFFFFFF indicates no base
- int index = 0x04; // 0x04 indicates no index
- int scale = 0x00; // 0x00 indicates no scale
- int displace = $src1$$constant; // 0x00 indicates no displacement
- relocInfo::relocType disp_reloc = relocInfo::none;
- encode_RegMem(masm, reg_encoding, base, index, scale, displace, disp_reloc);
- %}
-
- enc_class min_enc (rRegI dst, rRegI src) %{ // MIN
- // Compare dst,src
- emit_opcode(masm,0x3B);
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- // jmp dst < src around move
- emit_opcode(masm,0x7C);
- emit_d8(masm,2);
- // move dst,src
- emit_opcode(masm,0x8B);
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class max_enc (rRegI dst, rRegI src) %{ // MAX
- // Compare dst,src
- emit_opcode(masm,0x3B);
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- // jmp dst > src around move
- emit_opcode(masm,0x7F);
- emit_d8(masm,2);
- // move dst,src
- emit_opcode(masm,0x8B);
- emit_rm(masm, 0x3, $dst$$reg, $src$$reg);
- %}
-
- enc_class enc_FPR_store(memory mem, regDPR src) %{
- // If src is FPR1, we can just FST to store it.
- // Else we need to FLD it to FPR1, then FSTP to store/pop it.
- int reg_encoding = 0x2; // Just store
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp;
- relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
- if( $src$$reg != FPR1L_enc ) {
- reg_encoding = 0x3; // Store & pop
- emit_opcode( masm, 0xD9 ); // FLD (i.e., push it)
- emit_d8( masm, 0xC0-1+$src$$reg );
- }
- __ set_inst_mark(); // Mark start of opcode for reloc info in mem operand
- emit_opcode(masm,$primary);
- encode_RegMem(masm, reg_encoding, base, index, scale, displace, disp_reloc);
- __ clear_inst_mark();
- %}
-
- enc_class neg_reg(rRegI dst) %{
- // NEG $dst
- emit_opcode(masm,0xF7);
- emit_rm(masm, 0x3, 0x03, $dst$$reg );
- %}
-
- enc_class setLT_reg(eCXRegI dst) %{
- // SETLT $dst
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0x9C);
- emit_rm( masm, 0x3, 0x4, $dst$$reg );
- %}
-
- enc_class enc_cmpLTP(ncxRegI p, ncxRegI q, ncxRegI y, eCXRegI tmp) %{ // cadd_cmpLT
- int tmpReg = $tmp$$reg;
-
- // SUB $p,$q
- emit_opcode(masm,0x2B);
- emit_rm(masm, 0x3, $p$$reg, $q$$reg);
- // SBB $tmp,$tmp
- emit_opcode(masm,0x1B);
- emit_rm(masm, 0x3, tmpReg, tmpReg);
- // AND $tmp,$y
- emit_opcode(masm,0x23);
- emit_rm(masm, 0x3, tmpReg, $y$$reg);
- // ADD $p,$tmp
- emit_opcode(masm,0x03);
- emit_rm(masm, 0x3, $p$$reg, tmpReg);
- %}
-
- enc_class shift_left_long( eRegL dst, eCXRegI shift ) %{
- // TEST shift,32
- emit_opcode(masm,0xF7);
- emit_rm(masm, 0x3, 0, ECX_enc);
- emit_d32(masm,0x20);
- // JEQ,s small
- emit_opcode(masm, 0x74);
- emit_d8(masm, 0x04);
- // MOV $dst.hi,$dst.lo
- emit_opcode( masm, 0x8B );
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), $dst$$reg );
- // CLR $dst.lo
- emit_opcode(masm, 0x33);
- emit_rm(masm, 0x3, $dst$$reg, $dst$$reg);
-// small:
- // SHLD $dst.hi,$dst.lo,$shift
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xA5);
- emit_rm(masm, 0x3, $dst$$reg, HIGH_FROM_LOW_ENC($dst$$reg));
- // SHL $dst.lo,$shift"
- emit_opcode(masm,0xD3);
- emit_rm(masm, 0x3, 0x4, $dst$$reg );
- %}
-
- enc_class shift_right_long( eRegL dst, eCXRegI shift ) %{
- // TEST shift,32
- emit_opcode(masm,0xF7);
- emit_rm(masm, 0x3, 0, ECX_enc);
- emit_d32(masm,0x20);
- // JEQ,s small
- emit_opcode(masm, 0x74);
- emit_d8(masm, 0x04);
- // MOV $dst.lo,$dst.hi
- emit_opcode( masm, 0x8B );
- emit_rm(masm, 0x3, $dst$$reg, HIGH_FROM_LOW_ENC($dst$$reg) );
- // CLR $dst.hi
- emit_opcode(masm, 0x33);
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), HIGH_FROM_LOW_ENC($dst$$reg));
-// small:
- // SHRD $dst.lo,$dst.hi,$shift
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xAD);
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), $dst$$reg);
- // SHR $dst.hi,$shift"
- emit_opcode(masm,0xD3);
- emit_rm(masm, 0x3, 0x5, HIGH_FROM_LOW_ENC($dst$$reg) );
- %}
-
- enc_class shift_right_arith_long( eRegL dst, eCXRegI shift ) %{
- // TEST shift,32
- emit_opcode(masm,0xF7);
- emit_rm(masm, 0x3, 0, ECX_enc);
- emit_d32(masm,0x20);
- // JEQ,s small
- emit_opcode(masm, 0x74);
- emit_d8(masm, 0x05);
- // MOV $dst.lo,$dst.hi
- emit_opcode( masm, 0x8B );
- emit_rm(masm, 0x3, $dst$$reg, HIGH_FROM_LOW_ENC($dst$$reg) );
- // SAR $dst.hi,31
- emit_opcode(masm, 0xC1);
- emit_rm(masm, 0x3, 7, HIGH_FROM_LOW_ENC($dst$$reg) );
- emit_d8(masm, 0x1F );
-// small:
- // SHRD $dst.lo,$dst.hi,$shift
- emit_opcode(masm,0x0F);
- emit_opcode(masm,0xAD);
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), $dst$$reg);
- // SAR $dst.hi,$shift"
- emit_opcode(masm,0xD3);
- emit_rm(masm, 0x3, 0x7, HIGH_FROM_LOW_ENC($dst$$reg) );
- %}
-
-
- // ----------------- Encodings for floating point unit -----------------
- // May leave result in FPU-TOS or FPU reg depending on opcodes
- enc_class OpcReg_FPR(regFPR src) %{ // FMUL, FDIV
- $$$emit8$primary;
- emit_rm(masm, 0x3, $secondary, $src$$reg );
- %}
-
- // Pop argument in FPR0 with FSTP ST(0)
- enc_class PopFPU() %{
- emit_opcode( masm, 0xDD );
- emit_d8( masm, 0xD8 );
- %}
-
- // !!!!! equivalent to Pop_Reg_F
- enc_class Pop_Reg_DPR( regDPR dst ) %{
- emit_opcode( masm, 0xDD ); // FSTP ST(i)
- emit_d8( masm, 0xD8+$dst$$reg );
- %}
-
- enc_class Push_Reg_DPR( regDPR dst ) %{
- emit_opcode( masm, 0xD9 );
- emit_d8( masm, 0xC0-1+$dst$$reg ); // FLD ST(i-1)
- %}
-
- enc_class strictfp_bias1( regDPR dst ) %{
- emit_opcode( masm, 0xDB ); // FLD m80real
- emit_opcode( masm, 0x2D );
- emit_d32( masm, (int)StubRoutines::x86::addr_fpu_subnormal_bias1() );
- emit_opcode( masm, 0xDE ); // FMULP ST(dst), ST0
- emit_opcode( masm, 0xC8+$dst$$reg );
- %}
-
- enc_class strictfp_bias2( regDPR dst ) %{
- emit_opcode( masm, 0xDB ); // FLD m80real
- emit_opcode( masm, 0x2D );
- emit_d32( masm, (int)StubRoutines::x86::addr_fpu_subnormal_bias2() );
- emit_opcode( masm, 0xDE ); // FMULP ST(dst), ST0
- emit_opcode( masm, 0xC8+$dst$$reg );
- %}
-
- // Special case for moving an integer register to a stack slot.
- enc_class OpcPRegSS( stackSlotI dst, rRegI src ) %{ // RegSS
- store_to_stackslot( masm, $primary, $src$$reg, $dst$$disp );
- %}
-
- // Special case for moving a register to a stack slot.
- enc_class RegSS( stackSlotI dst, rRegI src ) %{ // RegSS
- // Opcode already emitted
- emit_rm( masm, 0x02, $src$$reg, ESP_enc ); // R/M byte
- emit_rm( masm, 0x00, ESP_enc, ESP_enc); // SIB byte
- emit_d32(masm, $dst$$disp); // Displacement
- %}
-
- // Push the integer in stackSlot 'src' onto FP-stack
- enc_class Push_Mem_I( memory src ) %{ // FILD [ESP+src]
- store_to_stackslot( masm, $primary, $secondary, $src$$disp );
- %}
-
- // Push FPU's TOS float to a stack-slot, and pop FPU-stack
- enc_class Pop_Mem_FPR( stackSlotF dst ) %{ // FSTP_S [ESP+dst]
- store_to_stackslot( masm, 0xD9, 0x03, $dst$$disp );
- %}
-
- // Same as Pop_Mem_F except for opcode
- // Push FPU's TOS double to a stack-slot, and pop FPU-stack
- enc_class Pop_Mem_DPR( stackSlotD dst ) %{ // FSTP_D [ESP+dst]
- store_to_stackslot( masm, 0xDD, 0x03, $dst$$disp );
- %}
-
- enc_class Pop_Reg_FPR( regFPR dst ) %{
- emit_opcode( masm, 0xDD ); // FSTP ST(i)
- emit_d8( masm, 0xD8+$dst$$reg );
- %}
-
- enc_class Push_Reg_FPR( regFPR dst ) %{
- emit_opcode( masm, 0xD9 ); // FLD ST(i-1)
- emit_d8( masm, 0xC0-1+$dst$$reg );
- %}
-
- // Push FPU's float to a stack-slot, and pop FPU-stack
- enc_class Pop_Mem_Reg_FPR( stackSlotF dst, regFPR src ) %{
- int pop = 0x02;
- if ($src$$reg != FPR1L_enc) {
- emit_opcode( masm, 0xD9 ); // FLD ST(i-1)
- emit_d8( masm, 0xC0-1+$src$$reg );
- pop = 0x03;
- }
- store_to_stackslot( masm, 0xD9, pop, $dst$$disp ); // FST_S [ESP+dst]
- %}
-
- // Push FPU's double to a stack-slot, and pop FPU-stack
- enc_class Pop_Mem_Reg_DPR( stackSlotD dst, regDPR src ) %{
- int pop = 0x02;
- if ($src$$reg != FPR1L_enc) {
- emit_opcode( masm, 0xD9 ); // FLD ST(i-1)
- emit_d8( masm, 0xC0-1+$src$$reg );
- pop = 0x03;
- }
- store_to_stackslot( masm, 0xDD, pop, $dst$$disp ); // FST
_D [ESP+dst]
- %}
-
- // Push FPU's double to a FPU-stack-slot, and pop FPU-stack
- enc_class Pop_Reg_Reg_DPR( regDPR dst, regFPR src ) %{
- int pop = 0xD0 - 1; // -1 since we skip FLD
- if ($src$$reg != FPR1L_enc) {
- emit_opcode( masm, 0xD9 ); // FLD ST(src-1)
- emit_d8( masm, 0xC0-1+$src$$reg );
- pop = 0xD8;
- }
- emit_opcode( masm, 0xDD );
- emit_d8( masm, pop+$dst$$reg ); // FST
ST(i)
- %}
-
-
- enc_class Push_Reg_Mod_DPR( regDPR dst, regDPR src) %{
- // load dst in FPR0
- emit_opcode( masm, 0xD9 );
- emit_d8( masm, 0xC0-1+$dst$$reg );
- if ($src$$reg != FPR1L_enc) {
- // fincstp
- emit_opcode (masm, 0xD9);
- emit_opcode (masm, 0xF7);
- // swap src with FPR1:
- // FXCH FPR1 with src
- emit_opcode(masm, 0xD9);
- emit_d8(masm, 0xC8-1+$src$$reg );
- // fdecstp
- emit_opcode (masm, 0xD9);
- emit_opcode (masm, 0xF6);
- }
- %}
-
- enc_class Push_ResultD(regD dst) %{
- __ fstp_d(Address(rsp, 0));
- __ movdbl($dst$$XMMRegister, Address(rsp, 0));
- __ addptr(rsp, 8);
- %}
-
- enc_class Push_ResultF(regF dst, immI d8) %{
- __ fstp_s(Address(rsp, 0));
- __ movflt($dst$$XMMRegister, Address(rsp, 0));
- __ addptr(rsp, $d8$$constant);
- %}
-
- enc_class Push_SrcD(regD src) %{
- __ subptr(rsp, 8);
- __ movdbl(Address(rsp, 0), $src$$XMMRegister);
- __ fld_d(Address(rsp, 0));
- %}
-
- enc_class push_stack_temp_qword() %{
- __ subptr(rsp, 8);
- %}
-
- enc_class pop_stack_temp_qword() %{
- __ addptr(rsp, 8);
- %}
-
- enc_class push_xmm_to_fpr1(regD src) %{
- __ movdbl(Address(rsp, 0), $src$$XMMRegister);
- __ fld_d(Address(rsp, 0));
- %}
-
- enc_class fnstsw_sahf_skip_parity() %{
- // fnstsw ax
- emit_opcode( masm, 0xDF );
- emit_opcode( masm, 0xE0 );
- // sahf
- emit_opcode( masm, 0x9E );
- // jnp ::skip
- emit_opcode( masm, 0x7B );
- emit_opcode( masm, 0x05 );
- %}
-
- enc_class fpu_flags() %{
- // fnstsw_ax
- emit_opcode( masm, 0xDF);
- emit_opcode( masm, 0xE0);
- // test ax,0x0400
- emit_opcode( masm, 0x66 ); // operand-size prefix for 16-bit immediate
- emit_opcode( masm, 0xA9 );
- emit_d16 ( masm, 0x0400 );
- // // // This sequence works, but stalls for 12-16 cycles on PPro
- // // test rax,0x0400
- // emit_opcode( masm, 0xA9 );
- // emit_d32 ( masm, 0x00000400 );
- //
- // jz exit (no unordered comparison)
- emit_opcode( masm, 0x74 );
- emit_d8 ( masm, 0x02 );
- // mov ah,1 - treat as LT case (set carry flag)
- emit_opcode( masm, 0xB4 );
- emit_d8 ( masm, 0x01 );
- // sahf
- emit_opcode( masm, 0x9E);
- %}
-
- enc_class cmpF_P6_fixup() %{
- // Fixup the integer flags in case comparison involved a NaN
- //
- // JNP exit (no unordered comparison, P-flag is set by NaN)
- emit_opcode( masm, 0x7B );
- emit_d8 ( masm, 0x03 );
- // MOV AH,1 - treat as LT case (set carry flag)
- emit_opcode( masm, 0xB4 );
- emit_d8 ( masm, 0x01 );
- // SAHF
- emit_opcode( masm, 0x9E);
- // NOP // target for branch to avoid branch to branch
- emit_opcode( masm, 0x90);
- %}
-
-// fnstsw_ax();
-// sahf();
-// movl(dst, nan_result);
-// jcc(Assembler::parity, exit);
-// movl(dst, less_result);
-// jcc(Assembler::below, exit);
-// movl(dst, equal_result);
-// jcc(Assembler::equal, exit);
-// movl(dst, greater_result);
-
-// less_result = 1;
-// greater_result = -1;
-// equal_result = 0;
-// nan_result = -1;
-
- enc_class CmpF_Result(rRegI dst) %{
- // fnstsw_ax();
- emit_opcode( masm, 0xDF);
- emit_opcode( masm, 0xE0);
- // sahf
- emit_opcode( masm, 0x9E);
- // movl(dst, nan_result);
- emit_opcode( masm, 0xB8 + $dst$$reg);
- emit_d32( masm, -1 );
- // jcc(Assembler::parity, exit);
- emit_opcode( masm, 0x7A );
- emit_d8 ( masm, 0x13 );
- // movl(dst, less_result);
- emit_opcode( masm, 0xB8 + $dst$$reg);
- emit_d32( masm, -1 );
- // jcc(Assembler::below, exit);
- emit_opcode( masm, 0x72 );
- emit_d8 ( masm, 0x0C );
- // movl(dst, equal_result);
- emit_opcode( masm, 0xB8 + $dst$$reg);
- emit_d32( masm, 0 );
- // jcc(Assembler::equal, exit);
- emit_opcode( masm, 0x74 );
- emit_d8 ( masm, 0x05 );
- // movl(dst, greater_result);
- emit_opcode( masm, 0xB8 + $dst$$reg);
- emit_d32( masm, 1 );
- %}
-
-
- // Compare the longs and set flags
- // BROKEN! Do Not use as-is
- enc_class cmpl_test( eRegL src1, eRegL src2 ) %{
- // CMP $src1.hi,$src2.hi
- emit_opcode( masm, 0x3B );
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($src1$$reg), HIGH_FROM_LOW_ENC($src2$$reg) );
- // JNE,s done
- emit_opcode(masm,0x75);
- emit_d8(masm, 2 );
- // CMP $src1.lo,$src2.lo
- emit_opcode( masm, 0x3B );
- emit_rm(masm, 0x3, $src1$$reg, $src2$$reg );
-// done:
- %}
-
- enc_class convert_int_long( regL dst, rRegI src ) %{
- // mov $dst.lo,$src
- int dst_encoding = $dst$$reg;
- int src_encoding = $src$$reg;
- encode_Copy( masm, dst_encoding , src_encoding );
- // mov $dst.hi,$src
- encode_Copy( masm, HIGH_FROM_LOW_ENC(dst_encoding), src_encoding );
- // sar $dst.hi,31
- emit_opcode( masm, 0xC1 );
- emit_rm(masm, 0x3, 7, HIGH_FROM_LOW_ENC(dst_encoding) );
- emit_d8(masm, 0x1F );
- %}
-
- enc_class convert_long_double( eRegL src ) %{
- // push $src.hi
- emit_opcode(masm, 0x50+HIGH_FROM_LOW_ENC($src$$reg));
- // push $src.lo
- emit_opcode(masm, 0x50+$src$$reg );
- // fild 64-bits at [SP]
- emit_opcode(masm,0xdf);
- emit_d8(masm, 0x6C);
- emit_d8(masm, 0x24);
- emit_d8(masm, 0x00);
- // pop stack
- emit_opcode(masm, 0x83); // add SP, #8
- emit_rm(masm, 0x3, 0x00, ESP_enc);
- emit_d8(masm, 0x8);
- %}
-
- enc_class multiply_con_and_shift_high( eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr ) %{
- // IMUL EDX:EAX,$src1
- emit_opcode( masm, 0xF7 );
- emit_rm( masm, 0x3, 0x5, $src1$$reg );
- // SAR EDX,$cnt-32
- int shift_count = ((int)$cnt$$constant) - 32;
- if (shift_count > 0) {
- emit_opcode(masm, 0xC1);
- emit_rm(masm, 0x3, 7, $dst$$reg );
- emit_d8(masm, shift_count);
- }
- %}
-
- // this version doesn't have add sp, 8
- enc_class convert_long_double2( eRegL src ) %{
- // push $src.hi
- emit_opcode(masm, 0x50+HIGH_FROM_LOW_ENC($src$$reg));
- // push $src.lo
- emit_opcode(masm, 0x50+$src$$reg );
- // fild 64-bits at [SP]
- emit_opcode(masm,0xdf);
- emit_d8(masm, 0x6C);
- emit_d8(masm, 0x24);
- emit_d8(masm, 0x00);
- %}
-
- enc_class long_int_multiply( eADXRegL dst, nadxRegI src) %{
- // Basic idea: long = (long)int * (long)int
- // IMUL EDX:EAX, src
- emit_opcode( masm, 0xF7 );
- emit_rm( masm, 0x3, 0x5, $src$$reg);
- %}
-
- enc_class long_uint_multiply( eADXRegL dst, nadxRegI src) %{
- // Basic Idea: long = (int & 0xffffffffL) * (int & 0xffffffffL)
- // MUL EDX:EAX, src
- emit_opcode( masm, 0xF7 );
- emit_rm( masm, 0x3, 0x4, $src$$reg);
- %}
-
- enc_class long_multiply( eADXRegL dst, eRegL src, rRegI tmp ) %{
- // Basic idea: lo(result) = lo(x_lo * y_lo)
- // hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
- // MOV $tmp,$src.lo
- encode_Copy( masm, $tmp$$reg, $src$$reg );
- // IMUL $tmp,EDX
- emit_opcode( masm, 0x0F );
- emit_opcode( masm, 0xAF );
- emit_rm( masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($dst$$reg) );
- // MOV EDX,$src.hi
- encode_Copy( masm, HIGH_FROM_LOW_ENC($dst$$reg), HIGH_FROM_LOW_ENC($src$$reg) );
- // IMUL EDX,EAX
- emit_opcode( masm, 0x0F );
- emit_opcode( masm, 0xAF );
- emit_rm( masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), $dst$$reg );
- // ADD $tmp,EDX
- emit_opcode( masm, 0x03 );
- emit_rm( masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($dst$$reg) );
- // MUL EDX:EAX,$src.lo
- emit_opcode( masm, 0xF7 );
- emit_rm( masm, 0x3, 0x4, $src$$reg );
- // ADD EDX,ESI
- emit_opcode( masm, 0x03 );
- emit_rm( masm, 0x3, HIGH_FROM_LOW_ENC($dst$$reg), $tmp$$reg );
- %}
-
- enc_class long_multiply_con( eADXRegL dst, immL_127 src, rRegI tmp ) %{
- // Basic idea: lo(result) = lo(src * y_lo)
- // hi(result) = hi(src * y_lo) + lo(src * y_hi)
- // IMUL $tmp,EDX,$src
- emit_opcode( masm, 0x6B );
- emit_rm( masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($dst$$reg) );
- emit_d8( masm, (int)$src$$constant );
- // MOV EDX,$src
- emit_opcode(masm, 0xB8 + EDX_enc);
- emit_d32( masm, (int)$src$$constant );
- // MUL EDX:EAX,EDX
- emit_opcode( masm, 0xF7 );
- emit_rm( masm, 0x3, 0x4, EDX_enc );
- // ADD EDX,ESI
- emit_opcode( masm, 0x03 );
- emit_rm( masm, 0x3, EDX_enc, $tmp$$reg );
- %}
-
- enc_class long_div( eRegL src1, eRegL src2 ) %{
- // PUSH src1.hi
- emit_opcode(masm, HIGH_FROM_LOW_ENC(0x50+$src1$$reg) );
- // PUSH src1.lo
- emit_opcode(masm, 0x50+$src1$$reg );
- // PUSH src2.hi
- emit_opcode(masm, HIGH_FROM_LOW_ENC(0x50+$src2$$reg) );
- // PUSH src2.lo
- emit_opcode(masm, 0x50+$src2$$reg );
- // CALL directly to the runtime
- __ set_inst_mark();
- emit_opcode(masm,0xE8); // Call into runtime
- emit_d32_reloc(masm, (CAST_FROM_FN_PTR(address, SharedRuntime::ldiv) - __ pc()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
- __ clear_inst_mark();
- __ post_call_nop();
- // Restore stack
- emit_opcode(masm, 0x83); // add SP, #framesize
- emit_rm(masm, 0x3, 0x00, ESP_enc);
- emit_d8(masm, 4*4);
- %}
-
- enc_class long_mod( eRegL src1, eRegL src2 ) %{
- // PUSH src1.hi
- emit_opcode(masm, HIGH_FROM_LOW_ENC(0x50+$src1$$reg) );
- // PUSH src1.lo
- emit_opcode(masm, 0x50+$src1$$reg );
- // PUSH src2.hi
- emit_opcode(masm, HIGH_FROM_LOW_ENC(0x50+$src2$$reg) );
- // PUSH src2.lo
- emit_opcode(masm, 0x50+$src2$$reg );
- // CALL directly to the runtime
- __ set_inst_mark();
- emit_opcode(masm,0xE8); // Call into runtime
- emit_d32_reloc(masm, (CAST_FROM_FN_PTR(address, SharedRuntime::lrem ) - __ pc()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
- __ clear_inst_mark();
- __ post_call_nop();
- // Restore stack
- emit_opcode(masm, 0x83); // add SP, #framesize
- emit_rm(masm, 0x3, 0x00, ESP_enc);
- emit_d8(masm, 4*4);
- %}
-
- enc_class long_cmp_flags0( eRegL src, rRegI tmp ) %{
- // MOV $tmp,$src.lo
- emit_opcode(masm, 0x8B);
- emit_rm(masm, 0x3, $tmp$$reg, $src$$reg);
- // OR $tmp,$src.hi
- emit_opcode(masm, 0x0B);
- emit_rm(masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($src$$reg));
- %}
-
- enc_class long_cmp_flags1( eRegL src1, eRegL src2 ) %{
- // CMP $src1.lo,$src2.lo
- emit_opcode( masm, 0x3B );
- emit_rm(masm, 0x3, $src1$$reg, $src2$$reg );
- // JNE,s skip
- emit_cc(masm, 0x70, 0x5);
- emit_d8(masm,2);
- // CMP $src1.hi,$src2.hi
- emit_opcode( masm, 0x3B );
- emit_rm(masm, 0x3, HIGH_FROM_LOW_ENC($src1$$reg), HIGH_FROM_LOW_ENC($src2$$reg) );
- %}
-
- enc_class long_cmp_flags2( eRegL src1, eRegL src2, rRegI tmp ) %{
- // CMP $src1.lo,$src2.lo\t! Long compare; set flags for low bits
- emit_opcode( masm, 0x3B );
- emit_rm(masm, 0x3, $src1$$reg, $src2$$reg );
- // MOV $tmp,$src1.hi
- emit_opcode( masm, 0x8B );
- emit_rm(masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($src1$$reg) );
- // SBB $tmp,$src2.hi\t! Compute flags for long compare
- emit_opcode( masm, 0x1B );
- emit_rm(masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($src2$$reg) );
- %}
-
- enc_class long_cmp_flags3( eRegL src, rRegI tmp ) %{
- // XOR $tmp,$tmp
- emit_opcode(masm,0x33); // XOR
- emit_rm(masm,0x3, $tmp$$reg, $tmp$$reg);
- // CMP $tmp,$src.lo
- emit_opcode( masm, 0x3B );
- emit_rm(masm, 0x3, $tmp$$reg, $src$$reg );
- // SBB $tmp,$src.hi
- emit_opcode( masm, 0x1B );
- emit_rm(masm, 0x3, $tmp$$reg, HIGH_FROM_LOW_ENC($src$$reg) );
- %}
-
- // Sniff, sniff... smells like Gnu Superoptimizer
- enc_class neg_long( eRegL dst ) %{
- emit_opcode(masm,0xF7); // NEG hi
- emit_rm (masm,0x3, 0x3, HIGH_FROM_LOW_ENC($dst$$reg));
- emit_opcode(masm,0xF7); // NEG lo
- emit_rm (masm,0x3, 0x3, $dst$$reg );
- emit_opcode(masm,0x83); // SBB hi,0
- emit_rm (masm,0x3, 0x3, HIGH_FROM_LOW_ENC($dst$$reg));
- emit_d8 (masm,0 );
- %}
-
- enc_class enc_pop_rdx() %{
- emit_opcode(masm,0x5A);
- %}
-
- enc_class enc_rethrow() %{
- __ set_inst_mark();
- emit_opcode(masm, 0xE9); // jmp entry
- emit_d32_reloc(masm, (int)OptoRuntime::rethrow_stub() - ((int)__ pc())-4,
- runtime_call_Relocation::spec(), RELOC_IMM32 );
- __ clear_inst_mark();
- __ post_call_nop();
- %}
-
-
- // Convert a double to an int. Java semantics require we do complex
- // manglelations in the corner cases. So we set the rounding mode to
- // 'zero', store the darned double down as an int, and reset the
- // rounding mode to 'nearest'. The hardware throws an exception which
- // patches up the correct value directly to the stack.
- enc_class DPR2I_encoding( regDPR src ) %{
- // Flip to round-to-zero mode. We attempted to allow invalid-op
- // exceptions here, so that a NAN or other corner-case value will
- // thrown an exception (but normal values get converted at full speed).
- // However, I2C adapters and other float-stack manglers leave pending
- // invalid-op exceptions hanging. We would have to clear them before
- // enabling them and that is more expensive than just testing for the
- // invalid value Intel stores down in the corner cases.
- emit_opcode(masm,0xD9); // FLDCW trunc
- emit_opcode(masm,0x2D);
- emit_d32(masm,(int)StubRoutines::x86::addr_fpu_cntrl_wrd_trunc());
- // Allocate a word
- emit_opcode(masm,0x83); // SUB ESP,4
- emit_opcode(masm,0xEC);
- emit_d8(masm,0x04);
- // Encoding assumes a double has been pushed into FPR0.
- // Store down the double as an int, popping the FPU stack
- emit_opcode(masm,0xDB); // FISTP [ESP]
- emit_opcode(masm,0x1C);
- emit_d8(masm,0x24);
- // Restore the rounding mode; mask the exception
- emit_opcode(masm,0xD9); // FLDCW std/24-bit mode
- emit_opcode(masm,0x2D);
- emit_d32( masm, Compile::current()->in_24_bit_fp_mode()
- ? (int)StubRoutines::x86::addr_fpu_cntrl_wrd_24()
- : (int)StubRoutines::x86::addr_fpu_cntrl_wrd_std());
-
- // Load the converted int; adjust CPU stack
- emit_opcode(masm,0x58); // POP EAX
- emit_opcode(masm,0x3D); // CMP EAX,imm
- emit_d32 (masm,0x80000000); // 0x80000000
- emit_opcode(masm,0x75); // JNE around_slow_call
- emit_d8 (masm,0x07); // Size of slow_call
- // Push src onto stack slow-path
- emit_opcode(masm,0xD9 ); // FLD ST(i)
- emit_d8 (masm,0xC0-1+$src$$reg );
- // CALL directly to the runtime
- __ set_inst_mark();
- emit_opcode(masm,0xE8); // Call into runtime
- emit_d32_reloc(masm, (StubRoutines::x86::d2i_wrapper() - __ pc()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
- __ clear_inst_mark();
- __ post_call_nop();
- // Carry on here...
- %}
-
- enc_class DPR2L_encoding( regDPR src ) %{
- emit_opcode(masm,0xD9); // FLDCW trunc
- emit_opcode(masm,0x2D);
- emit_d32(masm,(int)StubRoutines::x86::addr_fpu_cntrl_wrd_trunc());
- // Allocate a word
- emit_opcode(masm,0x83); // SUB ESP,8
- emit_opcode(masm,0xEC);
- emit_d8(masm,0x08);
- // Encoding assumes a double has been pushed into FPR0.
- // Store down the double as a long, popping the FPU stack
- emit_opcode(masm,0xDF); // FISTP [ESP]
- emit_opcode(masm,0x3C);
- emit_d8(masm,0x24);
- // Restore the rounding mode; mask the exception
- emit_opcode(masm,0xD9); // FLDCW std/24-bit mode
- emit_opcode(masm,0x2D);
- emit_d32( masm, Compile::current()->in_24_bit_fp_mode()
- ? (int)StubRoutines::x86::addr_fpu_cntrl_wrd_24()
- : (int)StubRoutines::x86::addr_fpu_cntrl_wrd_std());
-
- // Load the converted int; adjust CPU stack
- emit_opcode(masm,0x58); // POP EAX
- emit_opcode(masm,0x5A); // POP EDX
- emit_opcode(masm,0x81); // CMP EDX,imm
- emit_d8 (masm,0xFA); // rdx
- emit_d32 (masm,0x80000000); // 0x80000000
- emit_opcode(masm,0x75); // JNE around_slow_call
- emit_d8 (masm,0x07+4); // Size of slow_call
- emit_opcode(masm,0x85); // TEST EAX,EAX
- emit_opcode(masm,0xC0); // 2/rax,/rax,
- emit_opcode(masm,0x75); // JNE around_slow_call
- emit_d8 (masm,0x07); // Size of slow_call
- // Push src onto stack slow-path
- emit_opcode(masm,0xD9 ); // FLD ST(i)
- emit_d8 (masm,0xC0-1+$src$$reg );
- // CALL directly to the runtime
- __ set_inst_mark();
- emit_opcode(masm,0xE8); // Call into runtime
- emit_d32_reloc(masm, (StubRoutines::x86::d2l_wrapper() - __ pc()) - 4, runtime_call_Relocation::spec(), RELOC_IMM32 );
- __ clear_inst_mark();
- __ post_call_nop();
- // Carry on here...
- %}
-
- enc_class FMul_ST_reg( eRegFPR src1 ) %{
- // Operand was loaded from memory into fp ST (stack top)
- // FMUL ST,$src /* D8 C8+i */
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xC8 + $src1$$reg);
- %}
-
- enc_class FAdd_ST_reg( eRegFPR src2 ) %{
- // FADDP ST,src2 /* D8 C0+i */
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xC0 + $src2$$reg);
- //could use FADDP src2,fpST /* DE C0+i */
- %}
-
- enc_class FAddP_reg_ST( eRegFPR src2 ) %{
- // FADDP src2,ST /* DE C0+i */
- emit_opcode(masm, 0xDE);
- emit_opcode(masm, 0xC0 + $src2$$reg);
- %}
-
- enc_class subFPR_divFPR_encode( eRegFPR src1, eRegFPR src2) %{
- // Operand has been loaded into fp ST (stack top)
- // FSUB ST,$src1
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xE0 + $src1$$reg);
-
- // FDIV
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xF0 + $src2$$reg);
- %}
-
- enc_class MulFAddF (eRegFPR src1, eRegFPR src2) %{
- // Operand was loaded from memory into fp ST (stack top)
- // FADD ST,$src /* D8 C0+i */
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xC0 + $src1$$reg);
-
- // FMUL ST,src2 /* D8 C*+i */
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xC8 + $src2$$reg);
- %}
-
-
- enc_class MulFAddFreverse (eRegFPR src1, eRegFPR src2) %{
- // Operand was loaded from memory into fp ST (stack top)
- // FADD ST,$src /* D8 C0+i */
- emit_opcode(masm, 0xD8);
- emit_opcode(masm, 0xC0 + $src1$$reg);
-
- // FMULP src2,ST /* DE C8+i */
- emit_opcode(masm, 0xDE);
- emit_opcode(masm, 0xC8 + $src2$$reg);
- %}
-
- // Atomically load the volatile long
- enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{
- emit_opcode(masm,0xDF);
- int rm_byte_opcode = 0x05;
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp;
- relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
- encode_RegMem(masm, rm_byte_opcode, base, index, scale, displace, disp_reloc);
- store_to_stackslot( masm, 0x0DF, 0x07, $dst$$disp );
- %}
-
- // Volatile Store Long. Must be atomic, so move it into
- // the FP TOS and then do a 64-bit FIST. Has to probe the
- // target address before the store (for null-ptr checks)
- // so the memory operand is used twice in the encoding.
- enc_class enc_storeL_volatile( memory mem, stackSlotL src ) %{
- store_to_stackslot( masm, 0x0DF, 0x05, $src$$disp );
- __ set_inst_mark(); // Mark start of FIST in case $mem has an oop
- emit_opcode(masm,0xDF);
- int rm_byte_opcode = 0x07;
- int base = $mem$$base;
- int index = $mem$$index;
- int scale = $mem$$scale;
- int displace = $mem$$disp;
- relocInfo::relocType disp_reloc = $mem->disp_reloc(); // disp-as-oop when working with static globals
- encode_RegMem(masm, rm_byte_opcode, base, index, scale, displace, disp_reloc);
- __ clear_inst_mark();
- %}
-
-%}
-
-
-//----------FRAME--------------------------------------------------------------
-// Definition of frame structure and management information.
-//
-// S T A C K L A Y O U T Allocators stack-slot number
-// | (to get allocators register number
-// G Owned by | | v add OptoReg::stack0())
-// r CALLER | |
-// o | +--------+ pad to even-align allocators stack-slot
-// w V | pad0 | numbers; owned by CALLER
-// t -----------+--------+----> Matcher::_in_arg_limit, unaligned
-// h ^ | in | 5
-// | | args | 4 Holes in incoming args owned by SELF
-// | | | | 3
-// | | +--------+
-// V | | old out| Empty on Intel, window on Sparc
-// | old |preserve| Must be even aligned.
-// | SP-+--------+----> Matcher::_old_SP, even aligned
-// | | in | 3 area for Intel ret address
-// Owned by |preserve| Empty on Sparc.
-// SELF +--------+
-// | | pad2 | 2 pad to align old SP
-// | +--------+ 1
-// | | locks | 0
-// | +--------+----> OptoReg::stack0(), even aligned
-// | | pad1 | 11 pad to align new SP
-// | +--------+
-// | | | 10
-// | | spills | 9 spills
-// V | | 8 (pad0 slot for callee)
-// -----------+--------+----> Matcher::_out_arg_limit, unaligned
-// ^ | out | 7
-// | | args | 6 Holes in outgoing args owned by CALLEE
-// Owned by +--------+
-// CALLEE | new out| 6 Empty on Intel, window on Sparc
-// | new |preserve| Must be even-aligned.
-// | SP-+--------+----> Matcher::_new_SP, even aligned
-// | | |
-//
-// Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
-// known from SELF's arguments and the Java calling convention.
-// Region 6-7 is determined per call site.
-// Note 2: If the calling convention leaves holes in the incoming argument
-// area, those holes are owned by SELF. Holes in the outgoing area
-// are owned by the CALLEE. Holes should not be necessary in the
-// incoming area, as the Java calling convention is completely under
-// the control of the AD file. Doubles can be sorted and packed to
-// avoid holes. Holes in the outgoing arguments may be necessary for
-// varargs C calling conventions.
-// Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
-// even aligned with pad0 as needed.
-// Region 6 is even aligned. Region 6-7 is NOT even aligned;
-// region 6-11 is even aligned; it may be padded out more so that
-// the region from SP to FP meets the minimum stack alignment.
-
-frame %{
- // These three registers define part of the calling convention
- // between compiled code and the interpreter.
- inline_cache_reg(EAX); // Inline Cache Register
-
- // Optional: name the operand used by cisc-spilling to access [stack_pointer + offset]
- cisc_spilling_operand_name(indOffset32);
-
- // Number of stack slots consumed by locking an object
- sync_stack_slots(1);
-
- // Compiled code's Frame Pointer
- frame_pointer(ESP);
- // Interpreter stores its frame pointer in a register which is
- // stored to the stack by I2CAdaptors.
- // I2CAdaptors convert from interpreted java to compiled java.
- interpreter_frame_pointer(EBP);
-
- // Stack alignment requirement
- // Alignment size in bytes (128-bit -> 16 bytes)
- stack_alignment(StackAlignmentInBytes);
-
- // Number of outgoing stack slots killed above the out_preserve_stack_slots
- // for calls to C. Supports the var-args backing area for register parms.
- varargs_C_out_slots_killed(0);
-
- // The after-PROLOG location of the return address. Location of
- // return address specifies a type (REG or STACK) and a number
- // representing the register number (i.e. - use a register name) or
- // stack slot.
- // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
- // Otherwise, it is above the locks and verification slot and alignment word
- return_addr(STACK - 1 +
- align_up((Compile::current()->in_preserve_stack_slots() +
- Compile::current()->fixed_slots()),
- stack_alignment_in_slots()));
-
- // Location of C & interpreter return values
- c_return_value %{
- assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
- static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num, EAX_num, FPR1L_num, FPR1L_num, EAX_num };
- static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
-
- // in SSE2+ mode we want to keep the FPU stack clean so pretend
- // that C functions return float and double results in XMM0.
- if( ideal_reg == Op_RegD && UseSSE>=2 )
- return OptoRegPair(XMM0b_num,XMM0_num);
- if( ideal_reg == Op_RegF && UseSSE>=2 )
- return OptoRegPair(OptoReg::Bad,XMM0_num);
-
- return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
- %}
-
- // Location of return values
- return_value %{
- assert( ideal_reg >= Op_RegI && ideal_reg <= Op_RegL, "only return normal values" );
- static int lo[Op_RegL+1] = { 0, 0, OptoReg::Bad, EAX_num, EAX_num, FPR1L_num, FPR1L_num, EAX_num };
- static int hi[Op_RegL+1] = { 0, 0, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, OptoReg::Bad, FPR1H_num, EDX_num };
- if( ideal_reg == Op_RegD && UseSSE>=2 )
- return OptoRegPair(XMM0b_num,XMM0_num);
- if( ideal_reg == Op_RegF && UseSSE>=1 )
- return OptoRegPair(OptoReg::Bad,XMM0_num);
- return OptoRegPair(hi[ideal_reg],lo[ideal_reg]);
- %}
-
-%}
-
-//----------ATTRIBUTES---------------------------------------------------------
-//----------Operand Attributes-------------------------------------------------
-op_attrib op_cost(0); // Required cost attribute
-
-//----------Instruction Attributes---------------------------------------------
-ins_attrib ins_cost(100); // Required cost attribute
-ins_attrib ins_size(8); // Required size attribute (in bits)
-ins_attrib ins_short_branch(0); // Required flag: is this instruction a
- // non-matching short branch variant of some
- // long branch?
-ins_attrib ins_alignment(1); // Required alignment attribute (must be a power of 2)
- // specifies the alignment that some part of the instruction (not
- // necessarily the start) requires. If > 1, a compute_padding()
- // function must be provided for the instruction
-
-//----------OPERANDS-----------------------------------------------------------
-// Operand definitions must precede instruction definitions for correct parsing
-// in the ADLC because operands constitute user defined types which are used in
-// instruction definitions.
-
-//----------Simple Operands----------------------------------------------------
-// Immediate Operands
-// Integer Immediate
-operand immI() %{
- match(ConI);
-
- op_cost(10);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Constant for test vs zero
-operand immI_0() %{
- predicate(n->get_int() == 0);
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Constant for increment
-operand immI_1() %{
- predicate(n->get_int() == 1);
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Constant for decrement
-operand immI_M1() %{
- predicate(n->get_int() == -1);
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Valid scale values for addressing modes
-operand immI2() %{
- predicate(0 <= n->get_int() && (n->get_int() <= 3));
- match(ConI);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI8() %{
- predicate((-128 <= n->get_int()) && (n->get_int() <= 127));
- match(ConI);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immU8() %{
- predicate((0 <= n->get_int()) && (n->get_int() <= 255));
- match(ConI);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI16() %{
- predicate((-32768 <= n->get_int()) && (n->get_int() <= 32767));
- match(ConI);
-
- op_cost(10);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Int Immediate non-negative
-operand immU31()
-%{
- predicate(n->get_int() >= 0);
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Constant for long shifts
-operand immI_32() %{
- predicate( n->get_int() == 32 );
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_1_31() %{
- predicate( n->get_int() >= 1 && n->get_int() <= 31 );
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_32_63() %{
- predicate( n->get_int() >= 32 && n->get_int() <= 63 );
- match(ConI);
- op_cost(0);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_2() %{
- predicate( n->get_int() == 2 );
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_3() %{
- predicate( n->get_int() == 3 );
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_4()
-%{
- predicate(n->get_int() == 4);
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_8()
-%{
- predicate(n->get_int() == 8);
- match(ConI);
-
- op_cost(0);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Pointer Immediate
-operand immP() %{
- match(ConP);
-
- op_cost(10);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Null Pointer Immediate
-operand immP0() %{
- predicate( n->get_ptr() == 0 );
- match(ConP);
- op_cost(0);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Long Immediate
-operand immL() %{
- match(ConL);
-
- op_cost(20);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Long Immediate zero
-operand immL0() %{
- predicate( n->get_long() == 0L );
- match(ConL);
- op_cost(0);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Long Immediate zero
-operand immL_M1() %{
- predicate( n->get_long() == -1L );
- match(ConL);
- op_cost(0);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Long immediate from 0 to 127.
-// Used for a shorter form of long mul by 10.
-operand immL_127() %{
- predicate((0 <= n->get_long()) && (n->get_long() <= 127));
- match(ConL);
- op_cost(0);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Long Immediate: low 32-bit mask
-operand immL_32bits() %{
- predicate(n->get_long() == 0xFFFFFFFFL);
- match(ConL);
- op_cost(0);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Long Immediate: low 32-bit mask
-operand immL32() %{
- predicate(n->get_long() == (int)(n->get_long()));
- match(ConL);
- op_cost(20);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-//Double Immediate zero
-operand immDPR0() %{
- // Do additional (and counter-intuitive) test against NaN to work around VC++
- // bug that generates code such that NaNs compare equal to 0.0
- predicate( UseSSE<=1 && n->getd() == 0.0 && !g_isnan(n->getd()) );
- match(ConD);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Double Immediate one
-operand immDPR1() %{
- predicate( UseSSE<=1 && n->getd() == 1.0 );
- match(ConD);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Double Immediate
-operand immDPR() %{
- predicate(UseSSE<=1);
- match(ConD);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immD() %{
- predicate(UseSSE>=2);
- match(ConD);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Double Immediate zero
-operand immD0() %{
- // Do additional (and counter-intuitive) test against NaN to work around VC++
- // bug that generates code such that NaNs compare equal to 0.0 AND do not
- // compare equal to -0.0.
- predicate( UseSSE>=2 && jlong_cast(n->getd()) == 0 );
- match(ConD);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Float Immediate zero
-operand immFPR0() %{
- predicate(UseSSE == 0 && n->getf() == 0.0F);
- match(ConF);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Float Immediate one
-operand immFPR1() %{
- predicate(UseSSE == 0 && n->getf() == 1.0F);
- match(ConF);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Float Immediate
-operand immFPR() %{
- predicate( UseSSE == 0 );
- match(ConF);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Float Immediate
-operand immF() %{
- predicate(UseSSE >= 1);
- match(ConF);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Float Immediate zero. Zero and not -0.0
-operand immF0() %{
- predicate( UseSSE >= 1 && jint_cast(n->getf()) == 0 );
- match(ConF);
-
- op_cost(5);
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Immediates for special shifts (sign extend)
-
-// Constants for increment
-operand immI_16() %{
- predicate( n->get_int() == 16 );
- match(ConI);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand immI_24() %{
- predicate( n->get_int() == 24 );
- match(ConI);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Constant for byte-wide masking
-operand immI_255() %{
- predicate( n->get_int() == 255 );
- match(ConI);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-// Constant for short-wide masking
-operand immI_65535() %{
- predicate(n->get_int() == 65535);
- match(ConI);
-
- format %{ %}
- interface(CONST_INTER);
-%}
-
-operand kReg()
-%{
- constraint(ALLOC_IN_RC(vectmask_reg));
- match(RegVectMask);
- format %{%}
- interface(REG_INTER);
-%}
-
-// Register Operands
-// Integer Register
-operand rRegI() %{
- constraint(ALLOC_IN_RC(int_reg));
- match(RegI);
- match(xRegI);
- match(eAXRegI);
- match(eBXRegI);
- match(eCXRegI);
- match(eDXRegI);
- match(eDIRegI);
- match(eSIRegI);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Subset of Integer Register
-operand xRegI(rRegI reg) %{
- constraint(ALLOC_IN_RC(int_x_reg));
- match(reg);
- match(eAXRegI);
- match(eBXRegI);
- match(eCXRegI);
- match(eDXRegI);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Special Registers
-operand eAXRegI(xRegI reg) %{
- constraint(ALLOC_IN_RC(eax_reg));
- match(reg);
- match(rRegI);
-
- format %{ "EAX" %}
- interface(REG_INTER);
-%}
-
-// Special Registers
-operand eBXRegI(xRegI reg) %{
- constraint(ALLOC_IN_RC(ebx_reg));
- match(reg);
- match(rRegI);
-
- format %{ "EBX" %}
- interface(REG_INTER);
-%}
-
-operand eCXRegI(xRegI reg) %{
- constraint(ALLOC_IN_RC(ecx_reg));
- match(reg);
- match(rRegI);
-
- format %{ "ECX" %}
- interface(REG_INTER);
-%}
-
-operand eDXRegI(xRegI reg) %{
- constraint(ALLOC_IN_RC(edx_reg));
- match(reg);
- match(rRegI);
-
- format %{ "EDX" %}
- interface(REG_INTER);
-%}
-
-operand eDIRegI(xRegI reg) %{
- constraint(ALLOC_IN_RC(edi_reg));
- match(reg);
- match(rRegI);
-
- format %{ "EDI" %}
- interface(REG_INTER);
-%}
-
-operand nadxRegI() %{
- constraint(ALLOC_IN_RC(nadx_reg));
- match(RegI);
- match(eBXRegI);
- match(eCXRegI);
- match(eSIRegI);
- match(eDIRegI);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand ncxRegI() %{
- constraint(ALLOC_IN_RC(ncx_reg));
- match(RegI);
- match(eAXRegI);
- match(eDXRegI);
- match(eSIRegI);
- match(eDIRegI);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-// // This operand was used by cmpFastUnlock, but conflicted with 'object' reg
-// //
-operand eSIRegI(xRegI reg) %{
- constraint(ALLOC_IN_RC(esi_reg));
- match(reg);
- match(rRegI);
-
- format %{ "ESI" %}
- interface(REG_INTER);
-%}
-
-// Pointer Register
-operand anyRegP() %{
- constraint(ALLOC_IN_RC(any_reg));
- match(RegP);
- match(eAXRegP);
- match(eBXRegP);
- match(eCXRegP);
- match(eDIRegP);
- match(eRegP);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand eRegP() %{
- constraint(ALLOC_IN_RC(int_reg));
- match(RegP);
- match(eAXRegP);
- match(eBXRegP);
- match(eCXRegP);
- match(eDIRegP);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand rRegP() %{
- constraint(ALLOC_IN_RC(int_reg));
- match(RegP);
- match(eAXRegP);
- match(eBXRegP);
- match(eCXRegP);
- match(eDIRegP);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-// On windows95, EBP is not safe to use for implicit null tests.
-operand eRegP_no_EBP() %{
- constraint(ALLOC_IN_RC(int_reg_no_ebp));
- match(RegP);
- match(eAXRegP);
- match(eBXRegP);
- match(eCXRegP);
- match(eDIRegP);
-
- op_cost(100);
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand pRegP() %{
- constraint(ALLOC_IN_RC(p_reg));
- match(RegP);
- match(eBXRegP);
- match(eDXRegP);
- match(eSIRegP);
- match(eDIRegP);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Special Registers
-// Return a pointer value
-operand eAXRegP(eRegP reg) %{
- constraint(ALLOC_IN_RC(eax_reg));
- match(reg);
- format %{ "EAX" %}
- interface(REG_INTER);
-%}
-
-// Used in AtomicAdd
-operand eBXRegP(eRegP reg) %{
- constraint(ALLOC_IN_RC(ebx_reg));
- match(reg);
- format %{ "EBX" %}
- interface(REG_INTER);
-%}
-
-// Tail-call (interprocedural jump) to interpreter
-operand eCXRegP(eRegP reg) %{
- constraint(ALLOC_IN_RC(ecx_reg));
- match(reg);
- format %{ "ECX" %}
- interface(REG_INTER);
-%}
-
-operand eDXRegP(eRegP reg) %{
- constraint(ALLOC_IN_RC(edx_reg));
- match(reg);
- format %{ "EDX" %}
- interface(REG_INTER);
-%}
-
-operand eSIRegP(eRegP reg) %{
- constraint(ALLOC_IN_RC(esi_reg));
- match(reg);
- format %{ "ESI" %}
- interface(REG_INTER);
-%}
-
-// Used in rep stosw
-operand eDIRegP(eRegP reg) %{
- constraint(ALLOC_IN_RC(edi_reg));
- match(reg);
- format %{ "EDI" %}
- interface(REG_INTER);
-%}
-
-operand eRegL() %{
- constraint(ALLOC_IN_RC(long_reg));
- match(RegL);
- match(eADXRegL);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand eADXRegL( eRegL reg ) %{
- constraint(ALLOC_IN_RC(eadx_reg));
- match(reg);
-
- format %{ "EDX:EAX" %}
- interface(REG_INTER);
-%}
-
-operand eBCXRegL( eRegL reg ) %{
- constraint(ALLOC_IN_RC(ebcx_reg));
- match(reg);
-
- format %{ "EBX:ECX" %}
- interface(REG_INTER);
-%}
-
-operand eBDPRegL( eRegL reg ) %{
- constraint(ALLOC_IN_RC(ebpd_reg));
- match(reg);
-
- format %{ "EBP:EDI" %}
- interface(REG_INTER);
-%}
-// Special case for integer high multiply
-operand eADXRegL_low_only() %{
- constraint(ALLOC_IN_RC(eadx_reg));
- match(RegL);
-
- format %{ "EAX" %}
- interface(REG_INTER);
-%}
-
-// Flags register, used as output of compare instructions
-operand rFlagsReg() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
-
- format %{ "EFLAGS" %}
- interface(REG_INTER);
-%}
-
-// Flags register, used as output of compare instructions
-operand eFlagsReg() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
-
- format %{ "EFLAGS" %}
- interface(REG_INTER);
-%}
-
-// Flags register, used as output of FLOATING POINT compare instructions
-operand eFlagsRegU() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
-
- format %{ "EFLAGS_U" %}
- interface(REG_INTER);
-%}
-
-operand eFlagsRegUCF() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- predicate(false);
-
- format %{ "EFLAGS_U_CF" %}
- interface(REG_INTER);
-%}
-
-// Condition Code Register used by long compare
-operand flagsReg_long_LTGE() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- format %{ "FLAGS_LTGE" %}
- interface(REG_INTER);
-%}
-operand flagsReg_long_EQNE() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- format %{ "FLAGS_EQNE" %}
- interface(REG_INTER);
-%}
-operand flagsReg_long_LEGT() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- format %{ "FLAGS_LEGT" %}
- interface(REG_INTER);
-%}
-
-// Condition Code Register used by unsigned long compare
-operand flagsReg_ulong_LTGE() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- format %{ "FLAGS_U_LTGE" %}
- interface(REG_INTER);
-%}
-operand flagsReg_ulong_EQNE() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- format %{ "FLAGS_U_EQNE" %}
- interface(REG_INTER);
-%}
-operand flagsReg_ulong_LEGT() %{
- constraint(ALLOC_IN_RC(int_flags));
- match(RegFlags);
- format %{ "FLAGS_U_LEGT" %}
- interface(REG_INTER);
-%}
-
-// Float register operands
-operand regDPR() %{
- predicate( UseSSE < 2 );
- constraint(ALLOC_IN_RC(fp_dbl_reg));
- match(RegD);
- match(regDPR1);
- match(regDPR2);
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand regDPR1(regDPR reg) %{
- predicate( UseSSE < 2 );
- constraint(ALLOC_IN_RC(fp_dbl_reg0));
- match(reg);
- format %{ "FPR1" %}
- interface(REG_INTER);
-%}
-
-operand regDPR2(regDPR reg) %{
- predicate( UseSSE < 2 );
- constraint(ALLOC_IN_RC(fp_dbl_reg1));
- match(reg);
- format %{ "FPR2" %}
- interface(REG_INTER);
-%}
-
-operand regnotDPR1(regDPR reg) %{
- predicate( UseSSE < 2 );
- constraint(ALLOC_IN_RC(fp_dbl_notreg0));
- match(reg);
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Float register operands
-operand regFPR() %{
- predicate( UseSSE < 2 );
- constraint(ALLOC_IN_RC(fp_flt_reg));
- match(RegF);
- match(regFPR1);
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Float register operands
-operand regFPR1(regFPR reg) %{
- predicate( UseSSE < 2 );
- constraint(ALLOC_IN_RC(fp_flt_reg0));
- match(reg);
- format %{ "FPR1" %}
- interface(REG_INTER);
-%}
-
-// XMM Float register operands
-operand regF() %{
- predicate( UseSSE>=1 );
- constraint(ALLOC_IN_RC(float_reg_legacy));
- match(RegF);
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand legRegF() %{
- predicate( UseSSE>=1 );
- constraint(ALLOC_IN_RC(float_reg_legacy));
- match(RegF);
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Float register operands
-operand vlRegF() %{
- constraint(ALLOC_IN_RC(float_reg_vl));
- match(RegF);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-// XMM Double register operands
-operand regD() %{
- predicate( UseSSE>=2 );
- constraint(ALLOC_IN_RC(double_reg_legacy));
- match(RegD);
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Double register operands
-operand legRegD() %{
- predicate( UseSSE>=2 );
- constraint(ALLOC_IN_RC(double_reg_legacy));
- match(RegD);
- format %{ %}
- interface(REG_INTER);
-%}
-
-operand vlRegD() %{
- constraint(ALLOC_IN_RC(double_reg_vl));
- match(RegD);
-
- format %{ %}
- interface(REG_INTER);
-%}
-
-//----------Memory Operands----------------------------------------------------
-// Direct Memory Operand
-operand direct(immP addr) %{
- match(addr);
-
- format %{ "[$addr]" %}
- interface(MEMORY_INTER) %{
- base(0xFFFFFFFF);
- index(0x4);
- scale(0x0);
- disp($addr);
- %}
-%}
-
-// Indirect Memory Operand
-operand indirect(eRegP reg) %{
- constraint(ALLOC_IN_RC(int_reg));
- match(reg);
-
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index(0x4);
- scale(0x0);
- disp(0x0);
- %}
-%}
-
-// Indirect Memory Plus Short Offset Operand
-operand indOffset8(eRegP reg, immI8 off) %{
- match(AddP reg off);
-
- format %{ "[$reg + $off]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index(0x4);
- scale(0x0);
- disp($off);
- %}
-%}
-
-// Indirect Memory Plus Long Offset Operand
-operand indOffset32(eRegP reg, immI off) %{
- match(AddP reg off);
-
- format %{ "[$reg + $off]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index(0x4);
- scale(0x0);
- disp($off);
- %}
-%}
-
-// Indirect Memory Plus Long Offset Operand
-operand indOffset32X(rRegI reg, immP off) %{
- match(AddP off reg);
-
- format %{ "[$reg + $off]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index(0x4);
- scale(0x0);
- disp($off);
- %}
-%}
-
-// Indirect Memory Plus Index Register Plus Offset Operand
-operand indIndexOffset(eRegP reg, rRegI ireg, immI off) %{
- match(AddP (AddP reg ireg) off);
-
- op_cost(10);
- format %{"[$reg + $off + $ireg]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index($ireg);
- scale(0x0);
- disp($off);
- %}
-%}
-
-// Indirect Memory Plus Index Register Plus Offset Operand
-operand indIndex(eRegP reg, rRegI ireg) %{
- match(AddP reg ireg);
-
- op_cost(10);
- format %{"[$reg + $ireg]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index($ireg);
- scale(0x0);
- disp(0x0);
- %}
-%}
-
-// // -------------------------------------------------------------------------
-// // 486 architecture doesn't support "scale * index + offset" with out a base
-// // -------------------------------------------------------------------------
-// // Scaled Memory Operands
-// // Indirect Memory Times Scale Plus Offset Operand
-// operand indScaleOffset(immP off, rRegI ireg, immI2 scale) %{
-// match(AddP off (LShiftI ireg scale));
-//
-// op_cost(10);
-// format %{"[$off + $ireg << $scale]" %}
-// interface(MEMORY_INTER) %{
-// base(0x4);
-// index($ireg);
-// scale($scale);
-// disp($off);
-// %}
-// %}
-
-// Indirect Memory Times Scale Plus Index Register
-operand indIndexScale(eRegP reg, rRegI ireg, immI2 scale) %{
- match(AddP reg (LShiftI ireg scale));
-
- op_cost(10);
- format %{"[$reg + $ireg << $scale]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index($ireg);
- scale($scale);
- disp(0x0);
- %}
-%}
-
-// Indirect Memory Times Scale Plus Index Register Plus Offset Operand
-operand indIndexScaleOffset(eRegP reg, immI off, rRegI ireg, immI2 scale) %{
- match(AddP (AddP reg (LShiftI ireg scale)) off);
-
- op_cost(10);
- format %{"[$reg + $off + $ireg << $scale]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index($ireg);
- scale($scale);
- disp($off);
- %}
-%}
-
-//----------Load Long Memory Operands------------------------------------------
-// The load-long idiom will use it's address expression again after loading
-// the first word of the long. If the load-long destination overlaps with
-// registers used in the addressing expression, the 2nd half will be loaded
-// from a clobbered address. Fix this by requiring that load-long use
-// address registers that do not overlap with the load-long target.
-
-// load-long support
-operand load_long_RegP() %{
- constraint(ALLOC_IN_RC(esi_reg));
- match(RegP);
- match(eSIRegP);
- op_cost(100);
- format %{ %}
- interface(REG_INTER);
-%}
-
-// Indirect Memory Operand Long
-operand load_long_indirect(load_long_RegP reg) %{
- constraint(ALLOC_IN_RC(esi_reg));
- match(reg);
-
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index(0x4);
- scale(0x0);
- disp(0x0);
- %}
-%}
-
-// Indirect Memory Plus Long Offset Operand
-operand load_long_indOffset32(load_long_RegP reg, immI off) %{
- match(AddP reg off);
-
- format %{ "[$reg + $off]" %}
- interface(MEMORY_INTER) %{
- base($reg);
- index(0x4);
- scale(0x0);
- disp($off);
- %}
-%}
-
-opclass load_long_memory(load_long_indirect, load_long_indOffset32);
-
-
-//----------Special Memory Operands--------------------------------------------
-// Stack Slot Operand - This operand is used for loading and storing temporary
-// values on the stack where a match requires a value to
-// flow through memory.
-operand stackSlotP(sRegP reg) %{
- constraint(ALLOC_IN_RC(stack_slots));
- // No match rule because this operand is only generated in matching
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base(0x4); // ESP
- index(0x4); // No Index
- scale(0x0); // No Scale
- disp($reg); // Stack Offset
- %}
-%}
-
-operand stackSlotI(sRegI reg) %{
- constraint(ALLOC_IN_RC(stack_slots));
- // No match rule because this operand is only generated in matching
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base(0x4); // ESP
- index(0x4); // No Index
- scale(0x0); // No Scale
- disp($reg); // Stack Offset
- %}
-%}
-
-operand stackSlotF(sRegF reg) %{
- constraint(ALLOC_IN_RC(stack_slots));
- // No match rule because this operand is only generated in matching
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base(0x4); // ESP
- index(0x4); // No Index
- scale(0x0); // No Scale
- disp($reg); // Stack Offset
- %}
-%}
-
-operand stackSlotD(sRegD reg) %{
- constraint(ALLOC_IN_RC(stack_slots));
- // No match rule because this operand is only generated in matching
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base(0x4); // ESP
- index(0x4); // No Index
- scale(0x0); // No Scale
- disp($reg); // Stack Offset
- %}
-%}
-
-operand stackSlotL(sRegL reg) %{
- constraint(ALLOC_IN_RC(stack_slots));
- // No match rule because this operand is only generated in matching
- format %{ "[$reg]" %}
- interface(MEMORY_INTER) %{
- base(0x4); // ESP
- index(0x4); // No Index
- scale(0x0); // No Scale
- disp($reg); // Stack Offset
- %}
-%}
-
-//----------Conditional Branch Operands----------------------------------------
-// Comparison Op - This is the operation of the comparison, and is limited to
-// the following set of codes:
-// L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
-//
-// Other attributes of the comparison, such as unsignedness, are specified
-// by the comparison instruction that sets a condition code flags register.
-// That result is represented by a flags operand whose subtype is appropriate
-// to the unsignedness (etc.) of the comparison.
-//
-// Later, the instruction which matches both the Comparison Op (a Bool) and
-// the flags (produced by the Cmp) specifies the coding of the comparison op
-// by matching a specific subtype of Bool operand below, such as cmpOpU.
-
-// Comparison Code
-operand cmpOp() %{
- match(Bool);
-
- format %{ "" %}
- interface(COND_INTER) %{
- equal(0x4, "e");
- not_equal(0x5, "ne");
- less(0xC, "l");
- greater_equal(0xD, "ge");
- less_equal(0xE, "le");
- greater(0xF, "g");
- overflow(0x0, "o");
- no_overflow(0x1, "no");
- %}
-%}
-
-// Comparison Code, unsigned compare. Used by FP also, with
-// C2 (unordered) turned into GT or LT already. The other bits
-// C0 and C3 are turned into Carry & Zero flags.
-operand cmpOpU() %{
- match(Bool);
-
- format %{ "" %}
- interface(COND_INTER) %{
- equal(0x4, "e");
- not_equal(0x5, "ne");
- less(0x2, "b");
- greater_equal(0x3, "nb");
- less_equal(0x6, "be");
- greater(0x7, "nbe");
- overflow(0x0, "o");
- no_overflow(0x1, "no");
- %}
-%}
-
-// Floating comparisons that don't require any fixup for the unordered case
-operand cmpOpUCF() %{
- match(Bool);
- predicate(n->as_Bool()->_test._test == BoolTest::lt ||
- n->as_Bool()->_test._test == BoolTest::ge ||
- n->as_Bool()->_test._test == BoolTest::le ||
- n->as_Bool()->_test._test == BoolTest::gt);
- format %{ "" %}
- interface(COND_INTER) %{
- equal(0x4, "e");
- not_equal(0x5, "ne");
- less(0x2, "b");
- greater_equal(0x3, "nb");
- less_equal(0x6, "be");
- greater(0x7, "nbe");
- overflow(0x0, "o");
- no_overflow(0x1, "no");
- %}
-%}
-
-
-// Floating comparisons that can be fixed up with extra conditional jumps
-operand cmpOpUCF2() %{
- match(Bool);
- predicate(n->as_Bool()->_test._test == BoolTest::ne ||
- n->as_Bool()->_test._test == BoolTest::eq);
- format %{ "" %}
- interface(COND_INTER) %{
- equal(0x4, "e");
- not_equal(0x5, "ne");
- less(0x2, "b");
- greater_equal(0x3, "nb");
- less_equal(0x6, "be");
- greater(0x7, "nbe");
- overflow(0x0, "o");
- no_overflow(0x1, "no");
- %}
-%}
-
-// Comparison Code for FP conditional move
-operand cmpOp_fcmov() %{
- match(Bool);
-
- predicate(n->as_Bool()->_test._test != BoolTest::overflow &&
- n->as_Bool()->_test._test != BoolTest::no_overflow);
- format %{ "" %}
- interface(COND_INTER) %{
- equal (0x0C8);
- not_equal (0x1C8);
- less (0x0C0);
- greater_equal(0x1C0);
- less_equal (0x0D0);
- greater (0x1D0);
- overflow(0x0, "o"); // not really supported by the instruction
- no_overflow(0x1, "no"); // not really supported by the instruction
- %}
-%}
-
-// Comparison Code used in long compares
-operand cmpOp_commute() %{
- match(Bool);
-
- format %{ "" %}
- interface(COND_INTER) %{
- equal(0x4, "e");
- not_equal(0x5, "ne");
- less(0xF, "g");
- greater_equal(0xE, "le");
- less_equal(0xD, "ge");
- greater(0xC, "l");
- overflow(0x0, "o");
- no_overflow(0x1, "no");
- %}
-%}
-
-// Comparison Code used in unsigned long compares
-operand cmpOpU_commute() %{
- match(Bool);
-
- format %{ "" %}
- interface(COND_INTER) %{
- equal(0x4, "e");
- not_equal(0x5, "ne");
- less(0x7, "nbe");
- greater_equal(0x6, "be");
- less_equal(0x3, "nb");
- greater(0x2, "b");
- overflow(0x0, "o");
- no_overflow(0x1, "no");
- %}
-%}
-
-//----------OPERAND CLASSES----------------------------------------------------
-// Operand Classes are groups of operands that are used as to simplify
-// instruction definitions by not requiring the AD writer to specify separate
-// instructions for every form of operand when the instruction accepts
-// multiple operand types with the same basic encoding and format. The classic
-// case of this is memory operands.
-
-opclass memory(direct, indirect, indOffset8, indOffset32, indOffset32X, indIndexOffset,
- indIndex, indIndexScale, indIndexScaleOffset);
-
-// Long memory operations are encoded in 2 instructions and a +4 offset.
-// This means some kind of offset is always required and you cannot use
-// an oop as the offset (done when working on static globals).
-opclass long_memory(direct, indirect, indOffset8, indOffset32, indIndexOffset,
- indIndex, indIndexScale, indIndexScaleOffset);
-
-
-//----------PIPELINE-----------------------------------------------------------
-// Rules which define the behavior of the target architectures pipeline.
-pipeline %{
-
-//----------ATTRIBUTES---------------------------------------------------------
-attributes %{
- variable_size_instructions; // Fixed size instructions
- max_instructions_per_bundle = 3; // Up to 3 instructions per bundle
- instruction_unit_size = 1; // An instruction is 1 bytes long
- instruction_fetch_unit_size = 16; // The processor fetches one line
- instruction_fetch_units = 1; // of 16 bytes
-
- // List of nop instructions
- nops( MachNop );
-%}
-
-//----------RESOURCES----------------------------------------------------------
-// Resources are the functional units available to the machine
-
-// Generic P2/P3 pipeline
-// 3 decoders, only D0 handles big operands; a "bundle" is the limit of
-// 3 instructions decoded per cycle.
-// 2 load/store ops per cycle, 1 branch, 1 FPU,
-// 2 ALU op, only ALU0 handles mul/div instructions.
-resources( D0, D1, D2, DECODE = D0 | D1 | D2,
- MS0, MS1, MEM = MS0 | MS1,
- BR, FPU,
- ALU0, ALU1, ALU = ALU0 | ALU1 );
-
-//----------PIPELINE DESCRIPTION-----------------------------------------------
-// Pipeline Description specifies the stages in the machine's pipeline
-
-// Generic P2/P3 pipeline
-pipe_desc(S0, S1, S2, S3, S4, S5);
-
-//----------PIPELINE CLASSES---------------------------------------------------
-// Pipeline Classes describe the stages in which input and output are
-// referenced by the hardware pipeline.
-
-// Naming convention: ialu or fpu
-// Then: _reg
-// Then: _reg if there is a 2nd register
-// Then: _long if it's a pair of instructions implementing a long
-// Then: _fat if it requires the big decoder
-// Or: _mem if it requires the big decoder and a memory unit.
-
-// Integer ALU reg operation
-pipe_class ialu_reg(rRegI dst) %{
- single_instruction;
- dst : S4(write);
- dst : S3(read);
- DECODE : S0; // any decoder
- ALU : S3; // any alu
-%}
-
-// Long ALU reg operation
-pipe_class ialu_reg_long(eRegL dst) %{
- instruction_count(2);
- dst : S4(write);
- dst : S3(read);
- DECODE : S0(2); // any 2 decoders
- ALU : S3(2); // both alus
-%}
-
-// Integer ALU reg operation using big decoder
-pipe_class ialu_reg_fat(rRegI dst) %{
- single_instruction;
- dst : S4(write);
- dst : S3(read);
- D0 : S0; // big decoder only
- ALU : S3; // any alu
-%}
-
-// Long ALU reg operation using big decoder
-pipe_class ialu_reg_long_fat(eRegL dst) %{
- instruction_count(2);
- dst : S4(write);
- dst : S3(read);
- D0 : S0(2); // big decoder only; twice
- ALU : S3(2); // any 2 alus
-%}
-
-// Integer ALU reg-reg operation
-pipe_class ialu_reg_reg(rRegI dst, rRegI src) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- DECODE : S0; // any decoder
- ALU : S3; // any alu
-%}
-
-// Long ALU reg-reg operation
-pipe_class ialu_reg_reg_long(eRegL dst, eRegL src) %{
- instruction_count(2);
- dst : S4(write);
- src : S3(read);
- DECODE : S0(2); // any 2 decoders
- ALU : S3(2); // both alus
-%}
-
-// Integer ALU reg-reg operation
-pipe_class ialu_reg_reg_fat(rRegI dst, memory src) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- D0 : S0; // big decoder only
- ALU : S3; // any alu
-%}
-
-// Long ALU reg-reg operation
-pipe_class ialu_reg_reg_long_fat(eRegL dst, eRegL src) %{
- instruction_count(2);
- dst : S4(write);
- src : S3(read);
- D0 : S0(2); // big decoder only; twice
- ALU : S3(2); // both alus
-%}
-
-// Integer ALU reg-mem operation
-pipe_class ialu_reg_mem(rRegI dst, memory mem) %{
- single_instruction;
- dst : S5(write);
- mem : S3(read);
- D0 : S0; // big decoder only
- ALU : S4; // any alu
- MEM : S3; // any mem
-%}
-
-// Long ALU reg-mem operation
-pipe_class ialu_reg_long_mem(eRegL dst, load_long_memory mem) %{
- instruction_count(2);
- dst : S5(write);
- mem : S3(read);
- D0 : S0(2); // big decoder only; twice
- ALU : S4(2); // any 2 alus
- MEM : S3(2); // both mems
-%}
-
-// Integer mem operation (prefetch)
-pipe_class ialu_mem(memory mem)
-%{
- single_instruction;
- mem : S3(read);
- D0 : S0; // big decoder only
- MEM : S3; // any mem
-%}
-
-// Integer Store to Memory
-pipe_class ialu_mem_reg(memory mem, rRegI src) %{
- single_instruction;
- mem : S3(read);
- src : S5(read);
- D0 : S0; // big decoder only
- ALU : S4; // any alu
- MEM : S3;
-%}
-
-// Long Store to Memory
-pipe_class ialu_mem_long_reg(memory mem, eRegL src) %{
- instruction_count(2);
- mem : S3(read);
- src : S5(read);
- D0 : S0(2); // big decoder only; twice
- ALU : S4(2); // any 2 alus
- MEM : S3(2); // Both mems
-%}
-
-// Integer Store to Memory
-pipe_class ialu_mem_imm(memory mem) %{
- single_instruction;
- mem : S3(read);
- D0 : S0; // big decoder only
- ALU : S4; // any alu
- MEM : S3;
-%}
-
-// Integer ALU0 reg-reg operation
-pipe_class ialu_reg_reg_alu0(rRegI dst, rRegI src) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- D0 : S0; // Big decoder only
- ALU0 : S3; // only alu0
-%}
-
-// Integer ALU0 reg-mem operation
-pipe_class ialu_reg_mem_alu0(rRegI dst, memory mem) %{
- single_instruction;
- dst : S5(write);
- mem : S3(read);
- D0 : S0; // big decoder only
- ALU0 : S4; // ALU0 only
- MEM : S3; // any mem
-%}
-
-// Integer ALU reg-reg operation
-pipe_class ialu_cr_reg_reg(eFlagsReg cr, rRegI src1, rRegI src2) %{
- single_instruction;
- cr : S4(write);
- src1 : S3(read);
- src2 : S3(read);
- DECODE : S0; // any decoder
- ALU : S3; // any alu
-%}
-
-// Integer ALU reg-imm operation
-pipe_class ialu_cr_reg_imm(eFlagsReg cr, rRegI src1) %{
- single_instruction;
- cr : S4(write);
- src1 : S3(read);
- DECODE : S0; // any decoder
- ALU : S3; // any alu
-%}
-
-// Integer ALU reg-mem operation
-pipe_class ialu_cr_reg_mem(eFlagsReg cr, rRegI src1, memory src2) %{
- single_instruction;
- cr : S4(write);
- src1 : S3(read);
- src2 : S3(read);
- D0 : S0; // big decoder only
- ALU : S4; // any alu
- MEM : S3;
-%}
-
-// Conditional move reg-reg
-pipe_class pipe_cmplt( rRegI p, rRegI q, rRegI y ) %{
- instruction_count(4);
- y : S4(read);
- q : S3(read);
- p : S3(read);
- DECODE : S0(4); // any decoder
-%}
-
-// Conditional move reg-reg
-pipe_class pipe_cmov_reg( rRegI dst, rRegI src, eFlagsReg cr ) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- cr : S3(read);
- DECODE : S0; // any decoder
-%}
-
-// Conditional move reg-mem
-pipe_class pipe_cmov_mem( eFlagsReg cr, rRegI dst, memory src) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- cr : S3(read);
- DECODE : S0; // any decoder
- MEM : S3;
-%}
-
-// Conditional move reg-reg long
-pipe_class pipe_cmov_reg_long( eFlagsReg cr, eRegL dst, eRegL src) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- cr : S3(read);
- DECODE : S0(2); // any 2 decoders
-%}
-
-// Conditional move double reg-reg
-pipe_class pipe_cmovDPR_reg( eFlagsReg cr, regDPR1 dst, regDPR src) %{
- single_instruction;
- dst : S4(write);
- src : S3(read);
- cr : S3(read);
- DECODE : S0; // any decoder
-%}
-
-// Float reg-reg operation
-pipe_class fpu_reg(regDPR dst) %{
- instruction_count(2);
- dst : S3(read);
- DECODE : S0(2); // any 2 decoders
- FPU : S3;
-%}
-
-// Float reg-reg operation
-pipe_class fpu_reg_reg(regDPR dst, regDPR src) %{
- instruction_count(2);
- dst : S4(write);
- src : S3(read);
- DECODE : S0(2); // any 2 decoders
- FPU : S3;
-%}
-
-// Float reg-reg operation
-pipe_class fpu_reg_reg_reg(regDPR dst, regDPR src1, regDPR src2) %{
- instruction_count(3);
- dst : S4(write);
- src1 : S3(read);
- src2 : S3(read);
- DECODE : S0(3); // any 3 decoders
- FPU : S3(2);
-%}
-
-// Float reg-reg operation
-pipe_class fpu_reg_reg_reg_reg(regDPR dst, regDPR src1, regDPR src2, regDPR src3) %{
- instruction_count(4);
- dst : S4(write);
- src1 : S3(read);
- src2 : S3(read);
- src3 : S3(read);
- DECODE : S0(4); // any 3 decoders
- FPU : S3(2);
-%}
-
-// Float reg-reg operation
-pipe_class fpu_reg_mem_reg_reg(regDPR dst, memory src1, regDPR src2, regDPR src3) %{
- instruction_count(4);
- dst : S4(write);
- src1 : S3(read);
- src2 : S3(read);
- src3 : S3(read);
- DECODE : S1(3); // any 3 decoders
- D0 : S0; // Big decoder only
- FPU : S3(2);
- MEM : S3;
-%}
-
-// Float reg-mem operation
-pipe_class fpu_reg_mem(regDPR dst, memory mem) %{
- instruction_count(2);
- dst : S5(write);
- mem : S3(read);
- D0 : S0; // big decoder only
- DECODE : S1; // any decoder for FPU POP
- FPU : S4;
- MEM : S3; // any mem
-%}
-
-// Float reg-mem operation
-pipe_class fpu_reg_reg_mem(regDPR dst, regDPR src1, memory mem) %{
- instruction_count(3);
- dst : S5(write);
- src1 : S3(read);
- mem : S3(read);
- D0 : S0; // big decoder only
- DECODE : S1(2); // any decoder for FPU POP
- FPU : S4;
- MEM : S3; // any mem
-%}
-
-// Float mem-reg operation
-pipe_class fpu_mem_reg(memory mem, regDPR src) %{
- instruction_count(2);
- src : S5(read);
- mem : S3(read);
- DECODE : S0; // any decoder for FPU PUSH
- D0 : S1; // big decoder only
- FPU : S4;
- MEM : S3; // any mem
-%}
-
-pipe_class fpu_mem_reg_reg(memory mem, regDPR src1, regDPR src2) %{
- instruction_count(3);
- src1 : S3(read);
- src2 : S3(read);
- mem : S3(read);
- DECODE : S0(2); // any decoder for FPU PUSH
- D0 : S1; // big decoder only
- FPU : S4;
- MEM : S3; // any mem
-%}
-
-pipe_class fpu_mem_reg_mem(memory mem, regDPR src1, memory src2) %{
- instruction_count(3);
- src1 : S3(read);
- src2 : S3(read);
- mem : S4(read);
- DECODE : S0; // any decoder for FPU PUSH
- D0 : S0(2); // big decoder only
- FPU : S4;
- MEM : S3(2); // any mem
-%}
-
-pipe_class fpu_mem_mem(memory dst, memory src1) %{
- instruction_count(2);
- src1 : S3(read);
- dst : S4(read);
- D0 : S0(2); // big decoder only
- MEM : S3(2); // any mem
-%}
-
-pipe_class fpu_mem_mem_mem(memory dst, memory src1, memory src2) %{
- instruction_count(3);
- src1 : S3(read);
- src2 : S3(read);
- dst : S4(read);
- D0 : S0(3); // big decoder only
- FPU : S4;
- MEM : S3(3); // any mem
-%}
-
-pipe_class fpu_mem_reg_con(memory mem, regDPR src1) %{
- instruction_count(3);
- src1 : S4(read);
- mem : S4(read);
- DECODE : S0; // any decoder for FPU PUSH
- D0 : S0(2); // big decoder only
- FPU : S4;
- MEM : S3(2); // any mem
-%}
-
-// Float load constant
-pipe_class fpu_reg_con(regDPR dst) %{
- instruction_count(2);
- dst : S5(write);
- D0 : S0; // big decoder only for the load
- DECODE : S1; // any decoder for FPU POP
- FPU : S4;
- MEM : S3; // any mem
-%}
-
-// Float load constant
-pipe_class fpu_reg_reg_con(regDPR dst, regDPR src) %{
- instruction_count(3);
- dst : S5(write);
- src : S3(read);
- D0 : S0; // big decoder only for the load
- DECODE : S1(2); // any decoder for FPU POP
- FPU : S4;
- MEM : S3; // any mem
-%}
-
-// UnConditional branch
-pipe_class pipe_jmp( label labl ) %{
- single_instruction;
- BR : S3;
-%}
-
-// Conditional branch
-pipe_class pipe_jcc( cmpOp cmp, eFlagsReg cr, label labl ) %{
- single_instruction;
- cr : S1(read);
- BR : S3;
-%}
-
-// Allocation idiom
-pipe_class pipe_cmpxchg( eRegP dst, eRegP heap_ptr ) %{
- instruction_count(1); force_serialization;
- fixed_latency(6);
- heap_ptr : S3(read);
- DECODE : S0(3);
- D0 : S2;
- MEM : S3;
- ALU : S3(2);
- dst : S5(write);
- BR : S5;
-%}
-
-// Generic big/slow expanded idiom
-pipe_class pipe_slow( ) %{
- instruction_count(10); multiple_bundles; force_serialization;
- fixed_latency(100);
- D0 : S0(2);
- MEM : S3(2);
-%}
-
-// The real do-nothing guy
-pipe_class empty( ) %{
- instruction_count(0);
-%}
-
-// Define the class for the Nop node
-define %{
- MachNop = empty;
-%}
-
-%}
-
-//----------INSTRUCTIONS-------------------------------------------------------
-//
-// match -- States which machine-independent subtree may be replaced
-// by this instruction.
-// ins_cost -- The estimated cost of this instruction is used by instruction
-// selection to identify a minimum cost tree of machine
-// instructions that matches a tree of machine-independent
-// instructions.
-// format -- A string providing the disassembly for this instruction.
-// The value of an instruction's operand may be inserted
-// by referring to it with a '$' prefix.
-// opcode -- Three instruction opcodes may be provided. These are referred
-// to within an encode class as $primary, $secondary, and $tertiary
-// respectively. The primary opcode is commonly used to
-// indicate the type of machine instruction, while secondary
-// and tertiary are often used for prefix options or addressing
-// modes.
-// ins_encode -- A list of encode classes with parameters. The encode class
-// name must have been defined in an 'enc_class' specification
-// in the encode section of the architecture description.
-
-// Dummy reg-to-reg vector moves. Removed during post-selection cleanup.
-// Load Float
-instruct MoveF2LEG(legRegF dst, regF src) %{
- match(Set dst src);
- format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveLEG2F(regF dst, legRegF src) %{
- match(Set dst src);
- format %{ "movss $dst,$src\t# if src != dst load float (4 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveF2VL(vlRegF dst, regF src) %{
- match(Set dst src);
- format %{ "movss $dst,$src\t! load float (4 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-// Load Float
-instruct MoveVL2F(regF dst, vlRegF src) %{
- match(Set dst src);
- format %{ "movss $dst,$src\t! load float (4 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-
-
-// Load Double
-instruct MoveD2LEG(legRegD dst, regD src) %{
- match(Set dst src);
- format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveLEG2D(regD dst, legRegD src) %{
- match(Set dst src);
- format %{ "movsd $dst,$src\t# if src != dst load double (8 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveD2VL(vlRegD dst, regD src) %{
- match(Set dst src);
- format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-// Load Double
-instruct MoveVL2D(regD dst, vlRegD src) %{
- match(Set dst src);
- format %{ "movsd $dst,$src\t! load double (8 bytes)" %}
- ins_encode %{
- ShouldNotReachHere();
- %}
- ins_pipe( fpu_reg_reg );
-%}
-
-//----------BSWAP-Instruction--------------------------------------------------
-instruct bytes_reverse_int(rRegI dst) %{
- match(Set dst (ReverseBytesI dst));
-
- format %{ "BSWAP $dst" %}
- opcode(0x0F, 0xC8);
- ins_encode( OpcP, OpcSReg(dst) );
- ins_pipe( ialu_reg );
-%}
-
-instruct bytes_reverse_long(eRegL dst) %{
- match(Set dst (ReverseBytesL dst));
-
- format %{ "BSWAP $dst.lo\n\t"
- "BSWAP $dst.hi\n\t"
- "XCHG $dst.lo $dst.hi" %}
-
- ins_cost(125);
- ins_encode( bswap_long_bytes(dst) );
- ins_pipe( ialu_reg_reg);
-%}
-
-instruct bytes_reverse_unsigned_short(rRegI dst, eFlagsReg cr) %{
- match(Set dst (ReverseBytesUS dst));
- effect(KILL cr);
-
- format %{ "BSWAP $dst\n\t"
- "SHR $dst,16\n\t" %}
- ins_encode %{
- __ bswapl($dst$$Register);
- __ shrl($dst$$Register, 16);
- %}
- ins_pipe( ialu_reg );
-%}
-
-instruct bytes_reverse_short(rRegI dst, eFlagsReg cr) %{
- match(Set dst (ReverseBytesS dst));
- effect(KILL cr);
-
- format %{ "BSWAP $dst\n\t"
- "SAR $dst,16\n\t" %}
- ins_encode %{
- __ bswapl($dst$$Register);
- __ sarl($dst$$Register, 16);
- %}
- ins_pipe( ialu_reg );
-%}
-
-
-//---------- Zeros Count Instructions ------------------------------------------
-
-instruct countLeadingZerosI(rRegI dst, rRegI src, eFlagsReg cr) %{
- predicate(UseCountLeadingZerosInstruction);
- match(Set dst (CountLeadingZerosI src));
- effect(KILL cr);
-
- format %{ "LZCNT $dst, $src\t# count leading zeros (int)" %}
- ins_encode %{
- __ lzcntl($dst$$Register, $src$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countLeadingZerosI_bsr(rRegI dst, rRegI src, eFlagsReg cr) %{
- predicate(!UseCountLeadingZerosInstruction);
- match(Set dst (CountLeadingZerosI src));
- effect(KILL cr);
-
- format %{ "BSR $dst, $src\t# count leading zeros (int)\n\t"
- "JNZ skip\n\t"
- "MOV $dst, -1\n"
- "skip:\n\t"
- "NEG $dst\n\t"
- "ADD $dst, 31" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- Label skip;
- __ bsrl(Rdst, Rsrc);
- __ jccb(Assembler::notZero, skip);
- __ movl(Rdst, -1);
- __ bind(skip);
- __ negl(Rdst);
- __ addl(Rdst, BitsPerInt - 1);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countLeadingZerosL(rRegI dst, eRegL src, eFlagsReg cr) %{
- predicate(UseCountLeadingZerosInstruction);
- match(Set dst (CountLeadingZerosL src));
- effect(TEMP dst, KILL cr);
-
- format %{ "LZCNT $dst, $src.hi\t# count leading zeros (long)\n\t"
- "JNC done\n\t"
- "LZCNT $dst, $src.lo\n\t"
- "ADD $dst, 32\n"
- "done:" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- Label done;
- __ lzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
- __ jccb(Assembler::carryClear, done);
- __ lzcntl(Rdst, Rsrc);
- __ addl(Rdst, BitsPerInt);
- __ bind(done);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countLeadingZerosL_bsr(rRegI dst, eRegL src, eFlagsReg cr) %{
- predicate(!UseCountLeadingZerosInstruction);
- match(Set dst (CountLeadingZerosL src));
- effect(TEMP dst, KILL cr);
-
- format %{ "BSR $dst, $src.hi\t# count leading zeros (long)\n\t"
- "JZ msw_is_zero\n\t"
- "ADD $dst, 32\n\t"
- "JMP not_zero\n"
- "msw_is_zero:\n\t"
- "BSR $dst, $src.lo\n\t"
- "JNZ not_zero\n\t"
- "MOV $dst, -1\n"
- "not_zero:\n\t"
- "NEG $dst\n\t"
- "ADD $dst, 63\n" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- Label msw_is_zero;
- Label not_zero;
- __ bsrl(Rdst, HIGH_FROM_LOW(Rsrc));
- __ jccb(Assembler::zero, msw_is_zero);
- __ addl(Rdst, BitsPerInt);
- __ jmpb(not_zero);
- __ bind(msw_is_zero);
- __ bsrl(Rdst, Rsrc);
- __ jccb(Assembler::notZero, not_zero);
- __ movl(Rdst, -1);
- __ bind(not_zero);
- __ negl(Rdst);
- __ addl(Rdst, BitsPerLong - 1);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countTrailingZerosI(rRegI dst, rRegI src, eFlagsReg cr) %{
- predicate(UseCountTrailingZerosInstruction);
- match(Set dst (CountTrailingZerosI src));
- effect(KILL cr);
-
- format %{ "TZCNT $dst, $src\t# count trailing zeros (int)" %}
- ins_encode %{
- __ tzcntl($dst$$Register, $src$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countTrailingZerosI_bsf(rRegI dst, rRegI src, eFlagsReg cr) %{
- predicate(!UseCountTrailingZerosInstruction);
- match(Set dst (CountTrailingZerosI src));
- effect(KILL cr);
-
- format %{ "BSF $dst, $src\t# count trailing zeros (int)\n\t"
- "JNZ done\n\t"
- "MOV $dst, 32\n"
- "done:" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- Label done;
- __ bsfl(Rdst, $src$$Register);
- __ jccb(Assembler::notZero, done);
- __ movl(Rdst, BitsPerInt);
- __ bind(done);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countTrailingZerosL(rRegI dst, eRegL src, eFlagsReg cr) %{
- predicate(UseCountTrailingZerosInstruction);
- match(Set dst (CountTrailingZerosL src));
- effect(TEMP dst, KILL cr);
-
- format %{ "TZCNT $dst, $src.lo\t# count trailing zeros (long) \n\t"
- "JNC done\n\t"
- "TZCNT $dst, $src.hi\n\t"
- "ADD $dst, 32\n"
- "done:" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- Label done;
- __ tzcntl(Rdst, Rsrc);
- __ jccb(Assembler::carryClear, done);
- __ tzcntl(Rdst, HIGH_FROM_LOW(Rsrc));
- __ addl(Rdst, BitsPerInt);
- __ bind(done);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct countTrailingZerosL_bsf(rRegI dst, eRegL src, eFlagsReg cr) %{
- predicate(!UseCountTrailingZerosInstruction);
- match(Set dst (CountTrailingZerosL src));
- effect(TEMP dst, KILL cr);
-
- format %{ "BSF $dst, $src.lo\t# count trailing zeros (long)\n\t"
- "JNZ done\n\t"
- "BSF $dst, $src.hi\n\t"
- "JNZ msw_not_zero\n\t"
- "MOV $dst, 32\n"
- "msw_not_zero:\n\t"
- "ADD $dst, 32\n"
- "done:" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- Label msw_not_zero;
- Label done;
- __ bsfl(Rdst, Rsrc);
- __ jccb(Assembler::notZero, done);
- __ bsfl(Rdst, HIGH_FROM_LOW(Rsrc));
- __ jccb(Assembler::notZero, msw_not_zero);
- __ movl(Rdst, BitsPerInt);
- __ bind(msw_not_zero);
- __ addl(Rdst, BitsPerInt);
- __ bind(done);
- %}
- ins_pipe(ialu_reg);
-%}
-
-
-//---------- Population Count Instructions -------------------------------------
-
-instruct popCountI(rRegI dst, rRegI src, eFlagsReg cr) %{
- predicate(UsePopCountInstruction);
- match(Set dst (PopCountI src));
- effect(KILL cr);
-
- format %{ "POPCNT $dst, $src" %}
- ins_encode %{
- __ popcntl($dst$$Register, $src$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct popCountI_mem(rRegI dst, memory mem, eFlagsReg cr) %{
- predicate(UsePopCountInstruction);
- match(Set dst (PopCountI (LoadI mem)));
- effect(KILL cr);
-
- format %{ "POPCNT $dst, $mem" %}
- ins_encode %{
- __ popcntl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg);
-%}
-
-// Note: Long.bitCount(long) returns an int.
-instruct popCountL(rRegI dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
- predicate(UsePopCountInstruction);
- match(Set dst (PopCountL src));
- effect(KILL cr, TEMP tmp, TEMP dst);
-
- format %{ "POPCNT $dst, $src.lo\n\t"
- "POPCNT $tmp, $src.hi\n\t"
- "ADD $dst, $tmp" %}
- ins_encode %{
- __ popcntl($dst$$Register, $src$$Register);
- __ popcntl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
- __ addl($dst$$Register, $tmp$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-// Note: Long.bitCount(long) returns an int.
-instruct popCountL_mem(rRegI dst, memory mem, rRegI tmp, eFlagsReg cr) %{
- predicate(UsePopCountInstruction);
- match(Set dst (PopCountL (LoadL mem)));
- effect(KILL cr, TEMP tmp, TEMP dst);
-
- format %{ "POPCNT $dst, $mem\n\t"
- "POPCNT $tmp, $mem+4\n\t"
- "ADD $dst, $tmp" %}
- ins_encode %{
- //__ popcntl($dst$$Register, $mem$$Address$$first);
- //__ popcntl($tmp$$Register, $mem$$Address$$second);
- __ popcntl($dst$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none));
- __ popcntl($tmp$$Register, Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none));
- __ addl($dst$$Register, $tmp$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-
-//----------Load/Store/Move Instructions---------------------------------------
-//----------Load Instructions--------------------------------------------------
-// Load Byte (8bit signed)
-instruct loadB(xRegI dst, memory mem) %{
- match(Set dst (LoadB mem));
-
- ins_cost(125);
- format %{ "MOVSX8 $dst,$mem\t# byte" %}
-
- ins_encode %{
- __ movsbl($dst$$Register, $mem$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Byte (8bit signed) into Long Register
-instruct loadB2L(eRegL dst, memory mem, eFlagsReg cr) %{
- match(Set dst (ConvI2L (LoadB mem)));
- effect(KILL cr);
-
- ins_cost(375);
- format %{ "MOVSX8 $dst.lo,$mem\t# byte -> long\n\t"
- "MOV $dst.hi,$dst.lo\n\t"
- "SAR $dst.hi,7" %}
-
- ins_encode %{
- __ movsbl($dst$$Register, $mem$$Address);
- __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
- __ sarl(HIGH_FROM_LOW($dst$$Register), 7); // 24+1 MSB are already signed extended.
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Byte (8bit UNsigned)
-instruct loadUB(xRegI dst, memory mem) %{
- match(Set dst (LoadUB mem));
-
- ins_cost(125);
- format %{ "MOVZX8 $dst,$mem\t# ubyte -> int" %}
-
- ins_encode %{
- __ movzbl($dst$$Register, $mem$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Byte (8 bit UNsigned) into Long Register
-instruct loadUB2L(eRegL dst, memory mem, eFlagsReg cr) %{
- match(Set dst (ConvI2L (LoadUB mem)));
- effect(KILL cr);
-
- ins_cost(250);
- format %{ "MOVZX8 $dst.lo,$mem\t# ubyte -> long\n\t"
- "XOR $dst.hi,$dst.hi" %}
-
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movzbl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Byte (8 bit UNsigned) with mask into Long Register
-instruct loadUB2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
- match(Set dst (ConvI2L (AndI (LoadUB mem) mask)));
- effect(KILL cr);
-
- format %{ "MOVZX8 $dst.lo,$mem\t# ubyte & 32-bit mask -> long\n\t"
- "XOR $dst.hi,$dst.hi\n\t"
- "AND $dst.lo,right_n_bits($mask, 8)" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movzbl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- __ andl(Rdst, $mask$$constant & right_n_bits(8));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Short (16bit signed)
-instruct loadS(rRegI dst, memory mem) %{
- match(Set dst (LoadS mem));
-
- ins_cost(125);
- format %{ "MOVSX $dst,$mem\t# short" %}
-
- ins_encode %{
- __ movswl($dst$$Register, $mem$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Short (16 bit signed) to Byte (8 bit signed)
-instruct loadS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
- match(Set dst (RShiftI (LShiftI (LoadS mem) twentyfour) twentyfour));
-
- ins_cost(125);
- format %{ "MOVSX $dst, $mem\t# short -> byte" %}
- ins_encode %{
- __ movsbl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Short (16bit signed) into Long Register
-instruct loadS2L(eRegL dst, memory mem, eFlagsReg cr) %{
- match(Set dst (ConvI2L (LoadS mem)));
- effect(KILL cr);
-
- ins_cost(375);
- format %{ "MOVSX $dst.lo,$mem\t# short -> long\n\t"
- "MOV $dst.hi,$dst.lo\n\t"
- "SAR $dst.hi,15" %}
-
- ins_encode %{
- __ movswl($dst$$Register, $mem$$Address);
- __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
- __ sarl(HIGH_FROM_LOW($dst$$Register), 15); // 16+1 MSB are already signed extended.
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Short/Char (16bit unsigned)
-instruct loadUS(rRegI dst, memory mem) %{
- match(Set dst (LoadUS mem));
-
- ins_cost(125);
- format %{ "MOVZX $dst,$mem\t# ushort/char -> int" %}
-
- ins_encode %{
- __ movzwl($dst$$Register, $mem$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Short/Char (16 bit UNsigned) to Byte (8 bit signed)
-instruct loadUS2B(rRegI dst, memory mem, immI_24 twentyfour) %{
- match(Set dst (RShiftI (LShiftI (LoadUS mem) twentyfour) twentyfour));
-
- ins_cost(125);
- format %{ "MOVSX $dst, $mem\t# ushort -> byte" %}
- ins_encode %{
- __ movsbl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Short/Char (16 bit UNsigned) into Long Register
-instruct loadUS2L(eRegL dst, memory mem, eFlagsReg cr) %{
- match(Set dst (ConvI2L (LoadUS mem)));
- effect(KILL cr);
-
- ins_cost(250);
- format %{ "MOVZX $dst.lo,$mem\t# ushort/char -> long\n\t"
- "XOR $dst.hi,$dst.hi" %}
-
- ins_encode %{
- __ movzwl($dst$$Register, $mem$$Address);
- __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Short/Char (16 bit UNsigned) with mask 0xFF into Long Register
-instruct loadUS2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
- match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
- effect(KILL cr);
-
- format %{ "MOVZX8 $dst.lo,$mem\t# ushort/char & 0xFF -> long\n\t"
- "XOR $dst.hi,$dst.hi" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movzbl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Short/Char (16 bit UNsigned) with a 32-bit mask into Long Register
-instruct loadUS2L_immI(eRegL dst, memory mem, immI mask, eFlagsReg cr) %{
- match(Set dst (ConvI2L (AndI (LoadUS mem) mask)));
- effect(KILL cr);
-
- format %{ "MOVZX $dst.lo, $mem\t# ushort/char & 32-bit mask -> long\n\t"
- "XOR $dst.hi,$dst.hi\n\t"
- "AND $dst.lo,right_n_bits($mask, 16)" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movzwl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- __ andl(Rdst, $mask$$constant & right_n_bits(16));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer
-instruct loadI(rRegI dst, memory mem) %{
- match(Set dst (LoadI mem));
-
- ins_cost(125);
- format %{ "MOV $dst,$mem\t# int" %}
-
- ins_encode %{
- __ movl($dst$$Register, $mem$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer (32 bit signed) to Byte (8 bit signed)
-instruct loadI2B(rRegI dst, memory mem, immI_24 twentyfour) %{
- match(Set dst (RShiftI (LShiftI (LoadI mem) twentyfour) twentyfour));
-
- ins_cost(125);
- format %{ "MOVSX $dst, $mem\t# int -> byte" %}
- ins_encode %{
- __ movsbl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer (32 bit signed) to Unsigned Byte (8 bit UNsigned)
-instruct loadI2UB(rRegI dst, memory mem, immI_255 mask) %{
- match(Set dst (AndI (LoadI mem) mask));
-
- ins_cost(125);
- format %{ "MOVZX $dst, $mem\t# int -> ubyte" %}
- ins_encode %{
- __ movzbl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer (32 bit signed) to Short (16 bit signed)
-instruct loadI2S(rRegI dst, memory mem, immI_16 sixteen) %{
- match(Set dst (RShiftI (LShiftI (LoadI mem) sixteen) sixteen));
-
- ins_cost(125);
- format %{ "MOVSX $dst, $mem\t# int -> short" %}
- ins_encode %{
- __ movswl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer (32 bit signed) to Unsigned Short/Char (16 bit UNsigned)
-instruct loadI2US(rRegI dst, memory mem, immI_65535 mask) %{
- match(Set dst (AndI (LoadI mem) mask));
-
- ins_cost(125);
- format %{ "MOVZX $dst, $mem\t# int -> ushort/char" %}
- ins_encode %{
- __ movzwl($dst$$Register, $mem$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer into Long Register
-instruct loadI2L(eRegL dst, memory mem, eFlagsReg cr) %{
- match(Set dst (ConvI2L (LoadI mem)));
- effect(KILL cr);
-
- ins_cost(375);
- format %{ "MOV $dst.lo,$mem\t# int -> long\n\t"
- "MOV $dst.hi,$dst.lo\n\t"
- "SAR $dst.hi,31" %}
-
- ins_encode %{
- __ movl($dst$$Register, $mem$$Address);
- __ movl(HIGH_FROM_LOW($dst$$Register), $dst$$Register); // This is always a different register.
- __ sarl(HIGH_FROM_LOW($dst$$Register), 31);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer with mask 0xFF into Long Register
-instruct loadI2L_immI_255(eRegL dst, memory mem, immI_255 mask, eFlagsReg cr) %{
- match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
- effect(KILL cr);
-
- format %{ "MOVZX8 $dst.lo,$mem\t# int & 0xFF -> long\n\t"
- "XOR $dst.hi,$dst.hi" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movzbl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer with mask 0xFFFF into Long Register
-instruct loadI2L_immI_65535(eRegL dst, memory mem, immI_65535 mask, eFlagsReg cr) %{
- match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
- effect(KILL cr);
-
- format %{ "MOVZX $dst.lo,$mem\t# int & 0xFFFF -> long\n\t"
- "XOR $dst.hi,$dst.hi" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movzwl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Integer with 31-bit mask into Long Register
-instruct loadI2L_immU31(eRegL dst, memory mem, immU31 mask, eFlagsReg cr) %{
- match(Set dst (ConvI2L (AndI (LoadI mem) mask)));
- effect(KILL cr);
-
- format %{ "MOV $dst.lo,$mem\t# int & 31-bit mask -> long\n\t"
- "XOR $dst.hi,$dst.hi\n\t"
- "AND $dst.lo,$mask" %}
- ins_encode %{
- Register Rdst = $dst$$Register;
- __ movl(Rdst, $mem$$Address);
- __ xorl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rdst));
- __ andl(Rdst, $mask$$constant);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Unsigned Integer into Long Register
-instruct loadUI2L(eRegL dst, memory mem, immL_32bits mask, eFlagsReg cr) %{
- match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
- effect(KILL cr);
-
- ins_cost(250);
- format %{ "MOV $dst.lo,$mem\t# uint -> long\n\t"
- "XOR $dst.hi,$dst.hi" %}
-
- ins_encode %{
- __ movl($dst$$Register, $mem$$Address);
- __ xorl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register));
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Load Long. Cannot clobber address while loading, so restrict address
-// register to ESI
-instruct loadL(eRegL dst, load_long_memory mem) %{
- predicate(!((LoadLNode*)n)->require_atomic_access());
- match(Set dst (LoadL mem));
-
- ins_cost(250);
- format %{ "MOV $dst.lo,$mem\t# long\n\t"
- "MOV $dst.hi,$mem+4" %}
-
- ins_encode %{
- Address Amemlo = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp, relocInfo::none);
- Address Amemhi = Address::make_raw($mem$$base, $mem$$index, $mem$$scale, $mem$$disp + 4, relocInfo::none);
- __ movl($dst$$Register, Amemlo);
- __ movl(HIGH_FROM_LOW($dst$$Register), Amemhi);
- %}
-
- ins_pipe(ialu_reg_long_mem);
-%}
-
-// Volatile Load Long. Must be atomic, so do 64-bit FILD
-// then store it down to the stack and reload on the int
-// side.
-instruct loadL_volatile(stackSlotL dst, memory mem) %{
- predicate(UseSSE<=1 && ((LoadLNode*)n)->require_atomic_access());
- match(Set dst (LoadL mem));
-
- ins_cost(200);
- format %{ "FILD $mem\t# Atomic volatile long load\n\t"
- "FISTp $dst" %}
- ins_encode(enc_loadL_volatile(mem,dst));
- ins_pipe( fpu_reg_mem );
-%}
-
-instruct loadLX_volatile(stackSlotL dst, memory mem, regD tmp) %{
- predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
- match(Set dst (LoadL mem));
- effect(TEMP tmp);
- ins_cost(180);
- format %{ "MOVSD $tmp,$mem\t# Atomic volatile long load\n\t"
- "MOVSD $dst,$tmp" %}
- ins_encode %{
- __ movdbl($tmp$$XMMRegister, $mem$$Address);
- __ movdbl(Address(rsp, $dst$$disp), $tmp$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct loadLX_reg_volatile(eRegL dst, memory mem, regD tmp) %{
- predicate(UseSSE>=2 && ((LoadLNode*)n)->require_atomic_access());
- match(Set dst (LoadL mem));
- effect(TEMP tmp);
- ins_cost(160);
- format %{ "MOVSD $tmp,$mem\t# Atomic volatile long load\n\t"
- "MOVD $dst.lo,$tmp\n\t"
- "PSRLQ $tmp,32\n\t"
- "MOVD $dst.hi,$tmp" %}
- ins_encode %{
- __ movdbl($tmp$$XMMRegister, $mem$$Address);
- __ movdl($dst$$Register, $tmp$$XMMRegister);
- __ psrlq($tmp$$XMMRegister, 32);
- __ movdl(HIGH_FROM_LOW($dst$$Register), $tmp$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Load Range
-instruct loadRange(rRegI dst, memory mem) %{
- match(Set dst (LoadRange mem));
-
- ins_cost(125);
- format %{ "MOV $dst,$mem" %}
- opcode(0x8B);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_mem );
-%}
-
-
-// Load Pointer
-instruct loadP(eRegP dst, memory mem) %{
- match(Set dst (LoadP mem));
-
- ins_cost(125);
- format %{ "MOV $dst,$mem" %}
- opcode(0x8B);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_mem );
-%}
-
-// Load Klass Pointer
-instruct loadKlass(eRegP dst, memory mem) %{
- match(Set dst (LoadKlass mem));
-
- ins_cost(125);
- format %{ "MOV $dst,$mem" %}
- opcode(0x8B);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_mem );
-%}
-
-// Load Double
-instruct loadDPR(regDPR dst, memory mem) %{
- predicate(UseSSE<=1);
- match(Set dst (LoadD mem));
-
- ins_cost(150);
- format %{ "FLD_D ST,$mem\n\t"
- "FSTP $dst" %}
- opcode(0xDD); /* DD /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem),
- Pop_Reg_DPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-// Load Double to XMM
-instruct loadD(regD dst, memory mem) %{
- predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
- match(Set dst (LoadD mem));
- ins_cost(145);
- format %{ "MOVSD $dst,$mem" %}
- ins_encode %{
- __ movdbl ($dst$$XMMRegister, $mem$$Address);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct loadD_partial(regD dst, memory mem) %{
- predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
- match(Set dst (LoadD mem));
- ins_cost(145);
- format %{ "MOVLPD $dst,$mem" %}
- ins_encode %{
- __ movdbl ($dst$$XMMRegister, $mem$$Address);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Load to XMM register (single-precision floating point)
-// MOVSS instruction
-instruct loadF(regF dst, memory mem) %{
- predicate(UseSSE>=1);
- match(Set dst (LoadF mem));
- ins_cost(145);
- format %{ "MOVSS $dst,$mem" %}
- ins_encode %{
- __ movflt ($dst$$XMMRegister, $mem$$Address);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Load Float
-instruct loadFPR(regFPR dst, memory mem) %{
- predicate(UseSSE==0);
- match(Set dst (LoadF mem));
-
- ins_cost(150);
- format %{ "FLD_S ST,$mem\n\t"
- "FSTP $dst" %}
- opcode(0xD9); /* D9 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem),
- Pop_Reg_FPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-// Load Effective Address
-instruct leaP8(eRegP dst, indOffset8 mem) %{
- match(Set dst mem);
-
- ins_cost(110);
- format %{ "LEA $dst,$mem" %}
- opcode(0x8D);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_reg_fat );
-%}
-
-instruct leaP32(eRegP dst, indOffset32 mem) %{
- match(Set dst mem);
-
- ins_cost(110);
- format %{ "LEA $dst,$mem" %}
- opcode(0x8D);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_reg_fat );
-%}
-
-instruct leaPIdxOff(eRegP dst, indIndexOffset mem) %{
- match(Set dst mem);
-
- ins_cost(110);
- format %{ "LEA $dst,$mem" %}
- opcode(0x8D);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_reg_fat );
-%}
-
-instruct leaPIdxScale(eRegP dst, indIndexScale mem) %{
- match(Set dst mem);
-
- ins_cost(110);
- format %{ "LEA $dst,$mem" %}
- opcode(0x8D);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_reg_fat );
-%}
-
-instruct leaPIdxScaleOff(eRegP dst, indIndexScaleOffset mem) %{
- match(Set dst mem);
-
- ins_cost(110);
- format %{ "LEA $dst,$mem" %}
- opcode(0x8D);
- ins_encode( SetInstMark, OpcP, RegMem(dst,mem), ClearInstMark);
- ins_pipe( ialu_reg_reg_fat );
-%}
-
-// Load Constant
-instruct loadConI(rRegI dst, immI src) %{
- match(Set dst src);
-
- format %{ "MOV $dst,$src" %}
- ins_encode( SetInstMark, LdImmI(dst, src), ClearInstMark );
- ins_pipe( ialu_reg_fat );
-%}
-
-// Load Constant zero
-instruct loadConI0(rRegI dst, immI_0 src, eFlagsReg cr) %{
- match(Set dst src);
- effect(KILL cr);
-
- ins_cost(50);
- format %{ "XOR $dst,$dst" %}
- opcode(0x33); /* + rd */
- ins_encode( OpcP, RegReg( dst, dst ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct loadConP(eRegP dst, immP src) %{
- match(Set dst src);
-
- format %{ "MOV $dst,$src" %}
- opcode(0xB8); /* + rd */
- ins_encode( SetInstMark, LdImmP(dst, src), ClearInstMark );
- ins_pipe( ialu_reg_fat );
-%}
-
-instruct loadConL(eRegL dst, immL src, eFlagsReg cr) %{
- match(Set dst src);
- effect(KILL cr);
- ins_cost(200);
- format %{ "MOV $dst.lo,$src.lo\n\t"
- "MOV $dst.hi,$src.hi" %}
- opcode(0xB8);
- ins_encode( LdImmL_Lo(dst, src), LdImmL_Hi(dst, src) );
- ins_pipe( ialu_reg_long_fat );
-%}
-
-instruct loadConL0(eRegL dst, immL0 src, eFlagsReg cr) %{
- match(Set dst src);
- effect(KILL cr);
- ins_cost(150);
- format %{ "XOR $dst.lo,$dst.lo\n\t"
- "XOR $dst.hi,$dst.hi" %}
- opcode(0x33,0x33);
- ins_encode( RegReg_Lo(dst,dst), RegReg_Hi(dst, dst) );
- ins_pipe( ialu_reg_long );
-%}
-
-// The instruction usage is guarded by predicate in operand immFPR().
-instruct loadConFPR(regFPR dst, immFPR con) %{
- match(Set dst con);
- ins_cost(125);
- format %{ "FLD_S ST,[$constantaddress]\t# load from constant table: float=$con\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fld_s($constantaddress($con));
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_con);
-%}
-
-// The instruction usage is guarded by predicate in operand immFPR0().
-instruct loadConFPR0(regFPR dst, immFPR0 con) %{
- match(Set dst con);
- ins_cost(125);
- format %{ "FLDZ ST\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fldz();
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_con);
-%}
-
-// The instruction usage is guarded by predicate in operand immFPR1().
-instruct loadConFPR1(regFPR dst, immFPR1 con) %{
- match(Set dst con);
- ins_cost(125);
- format %{ "FLD1 ST\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fld1();
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_con);
-%}
-
-// The instruction usage is guarded by predicate in operand immF().
-instruct loadConF(regF dst, immF con) %{
- match(Set dst con);
- ins_cost(125);
- format %{ "MOVSS $dst,[$constantaddress]\t# load from constant table: float=$con" %}
- ins_encode %{
- __ movflt($dst$$XMMRegister, $constantaddress($con));
- %}
- ins_pipe(pipe_slow);
-%}
-
-// The instruction usage is guarded by predicate in operand immF0().
-instruct loadConF0(regF dst, immF0 src) %{
- match(Set dst src);
- ins_cost(100);
- format %{ "XORPS $dst,$dst\t# float 0.0" %}
- ins_encode %{
- __ xorps($dst$$XMMRegister, $dst$$XMMRegister);
- %}
- ins_pipe(pipe_slow);
-%}
-
-// The instruction usage is guarded by predicate in operand immDPR().
-instruct loadConDPR(regDPR dst, immDPR con) %{
- match(Set dst con);
- ins_cost(125);
-
- format %{ "FLD_D ST,[$constantaddress]\t# load from constant table: double=$con\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fld_d($constantaddress($con));
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_con);
-%}
-
-// The instruction usage is guarded by predicate in operand immDPR0().
-instruct loadConDPR0(regDPR dst, immDPR0 con) %{
- match(Set dst con);
- ins_cost(125);
-
- format %{ "FLDZ ST\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fldz();
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_con);
-%}
-
-// The instruction usage is guarded by predicate in operand immDPR1().
-instruct loadConDPR1(regDPR dst, immDPR1 con) %{
- match(Set dst con);
- ins_cost(125);
-
- format %{ "FLD1 ST\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fld1();
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_con);
-%}
-
-// The instruction usage is guarded by predicate in operand immD().
-instruct loadConD(regD dst, immD con) %{
- match(Set dst con);
- ins_cost(125);
- format %{ "MOVSD $dst,[$constantaddress]\t# load from constant table: double=$con" %}
- ins_encode %{
- __ movdbl($dst$$XMMRegister, $constantaddress($con));
- %}
- ins_pipe(pipe_slow);
-%}
-
-// The instruction usage is guarded by predicate in operand immD0().
-instruct loadConD0(regD dst, immD0 src) %{
- match(Set dst src);
- ins_cost(100);
- format %{ "XORPD $dst,$dst\t# double 0.0" %}
- ins_encode %{
- __ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Load Stack Slot
-instruct loadSSI(rRegI dst, stackSlotI src) %{
- match(Set dst src);
- ins_cost(125);
-
- format %{ "MOV $dst,$src" %}
- opcode(0x8B);
- ins_encode( SetInstMark, OpcP, RegMem(dst,src), ClearInstMark);
- ins_pipe( ialu_reg_mem );
-%}
-
-instruct loadSSL(eRegL dst, stackSlotL src) %{
- match(Set dst src);
-
- ins_cost(200);
- format %{ "MOV $dst,$src.lo\n\t"
- "MOV $dst+4,$src.hi" %}
- opcode(0x8B, 0x8B);
- ins_encode( SetInstMark, OpcP, RegMem( dst, src ), OpcS, RegMem_Hi( dst, src ), ClearInstMark );
- ins_pipe( ialu_mem_long_reg );
-%}
-
-// Load Stack Slot
-instruct loadSSP(eRegP dst, stackSlotP src) %{
- match(Set dst src);
- ins_cost(125);
-
- format %{ "MOV $dst,$src" %}
- opcode(0x8B);
- ins_encode( SetInstMark, OpcP, RegMem(dst,src), ClearInstMark);
- ins_pipe( ialu_reg_mem );
-%}
-
-// Load Stack Slot
-instruct loadSSF(regFPR dst, stackSlotF src) %{
- match(Set dst src);
- ins_cost(125);
-
- format %{ "FLD_S $src\n\t"
- "FSTP $dst" %}
- opcode(0xD9); /* D9 /0, FLD m32real */
- ins_encode( SetInstMark, OpcP, RMopc_Mem_no_oop(0x00,src),
- Pop_Reg_FPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-// Load Stack Slot
-instruct loadSSD(regDPR dst, stackSlotD src) %{
- match(Set dst src);
- ins_cost(125);
-
- format %{ "FLD_D $src\n\t"
- "FSTP $dst" %}
- opcode(0xDD); /* DD /0, FLD m64real */
- ins_encode( SetInstMark, OpcP, RMopc_Mem_no_oop(0x00,src),
- Pop_Reg_DPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-// Prefetch instructions for allocation.
-// Must be safe to execute with invalid address (cannot fault).
-
-instruct prefetchAlloc0( memory mem ) %{
- predicate(UseSSE==0 && AllocatePrefetchInstr!=3);
- match(PrefetchAllocation mem);
- ins_cost(0);
- size(0);
- format %{ "Prefetch allocation (non-SSE is empty encoding)" %}
- ins_encode();
- ins_pipe(empty);
-%}
-
-instruct prefetchAlloc( memory mem ) %{
- predicate(AllocatePrefetchInstr==3);
- match( PrefetchAllocation mem );
- ins_cost(100);
-
- format %{ "PREFETCHW $mem\t! Prefetch allocation into L1 cache and mark modified" %}
- ins_encode %{
- __ prefetchw($mem$$Address);
- %}
- ins_pipe(ialu_mem);
-%}
-
-instruct prefetchAllocNTA( memory mem ) %{
- predicate(UseSSE>=1 && AllocatePrefetchInstr==0);
- match(PrefetchAllocation mem);
- ins_cost(100);
-
- format %{ "PREFETCHNTA $mem\t! Prefetch allocation into non-temporal cache for write" %}
- ins_encode %{
- __ prefetchnta($mem$$Address);
- %}
- ins_pipe(ialu_mem);
-%}
-
-instruct prefetchAllocT0( memory mem ) %{
- predicate(UseSSE>=1 && AllocatePrefetchInstr==1);
- match(PrefetchAllocation mem);
- ins_cost(100);
-
- format %{ "PREFETCHT0 $mem\t! Prefetch allocation into L1 and L2 caches for write" %}
- ins_encode %{
- __ prefetcht0($mem$$Address);
- %}
- ins_pipe(ialu_mem);
-%}
-
-instruct prefetchAllocT2( memory mem ) %{
- predicate(UseSSE>=1 && AllocatePrefetchInstr==2);
- match(PrefetchAllocation mem);
- ins_cost(100);
-
- format %{ "PREFETCHT2 $mem\t! Prefetch allocation into L2 cache for write" %}
- ins_encode %{
- __ prefetcht2($mem$$Address);
- %}
- ins_pipe(ialu_mem);
-%}
-
-//----------Store Instructions-------------------------------------------------
-
-// Store Byte
-instruct storeB(memory mem, xRegI src) %{
- match(Set mem (StoreB mem src));
-
- ins_cost(125);
- format %{ "MOV8 $mem,$src" %}
- opcode(0x88);
- ins_encode( SetInstMark, OpcP, RegMem( src, mem ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Store Char/Short
-instruct storeC(memory mem, rRegI src) %{
- match(Set mem (StoreC mem src));
-
- ins_cost(125);
- format %{ "MOV16 $mem,$src" %}
- opcode(0x89, 0x66);
- ins_encode( SetInstMark, OpcS, OpcP, RegMem( src, mem ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Store Integer
-instruct storeI(memory mem, rRegI src) %{
- match(Set mem (StoreI mem src));
-
- ins_cost(125);
- format %{ "MOV $mem,$src" %}
- opcode(0x89);
- ins_encode( SetInstMark, OpcP, RegMem( src, mem ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Store Long
-instruct storeL(long_memory mem, eRegL src) %{
- predicate(!((StoreLNode*)n)->require_atomic_access());
- match(Set mem (StoreL mem src));
-
- ins_cost(200);
- format %{ "MOV $mem,$src.lo\n\t"
- "MOV $mem+4,$src.hi" %}
- opcode(0x89, 0x89);
- ins_encode( SetInstMark, OpcP, RegMem( src, mem ), OpcS, RegMem_Hi( src, mem ), ClearInstMark );
- ins_pipe( ialu_mem_long_reg );
-%}
-
-// Store Long to Integer
-instruct storeL2I(memory mem, eRegL src) %{
- match(Set mem (StoreI mem (ConvL2I src)));
-
- format %{ "MOV $mem,$src.lo\t# long -> int" %}
- ins_encode %{
- __ movl($mem$$Address, $src$$Register);
- %}
- ins_pipe(ialu_mem_reg);
-%}
-
-// Volatile Store Long. Must be atomic, so move it into
-// the FP TOS and then do a 64-bit FIST. Has to probe the
-// target address before the store (for null-ptr checks)
-// so the memory operand is used twice in the encoding.
-instruct storeL_volatile(memory mem, stackSlotL src, eFlagsReg cr ) %{
- predicate(UseSSE<=1 && ((StoreLNode*)n)->require_atomic_access());
- match(Set mem (StoreL mem src));
- effect( KILL cr );
- ins_cost(400);
- format %{ "CMP $mem,EAX\t# Probe address for implicit null check\n\t"
- "FILD $src\n\t"
- "FISTp $mem\t # 64-bit atomic volatile long store" %}
- opcode(0x3B);
- ins_encode( SetInstMark, OpcP, RegMem( EAX, mem ), enc_storeL_volatile(mem,src), ClearInstMark);
- ins_pipe( fpu_reg_mem );
-%}
-
-instruct storeLX_volatile(memory mem, stackSlotL src, regD tmp, eFlagsReg cr) %{
- predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
- match(Set mem (StoreL mem src));
- effect( TEMP tmp, KILL cr );
- ins_cost(380);
- format %{ "CMP $mem,EAX\t# Probe address for implicit null check\n\t"
- "MOVSD $tmp,$src\n\t"
- "MOVSD $mem,$tmp\t # 64-bit atomic volatile long store" %}
- ins_encode %{
- __ cmpl(rax, $mem$$Address);
- __ movdbl($tmp$$XMMRegister, Address(rsp, $src$$disp));
- __ movdbl($mem$$Address, $tmp$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct storeLX_reg_volatile(memory mem, eRegL src, regD tmp2, regD tmp, eFlagsReg cr) %{
- predicate(UseSSE>=2 && ((StoreLNode*)n)->require_atomic_access());
- match(Set mem (StoreL mem src));
- effect( TEMP tmp2 , TEMP tmp, KILL cr );
- ins_cost(360);
- format %{ "CMP $mem,EAX\t# Probe address for implicit null check\n\t"
- "MOVD $tmp,$src.lo\n\t"
- "MOVD $tmp2,$src.hi\n\t"
- "PUNPCKLDQ $tmp,$tmp2\n\t"
- "MOVSD $mem,$tmp\t # 64-bit atomic volatile long store" %}
- ins_encode %{
- __ cmpl(rax, $mem$$Address);
- __ movdl($tmp$$XMMRegister, $src$$Register);
- __ movdl($tmp2$$XMMRegister, HIGH_FROM_LOW($src$$Register));
- __ punpckldq($tmp$$XMMRegister, $tmp2$$XMMRegister);
- __ movdbl($mem$$Address, $tmp$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Store Pointer; for storing unknown oops and raw pointers
-instruct storeP(memory mem, anyRegP src) %{
- match(Set mem (StoreP mem src));
-
- ins_cost(125);
- format %{ "MOV $mem,$src" %}
- opcode(0x89);
- ins_encode( SetInstMark, OpcP, RegMem( src, mem ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Store Integer Immediate
-instruct storeImmI(memory mem, immI src) %{
- match(Set mem (StoreI mem src));
-
- ins_cost(150);
- format %{ "MOV $mem,$src" %}
- opcode(0xC7); /* C7 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem), Con32(src), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-// Store Short/Char Immediate
-instruct storeImmI16(memory mem, immI16 src) %{
- predicate(UseStoreImmI16);
- match(Set mem (StoreC mem src));
-
- ins_cost(150);
- format %{ "MOV16 $mem,$src" %}
- opcode(0xC7); /* C7 /0 Same as 32 store immediate with prefix */
- ins_encode( SetInstMark, SizePrefix, OpcP, RMopc_Mem(0x00,mem), Con16(src), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-// Store Pointer Immediate; null pointers or constant oops that do not
-// need card-mark barriers.
-instruct storeImmP(memory mem, immP src) %{
- match(Set mem (StoreP mem src));
-
- ins_cost(150);
- format %{ "MOV $mem,$src" %}
- opcode(0xC7); /* C7 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem), Con32( src ), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-// Store Byte Immediate
-instruct storeImmB(memory mem, immI8 src) %{
- match(Set mem (StoreB mem src));
-
- ins_cost(150);
- format %{ "MOV8 $mem,$src" %}
- opcode(0xC6); /* C6 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem), Con8or32(src), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-// Store Double
-instruct storeDPR( memory mem, regDPR1 src) %{
- predicate(UseSSE<=1);
- match(Set mem (StoreD mem src));
-
- ins_cost(100);
- format %{ "FST_D $mem,$src" %}
- opcode(0xDD); /* DD /2 */
- ins_encode( enc_FPR_store(mem,src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-// Store double does rounding on x86
-instruct storeDPR_rounded( memory mem, regDPR1 src) %{
- predicate(UseSSE<=1);
- match(Set mem (StoreD mem (RoundDouble src)));
-
- ins_cost(100);
- format %{ "FST_D $mem,$src\t# round" %}
- opcode(0xDD); /* DD /2 */
- ins_encode( enc_FPR_store(mem,src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-// Store XMM register to memory (double-precision floating points)
-// MOVSD instruction
-instruct storeD(memory mem, regD src) %{
- predicate(UseSSE>=2);
- match(Set mem (StoreD mem src));
- ins_cost(95);
- format %{ "MOVSD $mem,$src" %}
- ins_encode %{
- __ movdbl($mem$$Address, $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Store XMM register to memory (single-precision floating point)
-// MOVSS instruction
-instruct storeF(memory mem, regF src) %{
- predicate(UseSSE>=1);
- match(Set mem (StoreF mem src));
- ins_cost(95);
- format %{ "MOVSS $mem,$src" %}
- ins_encode %{
- __ movflt($mem$$Address, $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-
-// Store Float
-instruct storeFPR( memory mem, regFPR1 src) %{
- predicate(UseSSE==0);
- match(Set mem (StoreF mem src));
-
- ins_cost(100);
- format %{ "FST_S $mem,$src" %}
- opcode(0xD9); /* D9 /2 */
- ins_encode( enc_FPR_store(mem,src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-// Store Float does rounding on x86
-instruct storeFPR_rounded( memory mem, regFPR1 src) %{
- predicate(UseSSE==0);
- match(Set mem (StoreF mem (RoundFloat src)));
-
- ins_cost(100);
- format %{ "FST_S $mem,$src\t# round" %}
- opcode(0xD9); /* D9 /2 */
- ins_encode( enc_FPR_store(mem,src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-// Store Float does rounding on x86
-instruct storeFPR_Drounded( memory mem, regDPR1 src) %{
- predicate(UseSSE<=1);
- match(Set mem (StoreF mem (ConvD2F src)));
-
- ins_cost(100);
- format %{ "FST_S $mem,$src\t# D-round" %}
- opcode(0xD9); /* D9 /2 */
- ins_encode( enc_FPR_store(mem,src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-// Store immediate Float value (it is faster than store from FPU register)
-// The instruction usage is guarded by predicate in operand immFPR().
-instruct storeFPR_imm( memory mem, immFPR src) %{
- match(Set mem (StoreF mem src));
-
- ins_cost(50);
- format %{ "MOV $mem,$src\t# store float" %}
- opcode(0xC7); /* C7 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem), Con32FPR_as_bits(src), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-// Store immediate Float value (it is faster than store from XMM register)
-// The instruction usage is guarded by predicate in operand immF().
-instruct storeF_imm( memory mem, immF src) %{
- match(Set mem (StoreF mem src));
-
- ins_cost(50);
- format %{ "MOV $mem,$src\t# store float" %}
- opcode(0xC7); /* C7 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem), Con32F_as_bits(src), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-// Store Integer to stack slot
-instruct storeSSI(stackSlotI dst, rRegI src) %{
- match(Set dst src);
-
- ins_cost(100);
- format %{ "MOV $dst,$src" %}
- opcode(0x89);
- ins_encode( OpcPRegSS( dst, src ) );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Store Integer to stack slot
-instruct storeSSP(stackSlotP dst, eRegP src) %{
- match(Set dst src);
-
- ins_cost(100);
- format %{ "MOV $dst,$src" %}
- opcode(0x89);
- ins_encode( OpcPRegSS( dst, src ) );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Store Long to stack slot
-instruct storeSSL(stackSlotL dst, eRegL src) %{
- match(Set dst src);
-
- ins_cost(200);
- format %{ "MOV $dst,$src.lo\n\t"
- "MOV $dst+4,$src.hi" %}
- opcode(0x89, 0x89);
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_long_reg );
-%}
-
-//----------MemBar Instructions-----------------------------------------------
-// Memory barrier flavors
-
-instruct membar_acquire() %{
- match(MemBarAcquire);
- match(LoadFence);
- ins_cost(400);
-
- size(0);
- format %{ "MEMBAR-acquire ! (empty encoding)" %}
- ins_encode();
- ins_pipe(empty);
-%}
-
-instruct membar_acquire_lock() %{
- match(MemBarAcquireLock);
- ins_cost(0);
-
- size(0);
- format %{ "MEMBAR-acquire (prior CMPXCHG in FastLock so empty encoding)" %}
- ins_encode( );
- ins_pipe(empty);
-%}
-
-instruct membar_release() %{
- match(MemBarRelease);
- match(StoreFence);
- ins_cost(400);
-
- size(0);
- format %{ "MEMBAR-release ! (empty encoding)" %}
- ins_encode( );
- ins_pipe(empty);
-%}
-
-instruct membar_release_lock() %{
- match(MemBarReleaseLock);
- ins_cost(0);
-
- size(0);
- format %{ "MEMBAR-release (a FastUnlock follows so empty encoding)" %}
- ins_encode( );
- ins_pipe(empty);
-%}
-
-instruct membar_volatile(eFlagsReg cr) %{
- match(MemBarVolatile);
- effect(KILL cr);
- ins_cost(400);
-
- format %{
- $$template
- $$emit$$"LOCK ADDL [ESP + #0], 0\t! membar_volatile"
- %}
- ins_encode %{
- __ membar(Assembler::StoreLoad);
- %}
- ins_pipe(pipe_slow);
-%}
-
-instruct unnecessary_membar_volatile() %{
- match(MemBarVolatile);
- predicate(Matcher::post_store_load_barrier(n));
- ins_cost(0);
-
- size(0);
- format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
- ins_encode( );
- ins_pipe(empty);
-%}
-
-instruct membar_storestore() %{
- match(MemBarStoreStore);
- match(StoreStoreFence);
- ins_cost(0);
-
- size(0);
- format %{ "MEMBAR-storestore (empty encoding)" %}
- ins_encode( );
- ins_pipe(empty);
-%}
-
-//----------Move Instructions--------------------------------------------------
-instruct castX2P(eAXRegP dst, eAXRegI src) %{
- match(Set dst (CastX2P src));
- format %{ "# X2P $dst, $src" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe(empty);
-%}
-
-instruct castP2X(rRegI dst, eRegP src ) %{
- match(Set dst (CastP2X src));
- ins_cost(50);
- format %{ "MOV $dst, $src\t# CastP2X" %}
- ins_encode( enc_Copy( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-//----------Conditional Move---------------------------------------------------
-// Conditional move
-instruct jmovI_reg(cmpOp cop, eFlagsReg cr, rRegI dst, rRegI src) %{
- predicate(!VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "J$cop,us skip\t# signed cmove\n\t"
- "MOV $dst,$src\n"
- "skip:" %}
- ins_encode %{
- Label Lskip;
- // Invert sense of branch from sense of CMOV
- __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
- __ movl($dst$$Register, $src$$Register);
- __ bind(Lskip);
- %}
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct jmovI_regU(cmpOpU cop, eFlagsRegU cr, rRegI dst, rRegI src) %{
- predicate(!VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "J$cop,us skip\t# unsigned cmove\n\t"
- "MOV $dst,$src\n"
- "skip:" %}
- ins_encode %{
- Label Lskip;
- // Invert sense of branch from sense of CMOV
- __ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
- __ movl($dst$$Register, $src$$Register);
- __ bind(Lskip);
- %}
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovI_reg(rRegI dst, rRegI src, eFlagsReg cr, cmpOp cop ) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cop $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cop), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovI_regU( cmpOpU cop, eFlagsRegU cr, rRegI dst, rRegI src ) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cop $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cop), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovI_regUCF( cmpOpUCF cop, eFlagsRegUCF cr, rRegI dst, rRegI src ) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovI_regU(cop, cr, dst, src);
- %}
-%}
-
-// Conditional move
-instruct cmovI_mem(cmpOp cop, eFlagsReg cr, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
- ins_cost(250);
- format %{ "CMOV$cop $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cop), RegMem( dst, src ), ClearInstMark );
- ins_pipe( pipe_cmov_mem );
-%}
-
-// Conditional move
-instruct cmovI_memU(cmpOpU cop, eFlagsRegU cr, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
- ins_cost(250);
- format %{ "CMOV$cop $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cop), RegMem( dst, src ), ClearInstMark );
- ins_pipe( pipe_cmov_mem );
-%}
-
-instruct cmovI_memUCF(cmpOpUCF cop, eFlagsRegUCF cr, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
- ins_cost(250);
- expand %{
- cmovI_memU(cop, cr, dst, src);
- %}
-%}
-
-// Conditional move
-instruct cmovP_reg(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cop $dst,$src\t# ptr" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cop), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-// Conditional move (non-P6 version)
-// Note: a CMoveP is generated for stubs and native wrappers
-// regardless of whether we are on a P6, so we
-// emulate a cmov here
-instruct cmovP_reg_nonP6(eRegP dst, eRegP src, eFlagsReg cr, cmpOp cop ) %{
- match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
- ins_cost(300);
- format %{ "Jn$cop skip\n\t"
- "MOV $dst,$src\t# pointer\n"
- "skip:" %}
- opcode(0x8b);
- ins_encode( enc_cmov_branch(cop, 0x2), OpcP, RegReg(dst, src));
- ins_pipe( pipe_cmov_reg );
-%}
-
-// Conditional move
-instruct cmovP_regU(cmpOpU cop, eFlagsRegU cr, eRegP dst, eRegP src ) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cop $dst,$src\t# ptr" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cop), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovP_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegP dst, eRegP src ) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovP_regU(cop, cr, dst, src);
- %}
-%}
-
-// DISABLED: Requires the ADLC to emit a bottom_type call that
-// correctly meets the two pointer arguments; one is an incoming
-// register but the other is a memory operand. ALSO appears to
-// be buggy with implicit null checks.
-//
-//// Conditional move
-//instruct cmovP_mem(cmpOp cop, eFlagsReg cr, eRegP dst, memory src) %{
-// predicate(VM_Version::supports_cmov() );
-// match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
-// ins_cost(250);
-// format %{ "CMOV$cop $dst,$src\t# ptr" %}
-// opcode(0x0F,0x40);
-// ins_encode( enc_cmov(cop), RegMem( dst, src ) );
-// ins_pipe( pipe_cmov_mem );
-//%}
-//
-//// Conditional move
-//instruct cmovP_memU(cmpOpU cop, eFlagsRegU cr, eRegP dst, memory src) %{
-// predicate(VM_Version::supports_cmov() );
-// match(Set dst (CMoveP (Binary cop cr) (Binary dst (LoadP src))));
-// ins_cost(250);
-// format %{ "CMOV$cop $dst,$src\t# ptr" %}
-// opcode(0x0F,0x40);
-// ins_encode( enc_cmov(cop), RegMem( dst, src ) );
-// ins_pipe( pipe_cmov_mem );
-//%}
-
-// Conditional move
-instruct fcmovDPR_regU(cmpOp_fcmov cop, eFlagsRegU cr, regDPR1 dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "FCMOV$cop $dst,$src\t# double" %}
- opcode(0xDA);
- ins_encode( enc_cmov_dpr(cop,src) );
- ins_pipe( pipe_cmovDPR_reg );
-%}
-
-// Conditional move
-instruct fcmovFPR_regU(cmpOp_fcmov cop, eFlagsRegU cr, regFPR1 dst, regFPR src) %{
- predicate(UseSSE==0);
- match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "FCMOV$cop $dst,$src\t# float" %}
- opcode(0xDA);
- ins_encode( enc_cmov_dpr(cop,src) );
- ins_pipe( pipe_cmovDPR_reg );
-%}
-
-// Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
-instruct fcmovDPR_regS(cmpOp cop, eFlagsReg cr, regDPR dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "Jn$cop skip\n\t"
- "MOV $dst,$src\t# double\n"
- "skip:" %}
- opcode (0xdd, 0x3); /* DD D8+i or DD /3 */
- ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_DPR(src), OpcP, RegOpc(dst) );
- ins_pipe( pipe_cmovDPR_reg );
-%}
-
-// Float CMOV on Intel doesn't handle *signed* compares, only unsigned.
-instruct fcmovFPR_regS(cmpOp cop, eFlagsReg cr, regFPR dst, regFPR src) %{
- predicate(UseSSE==0);
- match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "Jn$cop skip\n\t"
- "MOV $dst,$src\t# float\n"
- "skip:" %}
- opcode (0xdd, 0x3); /* DD D8+i or DD /3 */
- ins_encode( enc_cmov_branch( cop, 0x4 ), Push_Reg_FPR(src), OpcP, RegOpc(dst) );
- ins_pipe( pipe_cmovDPR_reg );
-%}
-
-// No CMOVE with SSE/SSE2
-instruct fcmovF_regS(cmpOp cop, eFlagsReg cr, regF dst, regF src) %{
- predicate (UseSSE>=1);
- match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "Jn$cop skip\n\t"
- "MOVSS $dst,$src\t# float\n"
- "skip:" %}
- ins_encode %{
- Label skip;
- // Invert sense of branch from sense of CMOV
- __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
- __ movflt($dst$$XMMRegister, $src$$XMMRegister);
- __ bind(skip);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// No CMOVE with SSE/SSE2
-instruct fcmovD_regS(cmpOp cop, eFlagsReg cr, regD dst, regD src) %{
- predicate (UseSSE>=2);
- match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "Jn$cop skip\n\t"
- "MOVSD $dst,$src\t# float\n"
- "skip:" %}
- ins_encode %{
- Label skip;
- // Invert sense of branch from sense of CMOV
- __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
- __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
- __ bind(skip);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// unsigned version
-instruct fcmovF_regU(cmpOpU cop, eFlagsRegU cr, regF dst, regF src) %{
- predicate (UseSSE>=1);
- match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "Jn$cop skip\n\t"
- "MOVSS $dst,$src\t# float\n"
- "skip:" %}
- ins_encode %{
- Label skip;
- // Invert sense of branch from sense of CMOV
- __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
- __ movflt($dst$$XMMRegister, $src$$XMMRegister);
- __ bind(skip);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct fcmovF_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regF dst, regF src) %{
- predicate (UseSSE>=1);
- match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovF_regU(cop, cr, dst, src);
- %}
-%}
-
-// unsigned version
-instruct fcmovD_regU(cmpOpU cop, eFlagsRegU cr, regD dst, regD src) %{
- predicate (UseSSE>=2);
- match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "Jn$cop skip\n\t"
- "MOVSD $dst,$src\t# float\n"
- "skip:" %}
- ins_encode %{
- Label skip;
- // Invert sense of branch from sense of CMOV
- __ jccb((Assembler::Condition)($cop$$cmpcode^1), skip);
- __ movdbl($dst$$XMMRegister, $src$$XMMRegister);
- __ bind(skip);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct fcmovD_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, regD dst, regD src) %{
- predicate (UseSSE>=2);
- match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovD_regU(cop, cr, dst, src);
- %}
-%}
-
-instruct cmovL_reg(cmpOp cop, eFlagsReg cr, eRegL dst, eRegL src) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
- "CMOV$cop $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovL_regU(cmpOpU cop, eFlagsRegU cr, eRegL dst, eRegL src) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cop $dst.lo,$src.lo\n\t"
- "CMOV$cop $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cop), RegReg_Lo2( dst, src ), enc_cmov(cop), RegReg_Hi2( dst, src ) );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovL_regUCF(cmpOpUCF cop, eFlagsRegUCF cr, eRegL dst, eRegL src) %{
- predicate(VM_Version::supports_cmov() );
- match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovL_regU(cop, cr, dst, src);
- %}
-%}
-
-//----------Arithmetic Instructions--------------------------------------------
-//----------Addition Instructions----------------------------------------------
-
-// Integer Addition Instructions
-instruct addI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (AddI dst src));
- effect(KILL cr);
-
- size(2);
- format %{ "ADD $dst,$src" %}
- opcode(0x03);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct addI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
- match(Set dst (AddI dst src));
- effect(KILL cr);
-
- format %{ "ADD $dst,$src" %}
- opcode(0x81, 0x00); /* /0 id */
- ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct incI_eReg(rRegI dst, immI_1 src, eFlagsReg cr) %{
- predicate(UseIncDec);
- match(Set dst (AddI dst src));
- effect(KILL cr);
-
- size(1);
- format %{ "INC $dst" %}
- opcode(0x40); /* */
- ins_encode( Opc_plus( primary, dst ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct leaI_eReg_immI(rRegI dst, rRegI src0, immI src1) %{
- match(Set dst (AddI src0 src1));
- ins_cost(110);
-
- format %{ "LEA $dst,[$src0 + $src1]" %}
- opcode(0x8D); /* 0x8D /r */
- ins_encode( SetInstMark, OpcP, RegLea( dst, src0, src1 ), ClearInstMark );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct leaP_eReg_immI(eRegP dst, eRegP src0, immI src1) %{
- match(Set dst (AddP src0 src1));
- ins_cost(110);
-
- format %{ "LEA $dst,[$src0 + $src1]\t# ptr" %}
- opcode(0x8D); /* 0x8D /r */
- ins_encode( SetInstMark, OpcP, RegLea( dst, src0, src1 ), ClearInstMark );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct decI_eReg(rRegI dst, immI_M1 src, eFlagsReg cr) %{
- predicate(UseIncDec);
- match(Set dst (AddI dst src));
- effect(KILL cr);
-
- size(1);
- format %{ "DEC $dst" %}
- opcode(0x48); /* */
- ins_encode( Opc_plus( primary, dst ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct addP_eReg(eRegP dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (AddP dst src));
- effect(KILL cr);
-
- size(2);
- format %{ "ADD $dst,$src" %}
- opcode(0x03);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct addP_eReg_imm(eRegP dst, immI src, eFlagsReg cr) %{
- match(Set dst (AddP dst src));
- effect(KILL cr);
-
- format %{ "ADD $dst,$src" %}
- opcode(0x81,0x00); /* Opcode 81 /0 id */
- // ins_encode( RegImm( dst, src) );
- ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct addI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
- match(Set dst (AddI dst (LoadI src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "ADD $dst,$src" %}
- opcode(0x03);
- ins_encode( SetInstMark, OpcP, RegMem( dst, src), ClearInstMark );
- ins_pipe( ialu_reg_mem );
-%}
-
-instruct addI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (AddI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "ADD $dst,$src" %}
- opcode(0x01); /* Opcode 01 /r */
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Add Memory with Immediate
-instruct addI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (AddI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "ADD $dst,$src" %}
- opcode(0x81); /* Opcode 81 /0 id */
- ins_encode( SetInstMark, OpcSE( src ), RMopc_Mem(0x00,dst), Con8or32(src), ClearInstMark );
- ins_pipe( ialu_mem_imm );
-%}
-
-instruct incI_mem(memory dst, immI_1 src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (AddI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "INC $dst" %}
- opcode(0xFF); /* Opcode FF /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,dst), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-instruct decI_mem(memory dst, immI_M1 src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (AddI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "DEC $dst" %}
- opcode(0xFF); /* Opcode FF /1 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x01,dst), ClearInstMark);
- ins_pipe( ialu_mem_imm );
-%}
-
-
-instruct checkCastPP( eRegP dst ) %{
- match(Set dst (CheckCastPP dst));
-
- size(0);
- format %{ "#checkcastPP of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_pipe( empty );
-%}
-
-instruct castPP( eRegP dst ) %{
- match(Set dst (CastPP dst));
- format %{ "#castPP of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_pipe( empty );
-%}
-
-instruct castII( rRegI dst ) %{
- match(Set dst (CastII dst));
- format %{ "#castII of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe( empty );
-%}
-
-instruct castLL( eRegL dst ) %{
- match(Set dst (CastLL dst));
- format %{ "#castLL of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe( empty );
-%}
-
-instruct castFF( regF dst ) %{
- predicate(UseSSE >= 1);
- match(Set dst (CastFF dst));
- format %{ "#castFF of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe( empty );
-%}
-
-instruct castDD( regD dst ) %{
- predicate(UseSSE >= 2);
- match(Set dst (CastDD dst));
- format %{ "#castDD of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe( empty );
-%}
-
-instruct castFF_PR( regFPR dst ) %{
- predicate(UseSSE < 1);
- match(Set dst (CastFF dst));
- format %{ "#castFF of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe( empty );
-%}
-
-instruct castDD_PR( regDPR dst ) %{
- predicate(UseSSE < 2);
- match(Set dst (CastDD dst));
- format %{ "#castDD of $dst" %}
- ins_encode( /*empty encoding*/ );
- ins_cost(0);
- ins_pipe( empty );
-%}
-
-// No flag versions for CompareAndSwap{P,I,L} because matcher can't match them
-
-instruct compareAndSwapL( rRegI res, eSIRegP mem_ptr, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
- match(Set res (CompareAndSwapL mem_ptr (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapL mem_ptr (Binary oldval newval)));
- effect(KILL cr, KILL oldval);
- format %{ "CMPXCHG8 [$mem_ptr],$newval\t# If EDX:EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
- "MOV $res,0\n\t"
- "JNE,s fail\n\t"
- "MOV $res,1\n"
- "fail:" %}
- ins_encode( enc_cmpxchg8(mem_ptr),
- enc_flags_ne_to_boolean(res) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndSwapP( rRegI res, pRegP mem_ptr, eAXRegP oldval, eCXRegP newval, eFlagsReg cr) %{
- match(Set res (CompareAndSwapP mem_ptr (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapP mem_ptr (Binary oldval newval)));
- effect(KILL cr, KILL oldval);
- format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
- "MOV $res,0\n\t"
- "JNE,s fail\n\t"
- "MOV $res,1\n"
- "fail:" %}
- ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndSwapB( rRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr ) %{
- match(Set res (CompareAndSwapB mem_ptr (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapB mem_ptr (Binary oldval newval)));
- effect(KILL cr, KILL oldval);
- format %{ "CMPXCHGB [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
- "MOV $res,0\n\t"
- "JNE,s fail\n\t"
- "MOV $res,1\n"
- "fail:" %}
- ins_encode( enc_cmpxchgb(mem_ptr),
- enc_flags_ne_to_boolean(res) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndSwapS( rRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr ) %{
- match(Set res (CompareAndSwapS mem_ptr (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapS mem_ptr (Binary oldval newval)));
- effect(KILL cr, KILL oldval);
- format %{ "CMPXCHGW [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
- "MOV $res,0\n\t"
- "JNE,s fail\n\t"
- "MOV $res,1\n"
- "fail:" %}
- ins_encode( enc_cmpxchgw(mem_ptr),
- enc_flags_ne_to_boolean(res) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndSwapI( rRegI res, pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
- match(Set res (CompareAndSwapI mem_ptr (Binary oldval newval)));
- match(Set res (WeakCompareAndSwapI mem_ptr (Binary oldval newval)));
- effect(KILL cr, KILL oldval);
- format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t"
- "MOV $res,0\n\t"
- "JNE,s fail\n\t"
- "MOV $res,1\n"
- "fail:" %}
- ins_encode( enc_cmpxchg(mem_ptr), enc_flags_ne_to_boolean(res) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndExchangeL( eSIRegP mem_ptr, eADXRegL oldval, eBCXRegL newval, eFlagsReg cr ) %{
- match(Set oldval (CompareAndExchangeL mem_ptr (Binary oldval newval)));
- effect(KILL cr);
- format %{ "CMPXCHG8 [$mem_ptr],$newval\t# If EDX:EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t" %}
- ins_encode( enc_cmpxchg8(mem_ptr) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndExchangeP( pRegP mem_ptr, eAXRegP oldval, eCXRegP newval, eFlagsReg cr) %{
- match(Set oldval (CompareAndExchangeP mem_ptr (Binary oldval newval)));
- effect(KILL cr);
- format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t" %}
- ins_encode( enc_cmpxchg(mem_ptr) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndExchangeB( pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
- match(Set oldval (CompareAndExchangeB mem_ptr (Binary oldval newval)));
- effect(KILL cr);
- format %{ "CMPXCHGB [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t" %}
- ins_encode( enc_cmpxchgb(mem_ptr) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndExchangeS( pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
- match(Set oldval (CompareAndExchangeS mem_ptr (Binary oldval newval)));
- effect(KILL cr);
- format %{ "CMPXCHGW [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t" %}
- ins_encode( enc_cmpxchgw(mem_ptr) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct compareAndExchangeI( pRegP mem_ptr, eAXRegI oldval, eCXRegI newval, eFlagsReg cr) %{
- match(Set oldval (CompareAndExchangeI mem_ptr (Binary oldval newval)));
- effect(KILL cr);
- format %{ "CMPXCHG [$mem_ptr],$newval\t# If EAX==[$mem_ptr] Then store $newval into [$mem_ptr]\n\t" %}
- ins_encode( enc_cmpxchg(mem_ptr) );
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xaddB_no_res( memory mem, Universe dummy, immI add, eFlagsReg cr) %{
- predicate(n->as_LoadStore()->result_not_used());
- match(Set dummy (GetAndAddB mem add));
- effect(KILL cr);
- format %{ "ADDB [$mem],$add" %}
- ins_encode %{
- __ lock();
- __ addb($mem$$Address, $add$$constant);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-// Important to match to xRegI: only 8-bit regs.
-instruct xaddB( memory mem, xRegI newval, eFlagsReg cr) %{
- match(Set newval (GetAndAddB mem newval));
- effect(KILL cr);
- format %{ "XADDB [$mem],$newval" %}
- ins_encode %{
- __ lock();
- __ xaddb($mem$$Address, $newval$$Register);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xaddS_no_res( memory mem, Universe dummy, immI add, eFlagsReg cr) %{
- predicate(n->as_LoadStore()->result_not_used());
- match(Set dummy (GetAndAddS mem add));
- effect(KILL cr);
- format %{ "ADDS [$mem],$add" %}
- ins_encode %{
- __ lock();
- __ addw($mem$$Address, $add$$constant);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xaddS( memory mem, rRegI newval, eFlagsReg cr) %{
- match(Set newval (GetAndAddS mem newval));
- effect(KILL cr);
- format %{ "XADDS [$mem],$newval" %}
- ins_encode %{
- __ lock();
- __ xaddw($mem$$Address, $newval$$Register);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xaddI_no_res( memory mem, Universe dummy, immI add, eFlagsReg cr) %{
- predicate(n->as_LoadStore()->result_not_used());
- match(Set dummy (GetAndAddI mem add));
- effect(KILL cr);
- format %{ "ADDL [$mem],$add" %}
- ins_encode %{
- __ lock();
- __ addl($mem$$Address, $add$$constant);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xaddI( memory mem, rRegI newval, eFlagsReg cr) %{
- match(Set newval (GetAndAddI mem newval));
- effect(KILL cr);
- format %{ "XADDL [$mem],$newval" %}
- ins_encode %{
- __ lock();
- __ xaddl($mem$$Address, $newval$$Register);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-// Important to match to xRegI: only 8-bit regs.
-instruct xchgB( memory mem, xRegI newval) %{
- match(Set newval (GetAndSetB mem newval));
- format %{ "XCHGB $newval,[$mem]" %}
- ins_encode %{
- __ xchgb($newval$$Register, $mem$$Address);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xchgS( memory mem, rRegI newval) %{
- match(Set newval (GetAndSetS mem newval));
- format %{ "XCHGW $newval,[$mem]" %}
- ins_encode %{
- __ xchgw($newval$$Register, $mem$$Address);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xchgI( memory mem, rRegI newval) %{
- match(Set newval (GetAndSetI mem newval));
- format %{ "XCHGL $newval,[$mem]" %}
- ins_encode %{
- __ xchgl($newval$$Register, $mem$$Address);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-instruct xchgP( memory mem, pRegP newval) %{
- match(Set newval (GetAndSetP mem newval));
- format %{ "XCHGL $newval,[$mem]" %}
- ins_encode %{
- __ xchgl($newval$$Register, $mem$$Address);
- %}
- ins_pipe( pipe_cmpxchg );
-%}
-
-//----------Subtraction Instructions-------------------------------------------
-
-// Integer Subtraction Instructions
-instruct subI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (SubI dst src));
- effect(KILL cr);
-
- size(2);
- format %{ "SUB $dst,$src" %}
- opcode(0x2B);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct subI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
- match(Set dst (SubI dst src));
- effect(KILL cr);
-
- format %{ "SUB $dst,$src" %}
- opcode(0x81,0x05); /* Opcode 81 /5 */
- // ins_encode( RegImm( dst, src) );
- ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct subI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
- match(Set dst (SubI dst (LoadI src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "SUB $dst,$src" %}
- opcode(0x2B);
- ins_encode( SetInstMark, OpcP, RegMem( dst, src), ClearInstMark );
- ins_pipe( ialu_reg_mem );
-%}
-
-instruct subI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (SubI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "SUB $dst,$src" %}
- opcode(0x29); /* Opcode 29 /r */
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Subtract from a pointer
-instruct subP_eReg(eRegP dst, rRegI src, immI_0 zero, eFlagsReg cr) %{
- match(Set dst (AddP dst (SubI zero src)));
- effect(KILL cr);
-
- size(2);
- format %{ "SUB $dst,$src" %}
- opcode(0x2B);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct negI_eReg(rRegI dst, immI_0 zero, eFlagsReg cr) %{
- match(Set dst (SubI zero dst));
- effect(KILL cr);
-
- size(2);
- format %{ "NEG $dst" %}
- opcode(0xF7,0x03); // Opcode F7 /3
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg );
-%}
-
-//----------Multiplication/Division Instructions-------------------------------
-// Integer Multiplication Instructions
-// Multiply Register
-instruct mulI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (MulI dst src));
- effect(KILL cr);
-
- size(3);
- ins_cost(300);
- format %{ "IMUL $dst,$src" %}
- opcode(0xAF, 0x0F);
- ins_encode( OpcS, OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg_alu0 );
-%}
-
-// Multiply 32-bit Immediate
-instruct mulI_eReg_imm(rRegI dst, rRegI src, immI imm, eFlagsReg cr) %{
- match(Set dst (MulI src imm));
- effect(KILL cr);
-
- ins_cost(300);
- format %{ "IMUL $dst,$src,$imm" %}
- opcode(0x69); /* 69 /r id */
- ins_encode( OpcSE(imm), RegReg( dst, src ), Con8or32( imm ) );
- ins_pipe( ialu_reg_reg_alu0 );
-%}
-
-instruct loadConL_low_only(eADXRegL_low_only dst, immL32 src, eFlagsReg cr) %{
- match(Set dst src);
- effect(KILL cr);
-
- // Note that this is artificially increased to make it more expensive than loadConL
- ins_cost(250);
- format %{ "MOV EAX,$src\t// low word only" %}
- opcode(0xB8);
- ins_encode( LdImmL_Lo(dst, src) );
- ins_pipe( ialu_reg_fat );
-%}
-
-// Multiply by 32-bit Immediate, taking the shifted high order results
-// (special case for shift by 32)
-instruct mulI_imm_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32 cnt, eFlagsReg cr) %{
- match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
- predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
- _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
- _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
- effect(USE src1, KILL cr);
-
- // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
- ins_cost(0*100 + 1*400 - 150);
- format %{ "IMUL EDX:EAX,$src1" %}
- ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
- ins_pipe( pipe_slow );
-%}
-
-// Multiply by 32-bit Immediate, taking the shifted high order results
-instruct mulI_imm_RShift_high(eDXRegI dst, nadxRegI src1, eADXRegL_low_only src2, immI_32_63 cnt, eFlagsReg cr) %{
- match(Set dst (ConvL2I (RShiftL (MulL (ConvI2L src1) src2) cnt)));
- predicate( _kids[0]->_kids[0]->_kids[1]->_leaf->Opcode() == Op_ConL &&
- _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() >= min_jint &&
- _kids[0]->_kids[0]->_kids[1]->_leaf->as_Type()->type()->is_long()->get_con() <= max_jint );
- effect(USE src1, KILL cr);
-
- // Note that this is adjusted by 150 to compensate for the overcosting of loadConL_low_only
- ins_cost(1*100 + 1*400 - 150);
- format %{ "IMUL EDX:EAX,$src1\n\t"
- "SAR EDX,$cnt-32" %}
- ins_encode( multiply_con_and_shift_high( dst, src1, src2, cnt, cr ) );
- ins_pipe( pipe_slow );
-%}
-
-// Multiply Memory 32-bit Immediate
-instruct mulI_mem_imm(rRegI dst, memory src, immI imm, eFlagsReg cr) %{
- match(Set dst (MulI (LoadI src) imm));
- effect(KILL cr);
-
- ins_cost(300);
- format %{ "IMUL $dst,$src,$imm" %}
- opcode(0x69); /* 69 /r id */
- ins_encode( SetInstMark, OpcSE(imm), RegMem( dst, src ), Con8or32( imm ), ClearInstMark );
- ins_pipe( ialu_reg_mem_alu0 );
-%}
-
-// Multiply Memory
-instruct mulI(rRegI dst, memory src, eFlagsReg cr) %{
- match(Set dst (MulI dst (LoadI src)));
- effect(KILL cr);
-
- ins_cost(350);
- format %{ "IMUL $dst,$src" %}
- opcode(0xAF, 0x0F);
- ins_encode( SetInstMark, OpcS, OpcP, RegMem( dst, src), ClearInstMark );
- ins_pipe( ialu_reg_mem_alu0 );
-%}
-
-instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, eFlagsReg cr)
-%{
- match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
- effect(KILL cr, KILL src2);
-
- expand %{ mulI_eReg(dst, src1, cr);
- mulI_eReg(src2, src3, cr);
- addI_eReg(dst, src2, cr); %}
-%}
-
-// Multiply Register Int to Long
-instruct mulI2L(eADXRegL dst, eAXRegI src, nadxRegI src1, eFlagsReg flags) %{
- // Basic Idea: long = (long)int * (long)int
- match(Set dst (MulL (ConvI2L src) (ConvI2L src1)));
- effect(DEF dst, USE src, USE src1, KILL flags);
-
- ins_cost(300);
- format %{ "IMUL $dst,$src1" %}
-
- ins_encode( long_int_multiply( dst, src1 ) );
- ins_pipe( ialu_reg_reg_alu0 );
-%}
-
-instruct mulIS_eReg(eADXRegL dst, immL_32bits mask, eFlagsReg flags, eAXRegI src, nadxRegI src1) %{
- // Basic Idea: long = (int & 0xffffffffL) * (int & 0xffffffffL)
- match(Set dst (MulL (AndL (ConvI2L src) mask) (AndL (ConvI2L src1) mask)));
- effect(KILL flags);
-
- ins_cost(300);
- format %{ "MUL $dst,$src1" %}
-
- ins_encode( long_uint_multiply(dst, src1) );
- ins_pipe( ialu_reg_reg_alu0 );
-%}
-
-// Multiply Register Long
-instruct mulL_eReg(eADXRegL dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
- match(Set dst (MulL dst src));
- effect(KILL cr, TEMP tmp);
- ins_cost(4*100+3*400);
-// Basic idea: lo(result) = lo(x_lo * y_lo)
-// hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) + lo(x_lo * y_hi)
- format %{ "MOV $tmp,$src.lo\n\t"
- "IMUL $tmp,EDX\n\t"
- "MOV EDX,$src.hi\n\t"
- "IMUL EDX,EAX\n\t"
- "ADD $tmp,EDX\n\t"
- "MUL EDX:EAX,$src.lo\n\t"
- "ADD EDX,$tmp" %}
- ins_encode( long_multiply( dst, src, tmp ) );
- ins_pipe( pipe_slow );
-%}
-
-// Multiply Register Long where the left operand's high 32 bits are zero
-instruct mulL_eReg_lhi0(eADXRegL dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
- predicate(is_operand_hi32_zero(n->in(1)));
- match(Set dst (MulL dst src));
- effect(KILL cr, TEMP tmp);
- ins_cost(2*100+2*400);
-// Basic idea: lo(result) = lo(x_lo * y_lo)
-// hi(result) = hi(x_lo * y_lo) + lo(x_lo * y_hi) where lo(x_hi * y_lo) = 0 because x_hi = 0
- format %{ "MOV $tmp,$src.hi\n\t"
- "IMUL $tmp,EAX\n\t"
- "MUL EDX:EAX,$src.lo\n\t"
- "ADD EDX,$tmp" %}
- ins_encode %{
- __ movl($tmp$$Register, HIGH_FROM_LOW($src$$Register));
- __ imull($tmp$$Register, rax);
- __ mull($src$$Register);
- __ addl(rdx, $tmp$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Multiply Register Long where the right operand's high 32 bits are zero
-instruct mulL_eReg_rhi0(eADXRegL dst, eRegL src, rRegI tmp, eFlagsReg cr) %{
- predicate(is_operand_hi32_zero(n->in(2)));
- match(Set dst (MulL dst src));
- effect(KILL cr, TEMP tmp);
- ins_cost(2*100+2*400);
-// Basic idea: lo(result) = lo(x_lo * y_lo)
-// hi(result) = hi(x_lo * y_lo) + lo(x_hi * y_lo) where lo(x_lo * y_hi) = 0 because y_hi = 0
- format %{ "MOV $tmp,$src.lo\n\t"
- "IMUL $tmp,EDX\n\t"
- "MUL EDX:EAX,$src.lo\n\t"
- "ADD EDX,$tmp" %}
- ins_encode %{
- __ movl($tmp$$Register, $src$$Register);
- __ imull($tmp$$Register, rdx);
- __ mull($src$$Register);
- __ addl(rdx, $tmp$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Multiply Register Long where the left and the right operands' high 32 bits are zero
-instruct mulL_eReg_hi0(eADXRegL dst, eRegL src, eFlagsReg cr) %{
- predicate(is_operand_hi32_zero(n->in(1)) && is_operand_hi32_zero(n->in(2)));
- match(Set dst (MulL dst src));
- effect(KILL cr);
- ins_cost(1*400);
-// Basic idea: lo(result) = lo(x_lo * y_lo)
-// hi(result) = hi(x_lo * y_lo) where lo(x_hi * y_lo) = 0 and lo(x_lo * y_hi) = 0 because x_hi = 0 and y_hi = 0
- format %{ "MUL EDX:EAX,$src.lo\n\t" %}
- ins_encode %{
- __ mull($src$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Multiply Register Long by small constant
-instruct mulL_eReg_con(eADXRegL dst, immL_127 src, rRegI tmp, eFlagsReg cr) %{
- match(Set dst (MulL dst src));
- effect(KILL cr, TEMP tmp);
- ins_cost(2*100+2*400);
- size(12);
-// Basic idea: lo(result) = lo(src * EAX)
-// hi(result) = hi(src * EAX) + lo(src * EDX)
- format %{ "IMUL $tmp,EDX,$src\n\t"
- "MOV EDX,$src\n\t"
- "MUL EDX\t# EDX*EAX -> EDX:EAX\n\t"
- "ADD EDX,$tmp" %}
- ins_encode( long_multiply_con( dst, src, tmp ) );
- ins_pipe( pipe_slow );
-%}
-
-// Integer DIV with Register
-instruct divI_eReg(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
- match(Set rax (DivI rax div));
- effect(KILL rdx, KILL cr);
- size(26);
- ins_cost(30*100+10*100);
- format %{ "CMP EAX,0x80000000\n\t"
- "JNE,s normal\n\t"
- "XOR EDX,EDX\n\t"
- "CMP ECX,-1\n\t"
- "JE,s done\n"
- "normal: CDQ\n\t"
- "IDIV $div\n\t"
- "done:" %}
- opcode(0xF7, 0x7); /* Opcode F7 /7 */
- ins_encode( cdq_enc, OpcP, RegOpc(div) );
- ins_pipe( ialu_reg_reg_alu0 );
-%}
-
-// Divide Register Long
-instruct divL_eReg(eADXRegL dst, eRegL src1, eRegL src2) %{
- match(Set dst (DivL src1 src2));
- effect(CALL);
- ins_cost(10000);
- format %{ "PUSH $src1.hi\n\t"
- "PUSH $src1.lo\n\t"
- "PUSH $src2.hi\n\t"
- "PUSH $src2.lo\n\t"
- "CALL SharedRuntime::ldiv\n\t"
- "ADD ESP,16" %}
- ins_encode( long_div(src1,src2) );
- ins_pipe( pipe_slow );
-%}
-
-// Integer DIVMOD with Register, both quotient and mod results
-instruct divModI_eReg_divmod(eAXRegI rax, eDXRegI rdx, eCXRegI div, eFlagsReg cr) %{
- match(DivModI rax div);
- effect(KILL cr);
- size(26);
- ins_cost(30*100+10*100);
- format %{ "CMP EAX,0x80000000\n\t"
- "JNE,s normal\n\t"
- "XOR EDX,EDX\n\t"
- "CMP ECX,-1\n\t"
- "JE,s done\n"
- "normal: CDQ\n\t"
- "IDIV $div\n\t"
- "done:" %}
- opcode(0xF7, 0x7); /* Opcode F7 /7 */
- ins_encode( cdq_enc, OpcP, RegOpc(div) );
- ins_pipe( pipe_slow );
-%}
-
-// Integer MOD with Register
-instruct modI_eReg(eDXRegI rdx, eAXRegI rax, eCXRegI div, eFlagsReg cr) %{
- match(Set rdx (ModI rax div));
- effect(KILL rax, KILL cr);
-
- size(26);
- ins_cost(300);
- format %{ "CDQ\n\t"
- "IDIV $div" %}
- opcode(0xF7, 0x7); /* Opcode F7 /7 */
- ins_encode( cdq_enc, OpcP, RegOpc(div) );
- ins_pipe( ialu_reg_reg_alu0 );
-%}
-
-// Remainder Register Long
-instruct modL_eReg(eADXRegL dst, eRegL src1, eRegL src2) %{
- match(Set dst (ModL src1 src2));
- effect(CALL);
- ins_cost(10000);
- format %{ "PUSH $src1.hi\n\t"
- "PUSH $src1.lo\n\t"
- "PUSH $src2.hi\n\t"
- "PUSH $src2.lo\n\t"
- "CALL SharedRuntime::lrem\n\t"
- "ADD ESP,16" %}
- ins_encode( long_mod(src1,src2) );
- ins_pipe( pipe_slow );
-%}
-
-// Divide Register Long (no special case since divisor != -1)
-instruct divL_eReg_imm32( eADXRegL dst, immL32 imm, rRegI tmp, rRegI tmp2, eFlagsReg cr ) %{
- match(Set dst (DivL dst imm));
- effect( TEMP tmp, TEMP tmp2, KILL cr );
- ins_cost(1000);
- format %{ "MOV $tmp,abs($imm) # ldiv EDX:EAX,$imm\n\t"
- "XOR $tmp2,$tmp2\n\t"
- "CMP $tmp,EDX\n\t"
- "JA,s fast\n\t"
- "MOV $tmp2,EAX\n\t"
- "MOV EAX,EDX\n\t"
- "MOV EDX,0\n\t"
- "JLE,s pos\n\t"
- "LNEG EAX : $tmp2\n\t"
- "DIV $tmp # unsigned division\n\t"
- "XCHG EAX,$tmp2\n\t"
- "DIV $tmp\n\t"
- "LNEG $tmp2 : EAX\n\t"
- "JMP,s done\n"
- "pos:\n\t"
- "DIV $tmp\n\t"
- "XCHG EAX,$tmp2\n"
- "fast:\n\t"
- "DIV $tmp\n"
- "done:\n\t"
- "MOV EDX,$tmp2\n\t"
- "NEG EDX:EAX # if $imm < 0" %}
- ins_encode %{
- int con = (int)$imm$$constant;
- assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
- int pcon = (con > 0) ? con : -con;
- Label Lfast, Lpos, Ldone;
-
- __ movl($tmp$$Register, pcon);
- __ xorl($tmp2$$Register,$tmp2$$Register);
- __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
- __ jccb(Assembler::above, Lfast); // result fits into 32 bit
-
- __ movl($tmp2$$Register, $dst$$Register); // save
- __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
- __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
- __ jccb(Assembler::lessEqual, Lpos); // result is positive
-
- // Negative dividend.
- // convert value to positive to use unsigned division
- __ lneg($dst$$Register, $tmp2$$Register);
- __ divl($tmp$$Register);
- __ xchgl($dst$$Register, $tmp2$$Register);
- __ divl($tmp$$Register);
- // revert result back to negative
- __ lneg($tmp2$$Register, $dst$$Register);
- __ jmpb(Ldone);
-
- __ bind(Lpos);
- __ divl($tmp$$Register); // Use unsigned division
- __ xchgl($dst$$Register, $tmp2$$Register);
- // Fallthrow for final divide, tmp2 has 32 bit hi result
-
- __ bind(Lfast);
- // fast path: src is positive
- __ divl($tmp$$Register); // Use unsigned division
-
- __ bind(Ldone);
- __ movl(HIGH_FROM_LOW($dst$$Register),$tmp2$$Register);
- if (con < 0) {
- __ lneg(HIGH_FROM_LOW($dst$$Register), $dst$$Register);
- }
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Remainder Register Long (remainder fit into 32 bits)
-instruct modL_eReg_imm32( eADXRegL dst, immL32 imm, rRegI tmp, rRegI tmp2, eFlagsReg cr ) %{
- match(Set dst (ModL dst imm));
- effect( TEMP tmp, TEMP tmp2, KILL cr );
- ins_cost(1000);
- format %{ "MOV $tmp,abs($imm) # lrem EDX:EAX,$imm\n\t"
- "CMP $tmp,EDX\n\t"
- "JA,s fast\n\t"
- "MOV $tmp2,EAX\n\t"
- "MOV EAX,EDX\n\t"
- "MOV EDX,0\n\t"
- "JLE,s pos\n\t"
- "LNEG EAX : $tmp2\n\t"
- "DIV $tmp # unsigned division\n\t"
- "MOV EAX,$tmp2\n\t"
- "DIV $tmp\n\t"
- "NEG EDX\n\t"
- "JMP,s done\n"
- "pos:\n\t"
- "DIV $tmp\n\t"
- "MOV EAX,$tmp2\n"
- "fast:\n\t"
- "DIV $tmp\n"
- "done:\n\t"
- "MOV EAX,EDX\n\t"
- "SAR EDX,31\n\t" %}
- ins_encode %{
- int con = (int)$imm$$constant;
- assert(con != 0 && con != -1 && con != min_jint, "wrong divisor");
- int pcon = (con > 0) ? con : -con;
- Label Lfast, Lpos, Ldone;
-
- __ movl($tmp$$Register, pcon);
- __ cmpl($tmp$$Register, HIGH_FROM_LOW($dst$$Register));
- __ jccb(Assembler::above, Lfast); // src is positive and result fits into 32 bit
-
- __ movl($tmp2$$Register, $dst$$Register); // save
- __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
- __ movl(HIGH_FROM_LOW($dst$$Register),0); // preserve flags
- __ jccb(Assembler::lessEqual, Lpos); // result is positive
-
- // Negative dividend.
- // convert value to positive to use unsigned division
- __ lneg($dst$$Register, $tmp2$$Register);
- __ divl($tmp$$Register);
- __ movl($dst$$Register, $tmp2$$Register);
- __ divl($tmp$$Register);
- // revert remainder back to negative
- __ negl(HIGH_FROM_LOW($dst$$Register));
- __ jmpb(Ldone);
-
- __ bind(Lpos);
- __ divl($tmp$$Register);
- __ movl($dst$$Register, $tmp2$$Register);
-
- __ bind(Lfast);
- // fast path: src is positive
- __ divl($tmp$$Register);
-
- __ bind(Ldone);
- __ movl($dst$$Register, HIGH_FROM_LOW($dst$$Register));
- __ sarl(HIGH_FROM_LOW($dst$$Register), 31); // result sign
-
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Integer Shift Instructions
-// Shift Left by one
-instruct shlI_eReg_1(rRegI dst, immI_1 shift, eFlagsReg cr) %{
- match(Set dst (LShiftI dst shift));
- effect(KILL cr);
-
- size(2);
- format %{ "SHL $dst,$shift" %}
- opcode(0xD1, 0x4); /* D1 /4 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg );
-%}
-
-// Shift Left by 8-bit immediate
-instruct salI_eReg_imm(rRegI dst, immI8 shift, eFlagsReg cr) %{
- match(Set dst (LShiftI dst shift));
- effect(KILL cr);
-
- size(3);
- format %{ "SHL $dst,$shift" %}
- opcode(0xC1, 0x4); /* C1 /4 ib */
- ins_encode( RegOpcImm( dst, shift) );
- ins_pipe( ialu_reg );
-%}
-
-// Shift Left by variable
-instruct salI_eReg_CL(rRegI dst, eCXRegI shift, eFlagsReg cr) %{
- match(Set dst (LShiftI dst shift));
- effect(KILL cr);
-
- size(2);
- format %{ "SHL $dst,$shift" %}
- opcode(0xD3, 0x4); /* D3 /4 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg_reg );
-%}
-
-// Arithmetic shift right by one
-instruct sarI_eReg_1(rRegI dst, immI_1 shift, eFlagsReg cr) %{
- match(Set dst (RShiftI dst shift));
- effect(KILL cr);
-
- size(2);
- format %{ "SAR $dst,$shift" %}
- opcode(0xD1, 0x7); /* D1 /7 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg );
-%}
-
-// Arithmetic shift right by one
-instruct sarI_mem_1(memory dst, immI_1 shift, eFlagsReg cr) %{
- match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
- effect(KILL cr);
- format %{ "SAR $dst,$shift" %}
- opcode(0xD1, 0x7); /* D1 /7 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(secondary,dst), ClearInstMark );
- ins_pipe( ialu_mem_imm );
-%}
-
-// Arithmetic Shift Right by 8-bit immediate
-instruct sarI_eReg_imm(rRegI dst, immI8 shift, eFlagsReg cr) %{
- match(Set dst (RShiftI dst shift));
- effect(KILL cr);
-
- size(3);
- format %{ "SAR $dst,$shift" %}
- opcode(0xC1, 0x7); /* C1 /7 ib */
- ins_encode( RegOpcImm( dst, shift ) );
- ins_pipe( ialu_mem_imm );
-%}
-
-// Arithmetic Shift Right by 8-bit immediate
-instruct sarI_mem_imm(memory dst, immI8 shift, eFlagsReg cr) %{
- match(Set dst (StoreI dst (RShiftI (LoadI dst) shift)));
- effect(KILL cr);
-
- format %{ "SAR $dst,$shift" %}
- opcode(0xC1, 0x7); /* C1 /7 ib */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(secondary, dst ), Con8or32(shift), ClearInstMark );
- ins_pipe( ialu_mem_imm );
-%}
-
-// Arithmetic Shift Right by variable
-instruct sarI_eReg_CL(rRegI dst, eCXRegI shift, eFlagsReg cr) %{
- match(Set dst (RShiftI dst shift));
- effect(KILL cr);
-
- size(2);
- format %{ "SAR $dst,$shift" %}
- opcode(0xD3, 0x7); /* D3 /7 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg_reg );
-%}
-
-// Logical shift right by one
-instruct shrI_eReg_1(rRegI dst, immI_1 shift, eFlagsReg cr) %{
- match(Set dst (URShiftI dst shift));
- effect(KILL cr);
-
- size(2);
- format %{ "SHR $dst,$shift" %}
- opcode(0xD1, 0x5); /* D1 /5 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg );
-%}
-
-// Logical Shift Right by 8-bit immediate
-instruct shrI_eReg_imm(rRegI dst, immI8 shift, eFlagsReg cr) %{
- match(Set dst (URShiftI dst shift));
- effect(KILL cr);
-
- size(3);
- format %{ "SHR $dst,$shift" %}
- opcode(0xC1, 0x5); /* C1 /5 ib */
- ins_encode( RegOpcImm( dst, shift) );
- ins_pipe( ialu_reg );
-%}
-
-
-// Logical Shift Right by 24, followed by Arithmetic Shift Left by 24.
-// This idiom is used by the compiler for the i2b bytecode.
-instruct i2b(rRegI dst, xRegI src, immI_24 twentyfour) %{
- match(Set dst (RShiftI (LShiftI src twentyfour) twentyfour));
-
- size(3);
- format %{ "MOVSX $dst,$src :8" %}
- ins_encode %{
- __ movsbl($dst$$Register, $src$$Register);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-// Logical Shift Right by 16, followed by Arithmetic Shift Left by 16.
-// This idiom is used by the compiler the i2s bytecode.
-instruct i2s(rRegI dst, xRegI src, immI_16 sixteen) %{
- match(Set dst (RShiftI (LShiftI src sixteen) sixteen));
-
- size(3);
- format %{ "MOVSX $dst,$src :16" %}
- ins_encode %{
- __ movswl($dst$$Register, $src$$Register);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-
-// Logical Shift Right by variable
-instruct shrI_eReg_CL(rRegI dst, eCXRegI shift, eFlagsReg cr) %{
- match(Set dst (URShiftI dst shift));
- effect(KILL cr);
-
- size(2);
- format %{ "SHR $dst,$shift" %}
- opcode(0xD3, 0x5); /* D3 /5 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg_reg );
-%}
-
-
-//----------Logical Instructions-----------------------------------------------
-//----------Integer Logical Instructions---------------------------------------
-// And Instructions
-// And Register with Register
-instruct andI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (AndI dst src));
- effect(KILL cr);
-
- size(2);
- format %{ "AND $dst,$src" %}
- opcode(0x23);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-// And Register with Immediate
-instruct andI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
- match(Set dst (AndI dst src));
- effect(KILL cr);
-
- format %{ "AND $dst,$src" %}
- opcode(0x81,0x04); /* Opcode 81 /4 */
- // ins_encode( RegImm( dst, src) );
- ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
- ins_pipe( ialu_reg );
-%}
-
-// And Register with Memory
-instruct andI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
- match(Set dst (AndI dst (LoadI src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "AND $dst,$src" %}
- opcode(0x23);
- ins_encode( SetInstMark, OpcP, RegMem( dst, src), ClearInstMark );
- ins_pipe( ialu_reg_mem );
-%}
-
-// And Memory with Register
-instruct andI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (AndI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "AND $dst,$src" %}
- opcode(0x21); /* Opcode 21 /r */
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// And Memory with Immediate
-instruct andI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (AndI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "AND $dst,$src" %}
- opcode(0x81, 0x4); /* Opcode 81 /4 id */
- // ins_encode( MemImm( dst, src) );
- ins_encode( SetInstMark, OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32(src), ClearInstMark );
- ins_pipe( ialu_mem_imm );
-%}
-
-// BMI1 instructions
-instruct andnI_rReg_rReg_rReg(rRegI dst, rRegI src1, rRegI src2, immI_M1 minus_1, eFlagsReg cr) %{
- match(Set dst (AndI (XorI src1 minus_1) src2));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- format %{ "ANDNL $dst, $src1, $src2" %}
-
- ins_encode %{
- __ andnl($dst$$Register, $src1$$Register, $src2$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct andnI_rReg_rReg_mem(rRegI dst, rRegI src1, memory src2, immI_M1 minus_1, eFlagsReg cr) %{
- match(Set dst (AndI (XorI src1 minus_1) (LoadI src2) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "ANDNL $dst, $src1, $src2" %}
-
- ins_encode %{
- __ andnl($dst$$Register, $src1$$Register, $src2$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct blsiI_rReg_rReg(rRegI dst, rRegI src, immI_0 imm_zero, eFlagsReg cr) %{
- match(Set dst (AndI (SubI imm_zero src) src));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- format %{ "BLSIL $dst, $src" %}
-
- ins_encode %{
- __ blsil($dst$$Register, $src$$Register);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct blsiI_rReg_mem(rRegI dst, memory src, immI_0 imm_zero, eFlagsReg cr) %{
- match(Set dst (AndI (SubI imm_zero (LoadI src) ) (LoadI src) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "BLSIL $dst, $src" %}
-
- ins_encode %{
- __ blsil($dst$$Register, $src$$Address);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct blsmskI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (XorI (AddI src minus_1) src));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- format %{ "BLSMSKL $dst, $src" %}
-
- ins_encode %{
- __ blsmskl($dst$$Register, $src$$Register);
- %}
-
- ins_pipe(ialu_reg);
-%}
-
-instruct blsmskI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (XorI (AddI (LoadI src) minus_1) (LoadI src) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "BLSMSKL $dst, $src" %}
-
- ins_encode %{
- __ blsmskl($dst$$Register, $src$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct blsrI_rReg_rReg(rRegI dst, rRegI src, immI_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (AndI (AddI src minus_1) src) );
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- format %{ "BLSRL $dst, $src" %}
-
- ins_encode %{
- __ blsrl($dst$$Register, $src$$Register);
- %}
-
- ins_pipe(ialu_reg);
-%}
-
-instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (AndI (AddI (LoadI src) minus_1) (LoadI src) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "BLSRL $dst, $src" %}
-
- ins_encode %{
- __ blsrl($dst$$Register, $src$$Address);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Or Instructions
-// Or Register with Register
-instruct orI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (OrI dst src));
- effect(KILL cr);
-
- size(2);
- format %{ "OR $dst,$src" %}
- opcode(0x0B);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct orI_eReg_castP2X(rRegI dst, eRegP src, eFlagsReg cr) %{
- match(Set dst (OrI dst (CastP2X src)));
- effect(KILL cr);
-
- size(2);
- format %{ "OR $dst,$src" %}
- opcode(0x0B);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-
-// Or Register with Immediate
-instruct orI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
- match(Set dst (OrI dst src));
- effect(KILL cr);
-
- format %{ "OR $dst,$src" %}
- opcode(0x81,0x01); /* Opcode 81 /1 id */
- // ins_encode( RegImm( dst, src) );
- ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
- ins_pipe( ialu_reg );
-%}
-
-// Or Register with Memory
-instruct orI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
- match(Set dst (OrI dst (LoadI src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "OR $dst,$src" %}
- opcode(0x0B);
- ins_encode( SetInstMark, OpcP, RegMem( dst, src), ClearInstMark );
- ins_pipe( ialu_reg_mem );
-%}
-
-// Or Memory with Register
-instruct orI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (OrI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "OR $dst,$src" %}
- opcode(0x09); /* Opcode 09 /r */
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Or Memory with Immediate
-instruct orI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (OrI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "OR $dst,$src" %}
- opcode(0x81,0x1); /* Opcode 81 /1 id */
- // ins_encode( MemImm( dst, src) );
- ins_encode( SetInstMark, OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32(src), ClearInstMark );
- ins_pipe( ialu_mem_imm );
-%}
-
-// ROL/ROR
-// ROL expand
-instruct rolI_eReg_imm1(rRegI dst, immI_1 shift, eFlagsReg cr) %{
- effect(USE_DEF dst, USE shift, KILL cr);
-
- format %{ "ROL $dst, $shift" %}
- opcode(0xD1, 0x0); /* Opcode D1 /0 */
- ins_encode( OpcP, RegOpc( dst ));
- ins_pipe( ialu_reg );
-%}
-
-instruct rolI_eReg_imm8(rRegI dst, immI8 shift, eFlagsReg cr) %{
- effect(USE_DEF dst, USE shift, KILL cr);
-
- format %{ "ROL $dst, $shift" %}
- opcode(0xC1, 0x0); /*Opcode /C1 /0 */
- ins_encode( RegOpcImm(dst, shift) );
- ins_pipe(ialu_reg);
-%}
-
-instruct rolI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr) %{
- effect(USE_DEF dst, USE shift, KILL cr);
-
- format %{ "ROL $dst, $shift" %}
- opcode(0xD3, 0x0); /* Opcode D3 /0 */
- ins_encode(OpcP, RegOpc(dst));
- ins_pipe( ialu_reg_reg );
-%}
-// end of ROL expand
-
-// ROL 32bit by one once
-instruct rolI_eReg_i1(rRegI dst, immI_1 lshift, immI_M1 rshift, eFlagsReg cr) %{
- match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
-
- expand %{
- rolI_eReg_imm1(dst, lshift, cr);
- %}
-%}
-
-// ROL 32bit var by imm8 once
-instruct rolI_eReg_i8(rRegI dst, immI8 lshift, immI8 rshift, eFlagsReg cr) %{
- predicate( 0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
- match(Set dst ( OrI (LShiftI dst lshift) (URShiftI dst rshift)));
-
- expand %{
- rolI_eReg_imm8(dst, lshift, cr);
- %}
-%}
-
-// ROL 32bit var by var once
-instruct rolI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI_0 zero, eFlagsReg cr) %{
- match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI zero shift))));
-
- expand %{
- rolI_eReg_CL(dst, shift, cr);
- %}
-%}
-
-// ROL 32bit var by var once
-instruct rolI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
- match(Set dst ( OrI (LShiftI dst shift) (URShiftI dst (SubI c32 shift))));
-
- expand %{
- rolI_eReg_CL(dst, shift, cr);
- %}
-%}
-
-// ROR expand
-instruct rorI_eReg_imm1(rRegI dst, immI_1 shift, eFlagsReg cr) %{
- effect(USE_DEF dst, USE shift, KILL cr);
-
- format %{ "ROR $dst, $shift" %}
- opcode(0xD1,0x1); /* Opcode D1 /1 */
- ins_encode( OpcP, RegOpc( dst ) );
- ins_pipe( ialu_reg );
-%}
-
-instruct rorI_eReg_imm8(rRegI dst, immI8 shift, eFlagsReg cr) %{
- effect (USE_DEF dst, USE shift, KILL cr);
-
- format %{ "ROR $dst, $shift" %}
- opcode(0xC1, 0x1); /* Opcode /C1 /1 ib */
- ins_encode( RegOpcImm(dst, shift) );
- ins_pipe( ialu_reg );
-%}
-
-instruct rorI_eReg_CL(ncxRegI dst, eCXRegI shift, eFlagsReg cr)%{
- effect(USE_DEF dst, USE shift, KILL cr);
-
- format %{ "ROR $dst, $shift" %}
- opcode(0xD3, 0x1); /* Opcode D3 /1 */
- ins_encode(OpcP, RegOpc(dst));
- ins_pipe( ialu_reg_reg );
-%}
-// end of ROR expand
-
-// ROR right once
-instruct rorI_eReg_i1(rRegI dst, immI_1 rshift, immI_M1 lshift, eFlagsReg cr) %{
- match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
-
- expand %{
- rorI_eReg_imm1(dst, rshift, cr);
- %}
-%}
-
-// ROR 32bit by immI8 once
-instruct rorI_eReg_i8(rRegI dst, immI8 rshift, immI8 lshift, eFlagsReg cr) %{
- predicate( 0 == ((n->in(1)->in(2)->get_int() + n->in(2)->in(2)->get_int()) & 0x1f));
- match(Set dst ( OrI (URShiftI dst rshift) (LShiftI dst lshift)));
-
- expand %{
- rorI_eReg_imm8(dst, rshift, cr);
- %}
-%}
-
-// ROR 32bit var by var once
-instruct rorI_eReg_Var_C0(ncxRegI dst, eCXRegI shift, immI_0 zero, eFlagsReg cr) %{
- match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI zero shift))));
-
- expand %{
- rorI_eReg_CL(dst, shift, cr);
- %}
-%}
-
-// ROR 32bit var by var once
-instruct rorI_eReg_Var_C32(ncxRegI dst, eCXRegI shift, immI_32 c32, eFlagsReg cr) %{
- match(Set dst ( OrI (URShiftI dst shift) (LShiftI dst (SubI c32 shift))));
-
- expand %{
- rorI_eReg_CL(dst, shift, cr);
- %}
-%}
-
-// Xor Instructions
-// Xor Register with Register
-instruct xorI_eReg(rRegI dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (XorI dst src));
- effect(KILL cr);
-
- size(2);
- format %{ "XOR $dst,$src" %}
- opcode(0x33);
- ins_encode( OpcP, RegReg( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-// Xor Register with Immediate -1
-instruct xorI_eReg_im1(rRegI dst, immI_M1 imm) %{
- match(Set dst (XorI dst imm));
-
- size(2);
- format %{ "NOT $dst" %}
- ins_encode %{
- __ notl($dst$$Register);
- %}
- ins_pipe( ialu_reg );
-%}
-
-// Xor Register with Immediate
-instruct xorI_eReg_imm(rRegI dst, immI src, eFlagsReg cr) %{
- match(Set dst (XorI dst src));
- effect(KILL cr);
-
- format %{ "XOR $dst,$src" %}
- opcode(0x81,0x06); /* Opcode 81 /6 id */
- // ins_encode( RegImm( dst, src) );
- ins_encode( OpcSErm( dst, src ), Con8or32( src ) );
- ins_pipe( ialu_reg );
-%}
-
-// Xor Register with Memory
-instruct xorI_eReg_mem(rRegI dst, memory src, eFlagsReg cr) %{
- match(Set dst (XorI dst (LoadI src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "XOR $dst,$src" %}
- opcode(0x33);
- ins_encode( SetInstMark, OpcP, RegMem(dst, src), ClearInstMark );
- ins_pipe( ialu_reg_mem );
-%}
-
-// Xor Memory with Register
-instruct xorI_mem_eReg(memory dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (XorI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(150);
- format %{ "XOR $dst,$src" %}
- opcode(0x31); /* Opcode 31 /r */
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_reg );
-%}
-
-// Xor Memory with Immediate
-instruct xorI_mem_imm(memory dst, immI src, eFlagsReg cr) %{
- match(Set dst (StoreI dst (XorI (LoadI dst) src)));
- effect(KILL cr);
-
- ins_cost(125);
- format %{ "XOR $dst,$src" %}
- opcode(0x81,0x6); /* Opcode 81 /6 id */
- ins_encode( SetInstMark, OpcSE( src ), RMopc_Mem(secondary, dst ), Con8or32(src), ClearInstMark );
- ins_pipe( ialu_mem_imm );
-%}
-
-//----------Convert Int to Boolean---------------------------------------------
-
-instruct movI_nocopy(rRegI dst, rRegI src) %{
- effect( DEF dst, USE src );
- format %{ "MOV $dst,$src" %}
- ins_encode( enc_Copy( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct ci2b( rRegI dst, rRegI src, eFlagsReg cr ) %{
- effect( USE_DEF dst, USE src, KILL cr );
-
- size(4);
- format %{ "NEG $dst\n\t"
- "ADC $dst,$src" %}
- ins_encode( neg_reg(dst),
- OpcRegReg(0x13,dst,src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-instruct convI2B( rRegI dst, rRegI src, eFlagsReg cr ) %{
- match(Set dst (Conv2B src));
-
- expand %{
- movI_nocopy(dst,src);
- ci2b(dst,src,cr);
- %}
-%}
-
-instruct movP_nocopy(rRegI dst, eRegP src) %{
- effect( DEF dst, USE src );
- format %{ "MOV $dst,$src" %}
- ins_encode( enc_Copy( dst, src) );
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct cp2b( rRegI dst, eRegP src, eFlagsReg cr ) %{
- effect( USE_DEF dst, USE src, KILL cr );
- format %{ "NEG $dst\n\t"
- "ADC $dst,$src" %}
- ins_encode( neg_reg(dst),
- OpcRegReg(0x13,dst,src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-instruct convP2B( rRegI dst, eRegP src, eFlagsReg cr ) %{
- match(Set dst (Conv2B src));
-
- expand %{
- movP_nocopy(dst,src);
- cp2b(dst,src,cr);
- %}
-%}
-
-instruct cmpLTMask(eCXRegI dst, ncxRegI p, ncxRegI q, eFlagsReg cr) %{
- match(Set dst (CmpLTMask p q));
- effect(KILL cr);
- ins_cost(400);
-
- // SETlt can only use low byte of EAX,EBX, ECX, or EDX as destination
- format %{ "XOR $dst,$dst\n\t"
- "CMP $p,$q\n\t"
- "SETlt $dst\n\t"
- "NEG $dst" %}
- ins_encode %{
- Register Rp = $p$$Register;
- Register Rq = $q$$Register;
- Register Rd = $dst$$Register;
- Label done;
- __ xorl(Rd, Rd);
- __ cmpl(Rp, Rq);
- __ setb(Assembler::less, Rd);
- __ negl(Rd);
- %}
-
- ins_pipe(pipe_slow);
-%}
-
-instruct cmpLTMask0(rRegI dst, immI_0 zero, eFlagsReg cr) %{
- match(Set dst (CmpLTMask dst zero));
- effect(DEF dst, KILL cr);
- ins_cost(100);
-
- format %{ "SAR $dst,31\t# cmpLTMask0" %}
- ins_encode %{
- __ sarl($dst$$Register, 31);
- %}
- ins_pipe(ialu_reg);
-%}
-
-/* better to save a register than avoid a branch */
-instruct cadd_cmpLTMask(rRegI p, rRegI q, rRegI y, eFlagsReg cr) %{
- match(Set p (AddI (AndI (CmpLTMask p q) y) (SubI p q)));
- effect(KILL cr);
- ins_cost(400);
- format %{ "SUB $p,$q\t# cadd_cmpLTMask\n\t"
- "JGE done\n\t"
- "ADD $p,$y\n"
- "done: " %}
- ins_encode %{
- Register Rp = $p$$Register;
- Register Rq = $q$$Register;
- Register Ry = $y$$Register;
- Label done;
- __ subl(Rp, Rq);
- __ jccb(Assembler::greaterEqual, done);
- __ addl(Rp, Ry);
- __ bind(done);
- %}
-
- ins_pipe(pipe_cmplt);
-%}
-
-/* better to save a register than avoid a branch */
-instruct and_cmpLTMask(rRegI p, rRegI q, rRegI y, eFlagsReg cr) %{
- match(Set y (AndI (CmpLTMask p q) y));
- effect(KILL cr);
-
- ins_cost(300);
-
- format %{ "CMPL $p, $q\t# and_cmpLTMask\n\t"
- "JLT done\n\t"
- "XORL $y, $y\n"
- "done: " %}
- ins_encode %{
- Register Rp = $p$$Register;
- Register Rq = $q$$Register;
- Register Ry = $y$$Register;
- Label done;
- __ cmpl(Rp, Rq);
- __ jccb(Assembler::less, done);
- __ xorl(Ry, Ry);
- __ bind(done);
- %}
-
- ins_pipe(pipe_cmplt);
-%}
-
-/* If I enable this, I encourage spilling in the inner loop of compress.
-instruct cadd_cmpLTMask_mem(ncxRegI p, ncxRegI q, memory y, eCXRegI tmp, eFlagsReg cr) %{
- match(Set p (AddI (AndI (CmpLTMask p q) (LoadI y)) (SubI p q)));
-*/
-//----------Overflow Math Instructions-----------------------------------------
-
-instruct overflowAddI_eReg(eFlagsReg cr, eAXRegI op1, rRegI op2)
-%{
- match(Set cr (OverflowAddI op1 op2));
- effect(DEF cr, USE_KILL op1, USE op2);
-
- format %{ "ADD $op1, $op2\t# overflow check int" %}
-
- ins_encode %{
- __ addl($op1$$Register, $op2$$Register);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-instruct overflowAddI_rReg_imm(eFlagsReg cr, eAXRegI op1, immI op2)
-%{
- match(Set cr (OverflowAddI op1 op2));
- effect(DEF cr, USE_KILL op1, USE op2);
-
- format %{ "ADD $op1, $op2\t# overflow check int" %}
-
- ins_encode %{
- __ addl($op1$$Register, $op2$$constant);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-instruct overflowSubI_rReg(eFlagsReg cr, rRegI op1, rRegI op2)
-%{
- match(Set cr (OverflowSubI op1 op2));
-
- format %{ "CMP $op1, $op2\t# overflow check int" %}
- ins_encode %{
- __ cmpl($op1$$Register, $op2$$Register);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-instruct overflowSubI_rReg_imm(eFlagsReg cr, rRegI op1, immI op2)
-%{
- match(Set cr (OverflowSubI op1 op2));
-
- format %{ "CMP $op1, $op2\t# overflow check int" %}
- ins_encode %{
- __ cmpl($op1$$Register, $op2$$constant);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-instruct overflowNegI_rReg(eFlagsReg cr, immI_0 zero, eAXRegI op2)
-%{
- match(Set cr (OverflowSubI zero op2));
- effect(DEF cr, USE_KILL op2);
-
- format %{ "NEG $op2\t# overflow check int" %}
- ins_encode %{
- __ negl($op2$$Register);
- %}
- ins_pipe(ialu_reg_reg);
-%}
-
-instruct overflowMulI_rReg(eFlagsReg cr, eAXRegI op1, rRegI op2)
-%{
- match(Set cr (OverflowMulI op1 op2));
- effect(DEF cr, USE_KILL op1, USE op2);
-
- format %{ "IMUL $op1, $op2\t# overflow check int" %}
- ins_encode %{
- __ imull($op1$$Register, $op2$$Register);
- %}
- ins_pipe(ialu_reg_reg_alu0);
-%}
-
-instruct overflowMulI_rReg_imm(eFlagsReg cr, rRegI op1, immI op2, rRegI tmp)
-%{
- match(Set cr (OverflowMulI op1 op2));
- effect(DEF cr, TEMP tmp, USE op1, USE op2);
-
- format %{ "IMUL $tmp, $op1, $op2\t# overflow check int" %}
- ins_encode %{
- __ imull($tmp$$Register, $op1$$Register, $op2$$constant);
- %}
- ins_pipe(ialu_reg_reg_alu0);
-%}
-
-// Integer Absolute Instructions
-instruct absI_rReg(rRegI dst, rRegI src, rRegI tmp, eFlagsReg cr)
-%{
- match(Set dst (AbsI src));
- effect(TEMP dst, TEMP tmp, KILL cr);
- format %{ "movl $tmp, $src\n\t"
- "sarl $tmp, 31\n\t"
- "movl $dst, $src\n\t"
- "xorl $dst, $tmp\n\t"
- "subl $dst, $tmp\n"
- %}
- ins_encode %{
- __ movl($tmp$$Register, $src$$Register);
- __ sarl($tmp$$Register, 31);
- __ movl($dst$$Register, $src$$Register);
- __ xorl($dst$$Register, $tmp$$Register);
- __ subl($dst$$Register, $tmp$$Register);
- %}
-
- ins_pipe(ialu_reg_reg);
-%}
-
-//----------Long Instructions------------------------------------------------
-// Add Long Register with Register
-instruct addL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
- match(Set dst (AddL dst src));
- effect(KILL cr);
- ins_cost(200);
- format %{ "ADD $dst.lo,$src.lo\n\t"
- "ADC $dst.hi,$src.hi" %}
- opcode(0x03, 0x13);
- ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Add Long Register with Immediate
-instruct addL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
- match(Set dst (AddL dst src));
- effect(KILL cr);
- format %{ "ADD $dst.lo,$src.lo\n\t"
- "ADC $dst.hi,$src.hi" %}
- opcode(0x81,0x00,0x02); /* Opcode 81 /0, 81 /2 */
- ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Add Long Register with Memory
-instruct addL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
- match(Set dst (AddL dst (LoadL mem)));
- effect(KILL cr);
- ins_cost(125);
- format %{ "ADD $dst.lo,$mem\n\t"
- "ADC $dst.hi,$mem+4" %}
- opcode(0x03, 0x13);
- ins_encode( SetInstMark, OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem), ClearInstMark );
- ins_pipe( ialu_reg_long_mem );
-%}
-
-// Subtract Long Register with Register.
-instruct subL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
- match(Set dst (SubL dst src));
- effect(KILL cr);
- ins_cost(200);
- format %{ "SUB $dst.lo,$src.lo\n\t"
- "SBB $dst.hi,$src.hi" %}
- opcode(0x2B, 0x1B);
- ins_encode( RegReg_Lo(dst, src), RegReg_Hi(dst,src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Subtract Long Register with Immediate
-instruct subL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
- match(Set dst (SubL dst src));
- effect(KILL cr);
- format %{ "SUB $dst.lo,$src.lo\n\t"
- "SBB $dst.hi,$src.hi" %}
- opcode(0x81,0x05,0x03); /* Opcode 81 /5, 81 /3 */
- ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Subtract Long Register with Memory
-instruct subL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
- match(Set dst (SubL dst (LoadL mem)));
- effect(KILL cr);
- ins_cost(125);
- format %{ "SUB $dst.lo,$mem\n\t"
- "SBB $dst.hi,$mem+4" %}
- opcode(0x2B, 0x1B);
- ins_encode( SetInstMark, OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem), ClearInstMark );
- ins_pipe( ialu_reg_long_mem );
-%}
-
-instruct negL_eReg(eRegL dst, immL0 zero, eFlagsReg cr) %{
- match(Set dst (SubL zero dst));
- effect(KILL cr);
- ins_cost(300);
- format %{ "NEG $dst.hi\n\tNEG $dst.lo\n\tSBB $dst.hi,0" %}
- ins_encode( neg_long(dst) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// And Long Register with Register
-instruct andL_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
- match(Set dst (AndL dst src));
- effect(KILL cr);
- format %{ "AND $dst.lo,$src.lo\n\t"
- "AND $dst.hi,$src.hi" %}
- opcode(0x23,0x23);
- ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// And Long Register with Immediate
-instruct andL_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
- match(Set dst (AndL dst src));
- effect(KILL cr);
- format %{ "AND $dst.lo,$src.lo\n\t"
- "AND $dst.hi,$src.hi" %}
- opcode(0x81,0x04,0x04); /* Opcode 81 /4, 81 /4 */
- ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
- ins_pipe( ialu_reg_long );
-%}
-
-// And Long Register with Memory
-instruct andL_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
- match(Set dst (AndL dst (LoadL mem)));
- effect(KILL cr);
- ins_cost(125);
- format %{ "AND $dst.lo,$mem\n\t"
- "AND $dst.hi,$mem+4" %}
- opcode(0x23, 0x23);
- ins_encode( SetInstMark, OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem), ClearInstMark );
- ins_pipe( ialu_reg_long_mem );
-%}
-
-// BMI1 instructions
-instruct andnL_eReg_eReg_eReg(eRegL dst, eRegL src1, eRegL src2, immL_M1 minus_1, eFlagsReg cr) %{
- match(Set dst (AndL (XorL src1 minus_1) src2));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- format %{ "ANDNL $dst.lo, $src1.lo, $src2.lo\n\t"
- "ANDNL $dst.hi, $src1.hi, $src2.hi"
- %}
-
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc1 = $src1$$Register;
- Register Rsrc2 = $src2$$Register;
- __ andnl(Rdst, Rsrc1, Rsrc2);
- __ andnl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc1), HIGH_FROM_LOW(Rsrc2));
- %}
- ins_pipe(ialu_reg_reg_long);
-%}
-
-instruct andnL_eReg_eReg_mem(eRegL dst, eRegL src1, memory src2, immL_M1 minus_1, eFlagsReg cr) %{
- match(Set dst (AndL (XorL src1 minus_1) (LoadL src2) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- ins_cost(125);
- format %{ "ANDNL $dst.lo, $src1.lo, $src2\n\t"
- "ANDNL $dst.hi, $src1.hi, $src2+4"
- %}
-
- ins_encode %{
- Register Rdst = $dst$$Register;
- Register Rsrc1 = $src1$$Register;
- Address src2_hi = Address::make_raw($src2$$base, $src2$$index, $src2$$scale, $src2$$disp + 4, relocInfo::none);
-
- __ andnl(Rdst, Rsrc1, $src2$$Address);
- __ andnl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc1), src2_hi);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct blsiL_eReg_eReg(eRegL dst, eRegL src, immL0 imm_zero, eFlagsReg cr) %{
- match(Set dst (AndL (SubL imm_zero src) src));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- format %{ "MOVL $dst.hi, 0\n\t"
- "BLSIL $dst.lo, $src.lo\n\t"
- "JNZ done\n\t"
- "BLSIL $dst.hi, $src.hi\n"
- "done:"
- %}
-
- ins_encode %{
- Label done;
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- __ movl(HIGH_FROM_LOW(Rdst), 0);
- __ blsil(Rdst, Rsrc);
- __ jccb(Assembler::notZero, done);
- __ blsil(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
- __ bind(done);
- %}
- ins_pipe(ialu_reg);
-%}
-
-instruct blsiL_eReg_mem(eRegL dst, memory src, immL0 imm_zero, eFlagsReg cr) %{
- match(Set dst (AndL (SubL imm_zero (LoadL src) ) (LoadL src) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- ins_cost(125);
- format %{ "MOVL $dst.hi, 0\n\t"
- "BLSIL $dst.lo, $src\n\t"
- "JNZ done\n\t"
- "BLSIL $dst.hi, $src+4\n"
- "done:"
- %}
-
- ins_encode %{
- Label done;
- Register Rdst = $dst$$Register;
- Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none);
-
- __ movl(HIGH_FROM_LOW(Rdst), 0);
- __ blsil(Rdst, $src$$Address);
- __ jccb(Assembler::notZero, done);
- __ blsil(HIGH_FROM_LOW(Rdst), src_hi);
- __ bind(done);
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct blsmskL_eReg_eReg(eRegL dst, eRegL src, immL_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (XorL (AddL src minus_1) src));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- format %{ "MOVL $dst.hi, 0\n\t"
- "BLSMSKL $dst.lo, $src.lo\n\t"
- "JNC done\n\t"
- "BLSMSKL $dst.hi, $src.hi\n"
- "done:"
- %}
-
- ins_encode %{
- Label done;
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- __ movl(HIGH_FROM_LOW(Rdst), 0);
- __ blsmskl(Rdst, Rsrc);
- __ jccb(Assembler::carryClear, done);
- __ blsmskl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
- __ bind(done);
- %}
-
- ins_pipe(ialu_reg);
-%}
-
-instruct blsmskL_eReg_mem(eRegL dst, memory src, immL_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (XorL (AddL (LoadL src) minus_1) (LoadL src) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- ins_cost(125);
- format %{ "MOVL $dst.hi, 0\n\t"
- "BLSMSKL $dst.lo, $src\n\t"
- "JNC done\n\t"
- "BLSMSKL $dst.hi, $src+4\n"
- "done:"
- %}
-
- ins_encode %{
- Label done;
- Register Rdst = $dst$$Register;
- Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none);
-
- __ movl(HIGH_FROM_LOW(Rdst), 0);
- __ blsmskl(Rdst, $src$$Address);
- __ jccb(Assembler::carryClear, done);
- __ blsmskl(HIGH_FROM_LOW(Rdst), src_hi);
- __ bind(done);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-instruct blsrL_eReg_eReg(eRegL dst, eRegL src, immL_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (AndL (AddL src minus_1) src) );
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- format %{ "MOVL $dst.hi, $src.hi\n\t"
- "BLSRL $dst.lo, $src.lo\n\t"
- "JNC done\n\t"
- "BLSRL $dst.hi, $src.hi\n"
- "done:"
- %}
-
- ins_encode %{
- Label done;
- Register Rdst = $dst$$Register;
- Register Rsrc = $src$$Register;
- __ movl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
- __ blsrl(Rdst, Rsrc);
- __ jccb(Assembler::carryClear, done);
- __ blsrl(HIGH_FROM_LOW(Rdst), HIGH_FROM_LOW(Rsrc));
- __ bind(done);
- %}
-
- ins_pipe(ialu_reg);
-%}
-
-instruct blsrL_eReg_mem(eRegL dst, memory src, immL_M1 minus_1, eFlagsReg cr)
-%{
- match(Set dst (AndL (AddL (LoadL src) minus_1) (LoadL src) ));
- predicate(UseBMI1Instructions);
- effect(KILL cr, TEMP dst);
-
- ins_cost(125);
- format %{ "MOVL $dst.hi, $src+4\n\t"
- "BLSRL $dst.lo, $src\n\t"
- "JNC done\n\t"
- "BLSRL $dst.hi, $src+4\n"
- "done:"
- %}
-
- ins_encode %{
- Label done;
- Register Rdst = $dst$$Register;
- Address src_hi = Address::make_raw($src$$base, $src$$index, $src$$scale, $src$$disp + 4, relocInfo::none);
- __ movl(HIGH_FROM_LOW(Rdst), src_hi);
- __ blsrl(Rdst, $src$$Address);
- __ jccb(Assembler::carryClear, done);
- __ blsrl(HIGH_FROM_LOW(Rdst), src_hi);
- __ bind(done);
- %}
-
- ins_pipe(ialu_reg_mem);
-%}
-
-// Or Long Register with Register
-instruct orl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
- match(Set dst (OrL dst src));
- effect(KILL cr);
- format %{ "OR $dst.lo,$src.lo\n\t"
- "OR $dst.hi,$src.hi" %}
- opcode(0x0B,0x0B);
- ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Or Long Register with Immediate
-instruct orl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
- match(Set dst (OrL dst src));
- effect(KILL cr);
- format %{ "OR $dst.lo,$src.lo\n\t"
- "OR $dst.hi,$src.hi" %}
- opcode(0x81,0x01,0x01); /* Opcode 81 /1, 81 /1 */
- ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Or Long Register with Memory
-instruct orl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
- match(Set dst (OrL dst (LoadL mem)));
- effect(KILL cr);
- ins_cost(125);
- format %{ "OR $dst.lo,$mem\n\t"
- "OR $dst.hi,$mem+4" %}
- opcode(0x0B,0x0B);
- ins_encode( SetInstMark, OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem), ClearInstMark );
- ins_pipe( ialu_reg_long_mem );
-%}
-
-// Xor Long Register with Register
-instruct xorl_eReg(eRegL dst, eRegL src, eFlagsReg cr) %{
- match(Set dst (XorL dst src));
- effect(KILL cr);
- format %{ "XOR $dst.lo,$src.lo\n\t"
- "XOR $dst.hi,$src.hi" %}
- opcode(0x33,0x33);
- ins_encode( RegReg_Lo( dst, src), RegReg_Hi( dst, src) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Xor Long Register with Immediate -1
-instruct xorl_eReg_im1(eRegL dst, immL_M1 imm) %{
- match(Set dst (XorL dst imm));
- format %{ "NOT $dst.lo\n\t"
- "NOT $dst.hi" %}
- ins_encode %{
- __ notl($dst$$Register);
- __ notl(HIGH_FROM_LOW($dst$$Register));
- %}
- ins_pipe( ialu_reg_long );
-%}
-
-// Xor Long Register with Immediate
-instruct xorl_eReg_imm(eRegL dst, immL src, eFlagsReg cr) %{
- match(Set dst (XorL dst src));
- effect(KILL cr);
- format %{ "XOR $dst.lo,$src.lo\n\t"
- "XOR $dst.hi,$src.hi" %}
- opcode(0x81,0x06,0x06); /* Opcode 81 /6, 81 /6 */
- ins_encode( Long_OpcSErm_Lo( dst, src ), Long_OpcSErm_Hi( dst, src ) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Xor Long Register with Memory
-instruct xorl_eReg_mem(eRegL dst, load_long_memory mem, eFlagsReg cr) %{
- match(Set dst (XorL dst (LoadL mem)));
- effect(KILL cr);
- ins_cost(125);
- format %{ "XOR $dst.lo,$mem\n\t"
- "XOR $dst.hi,$mem+4" %}
- opcode(0x33,0x33);
- ins_encode( SetInstMark, OpcP, RegMem( dst, mem), OpcS, RegMem_Hi(dst,mem), ClearInstMark );
- ins_pipe( ialu_reg_long_mem );
-%}
-
-// Shift Left Long by 1
-instruct shlL_eReg_1(eRegL dst, immI_1 cnt, eFlagsReg cr) %{
- predicate(UseNewLongLShift);
- match(Set dst (LShiftL dst cnt));
- effect(KILL cr);
- ins_cost(100);
- format %{ "ADD $dst.lo,$dst.lo\n\t"
- "ADC $dst.hi,$dst.hi" %}
- ins_encode %{
- __ addl($dst$$Register,$dst$$Register);
- __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
- %}
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Left Long by 2
-instruct shlL_eReg_2(eRegL dst, immI_2 cnt, eFlagsReg cr) %{
- predicate(UseNewLongLShift);
- match(Set dst (LShiftL dst cnt));
- effect(KILL cr);
- ins_cost(100);
- format %{ "ADD $dst.lo,$dst.lo\n\t"
- "ADC $dst.hi,$dst.hi\n\t"
- "ADD $dst.lo,$dst.lo\n\t"
- "ADC $dst.hi,$dst.hi" %}
- ins_encode %{
- __ addl($dst$$Register,$dst$$Register);
- __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
- __ addl($dst$$Register,$dst$$Register);
- __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
- %}
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Left Long by 3
-instruct shlL_eReg_3(eRegL dst, immI_3 cnt, eFlagsReg cr) %{
- predicate(UseNewLongLShift);
- match(Set dst (LShiftL dst cnt));
- effect(KILL cr);
- ins_cost(100);
- format %{ "ADD $dst.lo,$dst.lo\n\t"
- "ADC $dst.hi,$dst.hi\n\t"
- "ADD $dst.lo,$dst.lo\n\t"
- "ADC $dst.hi,$dst.hi\n\t"
- "ADD $dst.lo,$dst.lo\n\t"
- "ADC $dst.hi,$dst.hi" %}
- ins_encode %{
- __ addl($dst$$Register,$dst$$Register);
- __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
- __ addl($dst$$Register,$dst$$Register);
- __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
- __ addl($dst$$Register,$dst$$Register);
- __ adcl(HIGH_FROM_LOW($dst$$Register),HIGH_FROM_LOW($dst$$Register));
- %}
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Left Long by 1-31
-instruct shlL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
- match(Set dst (LShiftL dst cnt));
- effect(KILL cr);
- ins_cost(200);
- format %{ "SHLD $dst.hi,$dst.lo,$cnt\n\t"
- "SHL $dst.lo,$cnt" %}
- opcode(0xC1, 0x4, 0xA4); /* 0F/A4, then C1 /4 ib */
- ins_encode( move_long_small_shift(dst,cnt) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Left Long by 32-63
-instruct shlL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
- match(Set dst (LShiftL dst cnt));
- effect(KILL cr);
- ins_cost(300);
- format %{ "MOV $dst.hi,$dst.lo\n"
- "\tSHL $dst.hi,$cnt-32\n"
- "\tXOR $dst.lo,$dst.lo" %}
- opcode(0xC1, 0x4); /* C1 /4 ib */
- ins_encode( move_long_big_shift_clr(dst,cnt) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Left Long by variable
-instruct salL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
- match(Set dst (LShiftL dst shift));
- effect(KILL cr);
- ins_cost(500+200);
- size(17);
- format %{ "TEST $shift,32\n\t"
- "JEQ,s small\n\t"
- "MOV $dst.hi,$dst.lo\n\t"
- "XOR $dst.lo,$dst.lo\n"
- "small:\tSHLD $dst.hi,$dst.lo,$shift\n\t"
- "SHL $dst.lo,$shift" %}
- ins_encode( shift_left_long( dst, shift ) );
- ins_pipe( pipe_slow );
-%}
-
-// Shift Right Long by 1-31
-instruct shrL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
- match(Set dst (URShiftL dst cnt));
- effect(KILL cr);
- ins_cost(200);
- format %{ "SHRD $dst.lo,$dst.hi,$cnt\n\t"
- "SHR $dst.hi,$cnt" %}
- opcode(0xC1, 0x5, 0xAC); /* 0F/AC, then C1 /5 ib */
- ins_encode( move_long_small_shift(dst,cnt) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Right Long by 32-63
-instruct shrL_eReg_32_63(eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
- match(Set dst (URShiftL dst cnt));
- effect(KILL cr);
- ins_cost(300);
- format %{ "MOV $dst.lo,$dst.hi\n"
- "\tSHR $dst.lo,$cnt-32\n"
- "\tXOR $dst.hi,$dst.hi" %}
- opcode(0xC1, 0x5); /* C1 /5 ib */
- ins_encode( move_long_big_shift_clr(dst,cnt) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Right Long by variable
-instruct shrL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
- match(Set dst (URShiftL dst shift));
- effect(KILL cr);
- ins_cost(600);
- size(17);
- format %{ "TEST $shift,32\n\t"
- "JEQ,s small\n\t"
- "MOV $dst.lo,$dst.hi\n\t"
- "XOR $dst.hi,$dst.hi\n"
- "small:\tSHRD $dst.lo,$dst.hi,$shift\n\t"
- "SHR $dst.hi,$shift" %}
- ins_encode( shift_right_long( dst, shift ) );
- ins_pipe( pipe_slow );
-%}
-
-// Shift Right Long by 1-31
-instruct sarL_eReg_1_31(eRegL dst, immI_1_31 cnt, eFlagsReg cr) %{
- match(Set dst (RShiftL dst cnt));
- effect(KILL cr);
- ins_cost(200);
- format %{ "SHRD $dst.lo,$dst.hi,$cnt\n\t"
- "SAR $dst.hi,$cnt" %}
- opcode(0xC1, 0x7, 0xAC); /* 0F/AC, then C1 /7 ib */
- ins_encode( move_long_small_shift(dst,cnt) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Right Long by 32-63
-instruct sarL_eReg_32_63( eRegL dst, immI_32_63 cnt, eFlagsReg cr) %{
- match(Set dst (RShiftL dst cnt));
- effect(KILL cr);
- ins_cost(300);
- format %{ "MOV $dst.lo,$dst.hi\n"
- "\tSAR $dst.lo,$cnt-32\n"
- "\tSAR $dst.hi,31" %}
- opcode(0xC1, 0x7); /* C1 /7 ib */
- ins_encode( move_long_big_shift_sign(dst,cnt) );
- ins_pipe( ialu_reg_long );
-%}
-
-// Shift Right arithmetic Long by variable
-instruct sarL_eReg_CL(eRegL dst, eCXRegI shift, eFlagsReg cr) %{
- match(Set dst (RShiftL dst shift));
- effect(KILL cr);
- ins_cost(600);
- size(18);
- format %{ "TEST $shift,32\n\t"
- "JEQ,s small\n\t"
- "MOV $dst.lo,$dst.hi\n\t"
- "SAR $dst.hi,31\n"
- "small:\tSHRD $dst.lo,$dst.hi,$shift\n\t"
- "SAR $dst.hi,$shift" %}
- ins_encode( shift_right_arith_long( dst, shift ) );
- ins_pipe( pipe_slow );
-%}
-
-
-//----------Double Instructions------------------------------------------------
-// Double Math
-
-// Compare & branch
-
-// P6 version of float compare, sets condition codes in EFLAGS
-instruct cmpDPR_cc_P6(eFlagsRegU cr, regDPR src1, regDPR src2, eAXRegI rax) %{
- predicate(VM_Version::supports_cmov() && UseSSE <=1);
- match(Set cr (CmpD src1 src2));
- effect(KILL rax);
- ins_cost(150);
- format %{ "FLD $src1\n\t"
- "FUCOMIP ST,$src2 // P6 instruction\n\t"
- "JNP exit\n\t"
- "MOV ah,1 // saw a NaN, set CF\n\t"
- "SAHF\n"
- "exit:\tNOP // avoid branch to branch" %}
- opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2),
- cmpF_P6_fixup );
- ins_pipe( pipe_slow );
-%}
-
-instruct cmpDPR_cc_P6CF(eFlagsRegUCF cr, regDPR src1, regDPR src2) %{
- predicate(VM_Version::supports_cmov() && UseSSE <=1);
- match(Set cr (CmpD src1 src2));
- ins_cost(150);
- format %{ "FLD $src1\n\t"
- "FUCOMIP ST,$src2 // P6 instruction" %}
- opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2));
- ins_pipe( pipe_slow );
-%}
-
-// Compare & branch
-instruct cmpDPR_cc(eFlagsRegU cr, regDPR src1, regDPR src2, eAXRegI rax) %{
- predicate(UseSSE<=1);
- match(Set cr (CmpD src1 src2));
- effect(KILL rax);
- ins_cost(200);
- format %{ "FLD $src1\n\t"
- "FCOMp $src2\n\t"
- "FNSTSW AX\n\t"
- "TEST AX,0x400\n\t"
- "JZ,s flags\n\t"
- "MOV AH,1\t# unordered treat as LT\n"
- "flags:\tSAHF" %}
- opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2),
- fpu_flags);
- ins_pipe( pipe_slow );
-%}
-
-// Compare vs zero into -1,0,1
-instruct cmpDPR_0(rRegI dst, regDPR src1, immDPR0 zero, eAXRegI rax, eFlagsReg cr) %{
- predicate(UseSSE<=1);
- match(Set dst (CmpD3 src1 zero));
- effect(KILL cr, KILL rax);
- ins_cost(280);
- format %{ "FTSTD $dst,$src1" %}
- opcode(0xE4, 0xD9);
- ins_encode( Push_Reg_DPR(src1),
- OpcS, OpcP, PopFPU,
- CmpF_Result(dst));
- ins_pipe( pipe_slow );
-%}
-
-// Compare into -1,0,1
-instruct cmpDPR_reg(rRegI dst, regDPR src1, regDPR src2, eAXRegI rax, eFlagsReg cr) %{
- predicate(UseSSE<=1);
- match(Set dst (CmpD3 src1 src2));
- effect(KILL cr, KILL rax);
- ins_cost(300);
- format %{ "FCMPD $dst,$src1,$src2" %}
- opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2),
- CmpF_Result(dst));
- ins_pipe( pipe_slow );
-%}
-
-// float compare and set condition codes in EFLAGS by XMM regs
-instruct cmpD_cc(eFlagsRegU cr, regD src1, regD src2) %{
- predicate(UseSSE>=2);
- match(Set cr (CmpD src1 src2));
- ins_cost(145);
- format %{ "UCOMISD $src1,$src2\n\t"
- "JNP,s exit\n\t"
- "PUSHF\t# saw NaN, set CF\n\t"
- "AND [rsp], #0xffffff2b\n\t"
- "POPF\n"
- "exit:" %}
- ins_encode %{
- __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
- emit_cmpfp_fixup(masm);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct cmpD_ccCF(eFlagsRegUCF cr, regD src1, regD src2) %{
- predicate(UseSSE>=2);
- match(Set cr (CmpD src1 src2));
- ins_cost(100);
- format %{ "UCOMISD $src1,$src2" %}
- ins_encode %{
- __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// float compare and set condition codes in EFLAGS by XMM regs
-instruct cmpD_ccmem(eFlagsRegU cr, regD src1, memory src2) %{
- predicate(UseSSE>=2);
- match(Set cr (CmpD src1 (LoadD src2)));
- ins_cost(145);
- format %{ "UCOMISD $src1,$src2\n\t"
- "JNP,s exit\n\t"
- "PUSHF\t# saw NaN, set CF\n\t"
- "AND [rsp], #0xffffff2b\n\t"
- "POPF\n"
- "exit:" %}
- ins_encode %{
- __ ucomisd($src1$$XMMRegister, $src2$$Address);
- emit_cmpfp_fixup(masm);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct cmpD_ccmemCF(eFlagsRegUCF cr, regD src1, memory src2) %{
- predicate(UseSSE>=2);
- match(Set cr (CmpD src1 (LoadD src2)));
- ins_cost(100);
- format %{ "UCOMISD $src1,$src2" %}
- ins_encode %{
- __ ucomisd($src1$$XMMRegister, $src2$$Address);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Compare into -1,0,1 in XMM
-instruct cmpD_reg(xRegI dst, regD src1, regD src2, eFlagsReg cr) %{
- predicate(UseSSE>=2);
- match(Set dst (CmpD3 src1 src2));
- effect(KILL cr);
- ins_cost(255);
- format %{ "UCOMISD $src1, $src2\n\t"
- "MOV $dst, #-1\n\t"
- "JP,s done\n\t"
- "JB,s done\n\t"
- "SETNE $dst\n\t"
- "MOVZB $dst, $dst\n"
- "done:" %}
- ins_encode %{
- __ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
- emit_cmpfp3(masm, $dst$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Compare into -1,0,1 in XMM and memory
-instruct cmpD_regmem(xRegI dst, regD src1, memory src2, eFlagsReg cr) %{
- predicate(UseSSE>=2);
- match(Set dst (CmpD3 src1 (LoadD src2)));
- effect(KILL cr);
- ins_cost(275);
- format %{ "UCOMISD $src1, $src2\n\t"
- "MOV $dst, #-1\n\t"
- "JP,s done\n\t"
- "JB,s done\n\t"
- "SETNE $dst\n\t"
- "MOVZB $dst, $dst\n"
- "done:" %}
- ins_encode %{
- __ ucomisd($src1$$XMMRegister, $src2$$Address);
- emit_cmpfp3(masm, $dst$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-
-instruct subDPR_reg(regDPR dst, regDPR src) %{
- predicate (UseSSE <=1);
- match(Set dst (SubD dst src));
-
- format %{ "FLD $src\n\t"
- "DSUBp $dst,ST" %}
- opcode(0xDE, 0x5); /* DE E8+i or DE /5 */
- ins_cost(150);
- ins_encode( Push_Reg_DPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct subDPR_reg_round(stackSlotD dst, regDPR src1, regDPR src2) %{
- predicate (UseSSE <=1);
- match(Set dst (RoundDouble (SubD src1 src2)));
- ins_cost(250);
-
- format %{ "FLD $src2\n\t"
- "DSUB ST,$src1\n\t"
- "FSTP_D $dst\t# D-round" %}
- opcode(0xD8, 0x5);
- ins_encode( Push_Reg_DPR(src2),
- OpcP, RegOpc(src1), Pop_Mem_DPR(dst) );
- ins_pipe( fpu_mem_reg_reg );
-%}
-
-
-instruct subDPR_reg_mem(regDPR dst, memory src) %{
- predicate (UseSSE <=1);
- match(Set dst (SubD dst (LoadD src)));
- ins_cost(150);
-
- format %{ "FLD $src\n\t"
- "DSUBp $dst,ST" %}
- opcode(0xDE, 0x5, 0xDD); /* DE C0+i */ /* LoadD DD /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src),
- OpcP, RegOpc(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-instruct absDPR_reg(regDPR1 dst, regDPR1 src) %{
- predicate (UseSSE<=1);
- match(Set dst (AbsD src));
- ins_cost(100);
- format %{ "FABS" %}
- opcode(0xE1, 0xD9);
- ins_encode( OpcS, OpcP );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct negDPR_reg(regDPR1 dst, regDPR1 src) %{
- predicate(UseSSE<=1);
- match(Set dst (NegD src));
- ins_cost(100);
- format %{ "FCHS" %}
- opcode(0xE0, 0xD9);
- ins_encode( OpcS, OpcP );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct addDPR_reg(regDPR dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (AddD dst src));
- format %{ "FLD $src\n\t"
- "DADD $dst,ST" %}
- size(4);
- ins_cost(150);
- opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
- ins_encode( Push_Reg_DPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-
-instruct addDPR_reg_round(stackSlotD dst, regDPR src1, regDPR src2) %{
- predicate(UseSSE<=1);
- match(Set dst (RoundDouble (AddD src1 src2)));
- ins_cost(250);
-
- format %{ "FLD $src2\n\t"
- "DADD ST,$src1\n\t"
- "FSTP_D $dst\t# D-round" %}
- opcode(0xD8, 0x0); /* D8 C0+i or D8 /0*/
- ins_encode( Push_Reg_DPR(src2),
- OpcP, RegOpc(src1), Pop_Mem_DPR(dst) );
- ins_pipe( fpu_mem_reg_reg );
-%}
-
-
-instruct addDPR_reg_mem(regDPR dst, memory src) %{
- predicate(UseSSE<=1);
- match(Set dst (AddD dst (LoadD src)));
- ins_cost(150);
-
- format %{ "FLD $src\n\t"
- "DADDp $dst,ST" %}
- opcode(0xDE, 0x0, 0xDD); /* DE C0+i */ /* LoadD DD /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src),
- OpcP, RegOpc(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-// add-to-memory
-instruct addDPR_mem_reg(memory dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (StoreD dst (RoundDouble (AddD (LoadD dst) src))));
- ins_cost(150);
-
- format %{ "FLD_D $dst\n\t"
- "DADD ST,$src\n\t"
- "FST_D $dst" %}
- opcode(0xDD, 0x0);
- ins_encode( SetInstMark, Opcode(0xDD), RMopc_Mem(0x00,dst),
- Opcode(0xD8), RegOpc(src), ClearInstMark,
- SetInstMark,
- Opcode(0xDD), RMopc_Mem(0x03,dst),
- ClearInstMark);
- ins_pipe( fpu_reg_mem );
-%}
-
-instruct addDPR_reg_imm1(regDPR dst, immDPR1 con) %{
- predicate(UseSSE<=1);
- match(Set dst (AddD dst con));
- ins_cost(125);
- format %{ "FLD1\n\t"
- "DADDp $dst,ST" %}
- ins_encode %{
- __ fld1();
- __ faddp($dst$$reg);
- %}
- ins_pipe(fpu_reg);
-%}
-
-instruct addDPR_reg_imm(regDPR dst, immDPR con) %{
- predicate(UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
- match(Set dst (AddD dst con));
- ins_cost(200);
- format %{ "FLD_D [$constantaddress]\t# load from constant table: double=$con\n\t"
- "DADDp $dst,ST" %}
- ins_encode %{
- __ fld_d($constantaddress($con));
- __ faddp($dst$$reg);
- %}
- ins_pipe(fpu_reg_mem);
-%}
-
-instruct addDPR_reg_imm_round(stackSlotD dst, regDPR src, immDPR con) %{
- predicate(UseSSE<=1 && _kids[0]->_kids[1]->_leaf->getd() != 0.0 && _kids[0]->_kids[1]->_leaf->getd() != 1.0 );
- match(Set dst (RoundDouble (AddD src con)));
- ins_cost(200);
- format %{ "FLD_D [$constantaddress]\t# load from constant table: double=$con\n\t"
- "DADD ST,$src\n\t"
- "FSTP_D $dst\t# D-round" %}
- ins_encode %{
- __ fld_d($constantaddress($con));
- __ fadd($src$$reg);
- __ fstp_d(Address(rsp, $dst$$disp));
- %}
- ins_pipe(fpu_mem_reg_con);
-%}
-
-instruct mulDPR_reg(regDPR dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (MulD dst src));
- format %{ "FLD $src\n\t"
- "DMULp $dst,ST" %}
- opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
- ins_cost(150);
- ins_encode( Push_Reg_DPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-// Strict FP instruction biases argument before multiply then
-// biases result to avoid double rounding of subnormals.
-//
-// scale arg1 by multiplying arg1 by 2^(-15360)
-// load arg2
-// multiply scaled arg1 by arg2
-// rescale product by 2^(15360)
-//
-instruct strictfp_mulDPR_reg(regDPR1 dst, regnotDPR1 src) %{
- predicate( UseSSE<=1 && Compile::current()->has_method() );
- match(Set dst (MulD dst src));
- ins_cost(1); // Select this instruction for all FP double multiplies
-
- format %{ "FLD StubRoutines::x86::_fpu_subnormal_bias1\n\t"
- "DMULp $dst,ST\n\t"
- "FLD $src\n\t"
- "DMULp $dst,ST\n\t"
- "FLD StubRoutines::x86::_fpu_subnormal_bias2\n\t"
- "DMULp $dst,ST\n\t" %}
- opcode(0xDE, 0x1); /* DE C8+i or DE /1*/
- ins_encode( strictfp_bias1(dst),
- Push_Reg_DPR(src),
- OpcP, RegOpc(dst),
- strictfp_bias2(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct mulDPR_reg_imm(regDPR dst, immDPR con) %{
- predicate( UseSSE<=1 && _kids[1]->_leaf->getd() != 0.0 && _kids[1]->_leaf->getd() != 1.0 );
- match(Set dst (MulD dst con));
- ins_cost(200);
- format %{ "FLD_D [$constantaddress]\t# load from constant table: double=$con\n\t"
- "DMULp $dst,ST" %}
- ins_encode %{
- __ fld_d($constantaddress($con));
- __ fmulp($dst$$reg);
- %}
- ins_pipe(fpu_reg_mem);
-%}
-
-
-instruct mulDPR_reg_mem(regDPR dst, memory src) %{
- predicate( UseSSE<=1 );
- match(Set dst (MulD dst (LoadD src)));
- ins_cost(200);
- format %{ "FLD_D $src\n\t"
- "DMULp $dst,ST" %}
- opcode(0xDE, 0x1, 0xDD); /* DE C8+i or DE /1*/ /* LoadD DD /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src),
- OpcP, RegOpc(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-//
-// Cisc-alternate to reg-reg multiply
-instruct mulDPR_reg_mem_cisc(regDPR dst, regDPR src, memory mem) %{
- predicate( UseSSE<=1 );
- match(Set dst (MulD src (LoadD mem)));
- ins_cost(250);
- format %{ "FLD_D $mem\n\t"
- "DMUL ST,$src\n\t"
- "FSTP_D $dst" %}
- opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */ /* LoadD D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,mem),
- OpcReg_FPR(src),
- Pop_Reg_DPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_reg_mem );
-%}
-
-
-// MACRO3 -- addDPR a mulDPR
-// This instruction is a '2-address' instruction in that the result goes
-// back to src2. This eliminates a move from the macro; possibly the
-// register allocator will have to add it back (and maybe not).
-instruct addDPR_mulDPR_reg(regDPR src2, regDPR src1, regDPR src0) %{
- predicate( UseSSE<=1 );
- match(Set src2 (AddD (MulD src0 src1) src2));
- format %{ "FLD $src0\t# ===MACRO3d===\n\t"
- "DMUL ST,$src1\n\t"
- "DADDp $src2,ST" %}
- ins_cost(250);
- opcode(0xDD); /* LoadD DD /0 */
- ins_encode( Push_Reg_FPR(src0),
- FMul_ST_reg(src1),
- FAddP_reg_ST(src2) );
- ins_pipe( fpu_reg_reg_reg );
-%}
-
-
-// MACRO3 -- subDPR a mulDPR
-instruct subDPR_mulDPR_reg(regDPR src2, regDPR src1, regDPR src0) %{
- predicate( UseSSE<=1 );
- match(Set src2 (SubD (MulD src0 src1) src2));
- format %{ "FLD $src0\t# ===MACRO3d===\n\t"
- "DMUL ST,$src1\n\t"
- "DSUBRp $src2,ST" %}
- ins_cost(250);
- ins_encode( Push_Reg_FPR(src0),
- FMul_ST_reg(src1),
- Opcode(0xDE), Opc_plus(0xE0,src2));
- ins_pipe( fpu_reg_reg_reg );
-%}
-
-
-instruct divDPR_reg(regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 );
- match(Set dst (DivD dst src));
-
- format %{ "FLD $src\n\t"
- "FDIVp $dst,ST" %}
- opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
- ins_cost(150);
- ins_encode( Push_Reg_DPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-// Strict FP instruction biases argument before division then
-// biases result, to avoid double rounding of subnormals.
-//
-// scale dividend by multiplying dividend by 2^(-15360)
-// load divisor
-// divide scaled dividend by divisor
-// rescale quotient by 2^(15360)
-//
-instruct strictfp_divDPR_reg(regDPR1 dst, regnotDPR1 src) %{
- predicate (UseSSE<=1);
- match(Set dst (DivD dst src));
- predicate( UseSSE<=1 && Compile::current()->has_method() );
- ins_cost(01);
-
- format %{ "FLD StubRoutines::x86::_fpu_subnormal_bias1\n\t"
- "DMULp $dst,ST\n\t"
- "FLD $src\n\t"
- "FDIVp $dst,ST\n\t"
- "FLD StubRoutines::x86::_fpu_subnormal_bias2\n\t"
- "DMULp $dst,ST\n\t" %}
- opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
- ins_encode( strictfp_bias1(dst),
- Push_Reg_DPR(src),
- OpcP, RegOpc(dst),
- strictfp_bias2(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct atanDPR_reg(regDPR dst, regDPR src) %{
- predicate (UseSSE<=1);
- match(Set dst(AtanD dst src));
- format %{ "DATA $dst,$src" %}
- opcode(0xD9, 0xF3);
- ins_encode( Push_Reg_DPR(src),
- OpcP, OpcS, RegOpc(dst) );
- ins_pipe( pipe_slow );
-%}
-
-instruct atanD_reg(regD dst, regD src, eFlagsReg cr) %{
- predicate (UseSSE>=2);
- match(Set dst(AtanD dst src));
- effect(KILL cr); // Push_{Src|Result}D() uses "{SUB|ADD} ESP,8"
- format %{ "DATA $dst,$src" %}
- opcode(0xD9, 0xF3);
- ins_encode( Push_SrcD(src),
- OpcP, OpcS, Push_ResultD(dst) );
- ins_pipe( pipe_slow );
-%}
-
-instruct sqrtDPR_reg(regDPR dst, regDPR src) %{
- predicate (UseSSE<=1);
- match(Set dst (SqrtD src));
- format %{ "DSQRT $dst,$src" %}
- opcode(0xFA, 0xD9);
- ins_encode( Push_Reg_DPR(src),
- OpcS, OpcP, Pop_Reg_DPR(dst) );
- ins_pipe( pipe_slow );
-%}
-
-//-------------Float Instructions-------------------------------
-// Float Math
-
-// Code for float compare:
-// fcompp();
-// fwait(); fnstsw_ax();
-// sahf();
-// movl(dst, unordered_result);
-// jcc(Assembler::parity, exit);
-// movl(dst, less_result);
-// jcc(Assembler::below, exit);
-// movl(dst, equal_result);
-// jcc(Assembler::equal, exit);
-// movl(dst, greater_result);
-// exit:
-
-// P6 version of float compare, sets condition codes in EFLAGS
-instruct cmpFPR_cc_P6(eFlagsRegU cr, regFPR src1, regFPR src2, eAXRegI rax) %{
- predicate(VM_Version::supports_cmov() && UseSSE == 0);
- match(Set cr (CmpF src1 src2));
- effect(KILL rax);
- ins_cost(150);
- format %{ "FLD $src1\n\t"
- "FUCOMIP ST,$src2 // P6 instruction\n\t"
- "JNP exit\n\t"
- "MOV ah,1 // saw a NaN, set CF (treat as LT)\n\t"
- "SAHF\n"
- "exit:\tNOP // avoid branch to branch" %}
- opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2),
- cmpF_P6_fixup );
- ins_pipe( pipe_slow );
-%}
-
-instruct cmpFPR_cc_P6CF(eFlagsRegUCF cr, regFPR src1, regFPR src2) %{
- predicate(VM_Version::supports_cmov() && UseSSE == 0);
- match(Set cr (CmpF src1 src2));
- ins_cost(100);
- format %{ "FLD $src1\n\t"
- "FUCOMIP ST,$src2 // P6 instruction" %}
- opcode(0xDF, 0x05); /* DF E8+i or DF /5 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2));
- ins_pipe( pipe_slow );
-%}
-
-
-// Compare & branch
-instruct cmpFPR_cc(eFlagsRegU cr, regFPR src1, regFPR src2, eAXRegI rax) %{
- predicate(UseSSE == 0);
- match(Set cr (CmpF src1 src2));
- effect(KILL rax);
- ins_cost(200);
- format %{ "FLD $src1\n\t"
- "FCOMp $src2\n\t"
- "FNSTSW AX\n\t"
- "TEST AX,0x400\n\t"
- "JZ,s flags\n\t"
- "MOV AH,1\t# unordered treat as LT\n"
- "flags:\tSAHF" %}
- opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2),
- fpu_flags);
- ins_pipe( pipe_slow );
-%}
-
-// Compare vs zero into -1,0,1
-instruct cmpFPR_0(rRegI dst, regFPR src1, immFPR0 zero, eAXRegI rax, eFlagsReg cr) %{
- predicate(UseSSE == 0);
- match(Set dst (CmpF3 src1 zero));
- effect(KILL cr, KILL rax);
- ins_cost(280);
- format %{ "FTSTF $dst,$src1" %}
- opcode(0xE4, 0xD9);
- ins_encode( Push_Reg_DPR(src1),
- OpcS, OpcP, PopFPU,
- CmpF_Result(dst));
- ins_pipe( pipe_slow );
-%}
-
-// Compare into -1,0,1
-instruct cmpFPR_reg(rRegI dst, regFPR src1, regFPR src2, eAXRegI rax, eFlagsReg cr) %{
- predicate(UseSSE == 0);
- match(Set dst (CmpF3 src1 src2));
- effect(KILL cr, KILL rax);
- ins_cost(300);
- format %{ "FCMPF $dst,$src1,$src2" %}
- opcode(0xD8, 0x3); /* D8 D8+i or D8 /3 */
- ins_encode( Push_Reg_DPR(src1),
- OpcP, RegOpc(src2),
- CmpF_Result(dst));
- ins_pipe( pipe_slow );
-%}
-
-// float compare and set condition codes in EFLAGS by XMM regs
-instruct cmpF_cc(eFlagsRegU cr, regF src1, regF src2) %{
- predicate(UseSSE>=1);
- match(Set cr (CmpF src1 src2));
- ins_cost(145);
- format %{ "UCOMISS $src1,$src2\n\t"
- "JNP,s exit\n\t"
- "PUSHF\t# saw NaN, set CF\n\t"
- "AND [rsp], #0xffffff2b\n\t"
- "POPF\n"
- "exit:" %}
- ins_encode %{
- __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
- emit_cmpfp_fixup(masm);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct cmpF_ccCF(eFlagsRegUCF cr, regF src1, regF src2) %{
- predicate(UseSSE>=1);
- match(Set cr (CmpF src1 src2));
- ins_cost(100);
- format %{ "UCOMISS $src1,$src2" %}
- ins_encode %{
- __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// float compare and set condition codes in EFLAGS by XMM regs
-instruct cmpF_ccmem(eFlagsRegU cr, regF src1, memory src2) %{
- predicate(UseSSE>=1);
- match(Set cr (CmpF src1 (LoadF src2)));
- ins_cost(165);
- format %{ "UCOMISS $src1,$src2\n\t"
- "JNP,s exit\n\t"
- "PUSHF\t# saw NaN, set CF\n\t"
- "AND [rsp], #0xffffff2b\n\t"
- "POPF\n"
- "exit:" %}
- ins_encode %{
- __ ucomiss($src1$$XMMRegister, $src2$$Address);
- emit_cmpfp_fixup(masm);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct cmpF_ccmemCF(eFlagsRegUCF cr, regF src1, memory src2) %{
- predicate(UseSSE>=1);
- match(Set cr (CmpF src1 (LoadF src2)));
- ins_cost(100);
- format %{ "UCOMISS $src1,$src2" %}
- ins_encode %{
- __ ucomiss($src1$$XMMRegister, $src2$$Address);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Compare into -1,0,1 in XMM
-instruct cmpF_reg(xRegI dst, regF src1, regF src2, eFlagsReg cr) %{
- predicate(UseSSE>=1);
- match(Set dst (CmpF3 src1 src2));
- effect(KILL cr);
- ins_cost(255);
- format %{ "UCOMISS $src1, $src2\n\t"
- "MOV $dst, #-1\n\t"
- "JP,s done\n\t"
- "JB,s done\n\t"
- "SETNE $dst\n\t"
- "MOVZB $dst, $dst\n"
- "done:" %}
- ins_encode %{
- __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
- emit_cmpfp3(masm, $dst$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Compare into -1,0,1 in XMM and memory
-instruct cmpF_regmem(xRegI dst, regF src1, memory src2, eFlagsReg cr) %{
- predicate(UseSSE>=1);
- match(Set dst (CmpF3 src1 (LoadF src2)));
- effect(KILL cr);
- ins_cost(275);
- format %{ "UCOMISS $src1, $src2\n\t"
- "MOV $dst, #-1\n\t"
- "JP,s done\n\t"
- "JB,s done\n\t"
- "SETNE $dst\n\t"
- "MOVZB $dst, $dst\n"
- "done:" %}
- ins_encode %{
- __ ucomiss($src1$$XMMRegister, $src2$$Address);
- emit_cmpfp3(masm, $dst$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Spill to obtain 24-bit precision
-instruct subFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (SubF src1 src2));
-
- format %{ "FSUB $dst,$src1 - $src2" %}
- opcode(0xD8, 0x4); /* D8 E0+i or D8 /4 mod==0x3 ;; result in TOS */
- ins_encode( Push_Reg_FPR(src1),
- OpcReg_FPR(src2),
- Pop_Mem_FPR(dst) );
- ins_pipe( fpu_mem_reg_reg );
-%}
-//
-// This instruction does not round to 24-bits
-instruct subFPR_reg(regFPR dst, regFPR src) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (SubF dst src));
-
- format %{ "FSUB $dst,$src" %}
- opcode(0xDE, 0x5); /* DE E8+i or DE /5 */
- ins_encode( Push_Reg_FPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-// Spill to obtain 24-bit precision
-instruct addFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src1 src2));
-
- format %{ "FADD $dst,$src1,$src2" %}
- opcode(0xD8, 0x0); /* D8 C0+i */
- ins_encode( Push_Reg_FPR(src2),
- OpcReg_FPR(src1),
- Pop_Mem_FPR(dst) );
- ins_pipe( fpu_mem_reg_reg );
-%}
-//
-// This instruction does not round to 24-bits
-instruct addFPR_reg(regFPR dst, regFPR src) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (AddF dst src));
-
- format %{ "FLD $src\n\t"
- "FADDp $dst,ST" %}
- opcode(0xDE, 0x0); /* DE C0+i or DE /0*/
- ins_encode( Push_Reg_FPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct absFPR_reg(regFPR1 dst, regFPR1 src) %{
- predicate(UseSSE==0);
- match(Set dst (AbsF src));
- ins_cost(100);
- format %{ "FABS" %}
- opcode(0xE1, 0xD9);
- ins_encode( OpcS, OpcP );
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct negFPR_reg(regFPR1 dst, regFPR1 src) %{
- predicate(UseSSE==0);
- match(Set dst (NegF src));
- ins_cost(100);
- format %{ "FCHS" %}
- opcode(0xE0, 0xD9);
- ins_encode( OpcS, OpcP );
- ins_pipe( fpu_reg_reg );
-%}
-
-// Cisc-alternate to addFPR_reg
-// Spill to obtain 24-bit precision
-instruct addFPR24_reg_mem(stackSlotF dst, regFPR src1, memory src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src1 (LoadF src2)));
-
- format %{ "FLD $src2\n\t"
- "FADD ST,$src1\n\t"
- "FSTP_S $dst" %}
- opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src2),
- OpcReg_FPR(src1),
- Pop_Mem_FPR(dst), ClearInstMark );
- ins_pipe( fpu_mem_reg_mem );
-%}
-//
-// Cisc-alternate to addFPR_reg
-// This instruction does not round to 24-bits
-instruct addFPR_reg_mem(regFPR dst, memory src) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (AddF dst (LoadF src)));
-
- format %{ "FADD $dst,$src" %}
- opcode(0xDE, 0x0, 0xD9); /* DE C0+i or DE /0*/ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src),
- OpcP, RegOpc(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-// // Following two instructions for _222_mpegaudio
-// Spill to obtain 24-bit precision
-instruct addFPR24_mem_reg(stackSlotF dst, regFPR src2, memory src1 ) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src1 src2));
-
- format %{ "FADD $dst,$src1,$src2" %}
- opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src1),
- OpcReg_FPR(src2),
- Pop_Mem_FPR(dst), ClearInstMark );
- ins_pipe( fpu_mem_reg_mem );
-%}
-
-// Cisc-spill variant
-// Spill to obtain 24-bit precision
-instruct addFPR24_mem_cisc(stackSlotF dst, memory src1, memory src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src1 (LoadF src2)));
-
- format %{ "FADD $dst,$src1,$src2 cisc" %}
- opcode(0xD8, 0x0, 0xD9); /* D8 C0+i */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src2),
- OpcP, RMopc_Mem(secondary,src1),
- Pop_Mem_FPR(dst),
- ClearInstMark);
- ins_pipe( fpu_mem_mem_mem );
-%}
-
-// Spill to obtain 24-bit precision
-instruct addFPR24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src1 src2));
-
- format %{ "FADD $dst,$src1,$src2" %}
- opcode(0xD8, 0x0, 0xD9); /* D8 /0 */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src2),
- OpcP, RMopc_Mem(secondary,src1),
- Pop_Mem_FPR(dst),
- ClearInstMark);
- ins_pipe( fpu_mem_mem_mem );
-%}
-
-
-// Spill to obtain 24-bit precision
-instruct addFPR24_reg_imm(stackSlotF dst, regFPR src, immFPR con) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src con));
- format %{ "FLD $src\n\t"
- "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
- "FSTP_S $dst" %}
- ins_encode %{
- __ fld_s($src$$reg - 1); // FLD ST(i-1)
- __ fadd_s($constantaddress($con));
- __ fstp_s(Address(rsp, $dst$$disp));
- %}
- ins_pipe(fpu_mem_reg_con);
-%}
-//
-// This instruction does not round to 24-bits
-instruct addFPR_reg_imm(regFPR dst, regFPR src, immFPR con) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (AddF src con));
- format %{ "FLD $src\n\t"
- "FADD_S [$constantaddress]\t# load from constant table: float=$con\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fld_s($src$$reg - 1); // FLD ST(i-1)
- __ fadd_s($constantaddress($con));
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_reg_con);
-%}
-
-// Spill to obtain 24-bit precision
-instruct mulFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src1 src2));
-
- format %{ "FLD $src1\n\t"
- "FMUL $src2\n\t"
- "FSTP_S $dst" %}
- opcode(0xD8, 0x1); /* D8 C8+i or D8 /1 ;; result in TOS */
- ins_encode( Push_Reg_FPR(src1),
- OpcReg_FPR(src2),
- Pop_Mem_FPR(dst) );
- ins_pipe( fpu_mem_reg_reg );
-%}
-//
-// This instruction does not round to 24-bits
-instruct mulFPR_reg(regFPR dst, regFPR src1, regFPR src2) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src1 src2));
-
- format %{ "FLD $src1\n\t"
- "FMUL $src2\n\t"
- "FSTP_S $dst" %}
- opcode(0xD8, 0x1); /* D8 C8+i */
- ins_encode( Push_Reg_FPR(src2),
- OpcReg_FPR(src1),
- Pop_Reg_FPR(dst) );
- ins_pipe( fpu_reg_reg_reg );
-%}
-
-
-// Spill to obtain 24-bit precision
-// Cisc-alternate to reg-reg multiply
-instruct mulFPR24_reg_mem(stackSlotF dst, regFPR src1, memory src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src1 (LoadF src2)));
-
- format %{ "FLD_S $src2\n\t"
- "FMUL $src1\n\t"
- "FSTP_S $dst" %}
- opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or DE /1*/ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src2),
- OpcReg_FPR(src1),
- Pop_Mem_FPR(dst), ClearInstMark );
- ins_pipe( fpu_mem_reg_mem );
-%}
-//
-// This instruction does not round to 24-bits
-// Cisc-alternate to reg-reg multiply
-instruct mulFPR_reg_mem(regFPR dst, regFPR src1, memory src2) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src1 (LoadF src2)));
-
- format %{ "FMUL $dst,$src1,$src2" %}
- opcode(0xD8, 0x1, 0xD9); /* D8 C8+i */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src2),
- OpcReg_FPR(src1),
- Pop_Reg_FPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_reg_mem );
-%}
-
-// Spill to obtain 24-bit precision
-instruct mulFPR24_mem_mem(stackSlotF dst, memory src1, memory src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src1 src2));
-
- format %{ "FMUL $dst,$src1,$src2" %}
- opcode(0xD8, 0x1, 0xD9); /* D8 /1 */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,src2),
- OpcP, RMopc_Mem(secondary,src1),
- Pop_Mem_FPR(dst),
- ClearInstMark );
- ins_pipe( fpu_mem_mem_mem );
-%}
-
-// Spill to obtain 24-bit precision
-instruct mulFPR24_reg_imm(stackSlotF dst, regFPR src, immFPR con) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src con));
-
- format %{ "FLD $src\n\t"
- "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
- "FSTP_S $dst" %}
- ins_encode %{
- __ fld_s($src$$reg - 1); // FLD ST(i-1)
- __ fmul_s($constantaddress($con));
- __ fstp_s(Address(rsp, $dst$$disp));
- %}
- ins_pipe(fpu_mem_reg_con);
-%}
-//
-// This instruction does not round to 24-bits
-instruct mulFPR_reg_imm(regFPR dst, regFPR src, immFPR con) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (MulF src con));
-
- format %{ "FLD $src\n\t"
- "FMUL_S [$constantaddress]\t# load from constant table: float=$con\n\t"
- "FSTP $dst" %}
- ins_encode %{
- __ fld_s($src$$reg - 1); // FLD ST(i-1)
- __ fmul_s($constantaddress($con));
- __ fstp_d($dst$$reg);
- %}
- ins_pipe(fpu_reg_reg_con);
-%}
-
-
-//
-// MACRO1 -- subsume unshared load into mulFPR
-// This instruction does not round to 24-bits
-instruct mulFPR_reg_load1(regFPR dst, regFPR src, memory mem1 ) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (MulF (LoadF mem1) src));
-
- format %{ "FLD $mem1 ===MACRO1===\n\t"
- "FMUL ST,$src\n\t"
- "FSTP $dst" %}
- opcode(0xD8, 0x1, 0xD9); /* D8 C8+i or D8 /1 */ /* LoadF D9 /0 */
- ins_encode( SetInstMark, Opcode(tertiary), RMopc_Mem(0x00,mem1),
- OpcReg_FPR(src),
- Pop_Reg_FPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_reg_mem );
-%}
-//
-// MACRO2 -- addFPR a mulFPR which subsumed an unshared load
-// This instruction does not round to 24-bits
-instruct addFPR_mulFPR_reg_load1(regFPR dst, memory mem1, regFPR src1, regFPR src2) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (AddF (MulF (LoadF mem1) src1) src2));
- ins_cost(95);
-
- format %{ "FLD $mem1 ===MACRO2===\n\t"
- "FMUL ST,$src1 subsume mulFPR left load\n\t"
- "FADD ST,$src2\n\t"
- "FSTP $dst" %}
- opcode(0xD9); /* LoadF D9 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem1),
- FMul_ST_reg(src1),
- FAdd_ST_reg(src2),
- Pop_Reg_FPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem_reg_reg );
-%}
-
-// MACRO3 -- addFPR a mulFPR
-// This instruction does not round to 24-bits. It is a '2-address'
-// instruction in that the result goes back to src2. This eliminates
-// a move from the macro; possibly the register allocator will have
-// to add it back (and maybe not).
-instruct addFPR_mulFPR_reg(regFPR src2, regFPR src1, regFPR src0) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set src2 (AddF (MulF src0 src1) src2));
-
- format %{ "FLD $src0 ===MACRO3===\n\t"
- "FMUL ST,$src1\n\t"
- "FADDP $src2,ST" %}
- opcode(0xD9); /* LoadF D9 /0 */
- ins_encode( Push_Reg_FPR(src0),
- FMul_ST_reg(src1),
- FAddP_reg_ST(src2) );
- ins_pipe( fpu_reg_reg_reg );
-%}
-
-// MACRO4 -- divFPR subFPR
-// This instruction does not round to 24-bits
-instruct subFPR_divFPR_reg(regFPR dst, regFPR src1, regFPR src2, regFPR src3) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (DivF (SubF src2 src1) src3));
-
- format %{ "FLD $src2 ===MACRO4===\n\t"
- "FSUB ST,$src1\n\t"
- "FDIV ST,$src3\n\t"
- "FSTP $dst" %}
- opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
- ins_encode( Push_Reg_FPR(src2),
- subFPR_divFPR_encode(src1,src3),
- Pop_Reg_FPR(dst) );
- ins_pipe( fpu_reg_reg_reg_reg );
-%}
-
-// Spill to obtain 24-bit precision
-instruct divFPR24_reg(stackSlotF dst, regFPR src1, regFPR src2) %{
- predicate(UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (DivF src1 src2));
-
- format %{ "FDIV $dst,$src1,$src2" %}
- opcode(0xD8, 0x6); /* D8 F0+i or DE /6*/
- ins_encode( Push_Reg_FPR(src1),
- OpcReg_FPR(src2),
- Pop_Mem_FPR(dst) );
- ins_pipe( fpu_mem_reg_reg );
-%}
-//
-// This instruction does not round to 24-bits
-instruct divFPR_reg(regFPR dst, regFPR src) %{
- predicate(UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (DivF dst src));
-
- format %{ "FDIV $dst,$src" %}
- opcode(0xDE, 0x7); /* DE F8+i or DE /7*/
- ins_encode( Push_Reg_FPR(src),
- OpcP, RegOpc(dst) );
- ins_pipe( fpu_reg_reg );
-%}
-
-
-//----------Arithmetic Conversion Instructions---------------------------------
-// The conversions operations are all Alpha sorted. Please keep it that way!
-
-instruct roundFloat_mem_reg(stackSlotF dst, regFPR src) %{
- predicate(UseSSE==0);
- match(Set dst (RoundFloat src));
- ins_cost(125);
- format %{ "FST_S $dst,$src\t# F-round" %}
- ins_encode( Pop_Mem_Reg_FPR(dst, src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-instruct roundDouble_mem_reg(stackSlotD dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (RoundDouble src));
- ins_cost(125);
- format %{ "FST_D $dst,$src\t# D-round" %}
- ins_encode( Pop_Mem_Reg_DPR(dst, src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-// Force rounding to 24-bit precision and 6-bit exponent
-instruct convDPR2FPR_reg(stackSlotF dst, regDPR src) %{
- predicate(UseSSE==0);
- match(Set dst (ConvD2F src));
- format %{ "FST_S $dst,$src\t# F-round" %}
- expand %{
- roundFloat_mem_reg(dst,src);
- %}
-%}
-
-// Force rounding to 24-bit precision and 6-bit exponent
-instruct convDPR2F_reg(regF dst, regDPR src, eFlagsReg cr) %{
- predicate(UseSSE==1);
- match(Set dst (ConvD2F src));
- effect( KILL cr );
- format %{ "SUB ESP,4\n\t"
- "FST_S [ESP],$src\t# F-round\n\t"
- "MOVSS $dst,[ESP]\n\t"
- "ADD ESP,4" %}
- ins_encode %{
- __ subptr(rsp, 4);
- if ($src$$reg != FPR1L_enc) {
- __ fld_s($src$$reg-1);
- __ fstp_s(Address(rsp, 0));
- } else {
- __ fst_s(Address(rsp, 0));
- }
- __ movflt($dst$$XMMRegister, Address(rsp, 0));
- __ addptr(rsp, 4);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Force rounding double precision to single precision
-instruct convD2F_reg(regF dst, regD src) %{
- predicate(UseSSE>=2);
- match(Set dst (ConvD2F src));
- format %{ "CVTSD2SS $dst,$src\t# F-round" %}
- ins_encode %{
- __ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convFPR2DPR_reg_reg(regDPR dst, regFPR src) %{
- predicate(UseSSE==0);
- match(Set dst (ConvF2D src));
- format %{ "FST_S $dst,$src\t# D-round" %}
- ins_encode( Pop_Reg_Reg_DPR(dst, src));
- ins_pipe( fpu_reg_reg );
-%}
-
-instruct convFPR2D_reg(stackSlotD dst, regFPR src) %{
- predicate(UseSSE==1);
- match(Set dst (ConvF2D src));
- format %{ "FST_D $dst,$src\t# D-round" %}
- expand %{
- roundDouble_mem_reg(dst,src);
- %}
-%}
-
-instruct convF2DPR_reg(regDPR dst, regF src, eFlagsReg cr) %{
- predicate(UseSSE==1);
- match(Set dst (ConvF2D src));
- effect( KILL cr );
- format %{ "SUB ESP,4\n\t"
- "MOVSS [ESP] $src\n\t"
- "FLD_S [ESP]\n\t"
- "ADD ESP,4\n\t"
- "FSTP $dst\t# D-round" %}
- ins_encode %{
- __ subptr(rsp, 4);
- __ movflt(Address(rsp, 0), $src$$XMMRegister);
- __ fld_s(Address(rsp, 0));
- __ addptr(rsp, 4);
- __ fstp_d($dst$$reg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convF2D_reg(regD dst, regF src) %{
- predicate(UseSSE>=2);
- match(Set dst (ConvF2D src));
- format %{ "CVTSS2SD $dst,$src\t# D-round" %}
- ins_encode %{
- __ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Convert a double to an int. If the double is a NAN, stuff a zero in instead.
-instruct convDPR2I_reg_reg( eAXRegI dst, eDXRegI tmp, regDPR src, eFlagsReg cr ) %{
- predicate(UseSSE<=1);
- match(Set dst (ConvD2I src));
- effect( KILL tmp, KILL cr );
- format %{ "FLD $src\t# Convert double to int \n\t"
- "FLDCW trunc mode\n\t"
- "SUB ESP,4\n\t"
- "FISTp [ESP + #0]\n\t"
- "FLDCW std/24-bit mode\n\t"
- "POP EAX\n\t"
- "CMP EAX,0x80000000\n\t"
- "JNE,s fast\n\t"
- "FLD_D $src\n\t"
- "CALL d2i_wrapper\n"
- "fast:" %}
- ins_encode( Push_Reg_DPR(src), DPR2I_encoding(src) );
- ins_pipe( pipe_slow );
-%}
-
-// Convert a double to an int. If the double is a NAN, stuff a zero in instead.
-instruct convD2I_reg_reg( eAXRegI dst, eDXRegI tmp, regD src, eFlagsReg cr ) %{
- predicate(UseSSE>=2);
- match(Set dst (ConvD2I src));
- effect( KILL tmp, KILL cr );
- format %{ "CVTTSD2SI $dst, $src\n\t"
- "CMP $dst,0x80000000\n\t"
- "JNE,s fast\n\t"
- "SUB ESP, 8\n\t"
- "MOVSD [ESP], $src\n\t"
- "FLD_D [ESP]\n\t"
- "ADD ESP, 8\n\t"
- "CALL d2i_wrapper\n"
- "fast:" %}
- ins_encode %{
- Label fast;
- __ cvttsd2sil($dst$$Register, $src$$XMMRegister);
- __ cmpl($dst$$Register, 0x80000000);
- __ jccb(Assembler::notEqual, fast);
- __ subptr(rsp, 8);
- __ movdbl(Address(rsp, 0), $src$$XMMRegister);
- __ fld_d(Address(rsp, 0));
- __ addptr(rsp, 8);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_wrapper())));
- __ post_call_nop();
- __ bind(fast);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convDPR2L_reg_reg( eADXRegL dst, regDPR src, eFlagsReg cr ) %{
- predicate(UseSSE<=1);
- match(Set dst (ConvD2L src));
- effect( KILL cr );
- format %{ "FLD $src\t# Convert double to long\n\t"
- "FLDCW trunc mode\n\t"
- "SUB ESP,8\n\t"
- "FISTp [ESP + #0]\n\t"
- "FLDCW std/24-bit mode\n\t"
- "POP EAX\n\t"
- "POP EDX\n\t"
- "CMP EDX,0x80000000\n\t"
- "JNE,s fast\n\t"
- "TEST EAX,EAX\n\t"
- "JNE,s fast\n\t"
- "FLD $src\n\t"
- "CALL d2l_wrapper\n"
- "fast:" %}
- ins_encode( Push_Reg_DPR(src), DPR2L_encoding(src) );
- ins_pipe( pipe_slow );
-%}
-
-// XMM lacks a float/double->long conversion, so use the old FPU stack.
-instruct convD2L_reg_reg( eADXRegL dst, regD src, eFlagsReg cr ) %{
- predicate (UseSSE>=2);
- match(Set dst (ConvD2L src));
- effect( KILL cr );
- format %{ "SUB ESP,8\t# Convert double to long\n\t"
- "MOVSD [ESP],$src\n\t"
- "FLD_D [ESP]\n\t"
- "FLDCW trunc mode\n\t"
- "FISTp [ESP + #0]\n\t"
- "FLDCW std/24-bit mode\n\t"
- "POP EAX\n\t"
- "POP EDX\n\t"
- "CMP EDX,0x80000000\n\t"
- "JNE,s fast\n\t"
- "TEST EAX,EAX\n\t"
- "JNE,s fast\n\t"
- "SUB ESP,8\n\t"
- "MOVSD [ESP],$src\n\t"
- "FLD_D [ESP]\n\t"
- "ADD ESP,8\n\t"
- "CALL d2l_wrapper\n"
- "fast:" %}
- ins_encode %{
- Label fast;
- __ subptr(rsp, 8);
- __ movdbl(Address(rsp, 0), $src$$XMMRegister);
- __ fld_d(Address(rsp, 0));
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_trunc()));
- __ fistp_d(Address(rsp, 0));
- // Restore the rounding mode, mask the exception
- if (Compile::current()->in_24_bit_fp_mode()) {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
- } else {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
- }
- // Load the converted long, adjust CPU stack
- __ pop(rax);
- __ pop(rdx);
- __ cmpl(rdx, 0x80000000);
- __ jccb(Assembler::notEqual, fast);
- __ testl(rax, rax);
- __ jccb(Assembler::notEqual, fast);
- __ subptr(rsp, 8);
- __ movdbl(Address(rsp, 0), $src$$XMMRegister);
- __ fld_d(Address(rsp, 0));
- __ addptr(rsp, 8);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_wrapper())));
- __ post_call_nop();
- __ bind(fast);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Convert a double to an int. Java semantics require we do complex
-// manglations in the corner cases. So we set the rounding mode to
-// 'zero', store the darned double down as an int, and reset the
-// rounding mode to 'nearest'. The hardware stores a flag value down
-// if we would overflow or converted a NAN; we check for this and
-// and go the slow path if needed.
-instruct convFPR2I_reg_reg(eAXRegI dst, eDXRegI tmp, regFPR src, eFlagsReg cr ) %{
- predicate(UseSSE==0);
- match(Set dst (ConvF2I src));
- effect( KILL tmp, KILL cr );
- format %{ "FLD $src\t# Convert float to int \n\t"
- "FLDCW trunc mode\n\t"
- "SUB ESP,4\n\t"
- "FISTp [ESP + #0]\n\t"
- "FLDCW std/24-bit mode\n\t"
- "POP EAX\n\t"
- "CMP EAX,0x80000000\n\t"
- "JNE,s fast\n\t"
- "FLD $src\n\t"
- "CALL d2i_wrapper\n"
- "fast:" %}
- // DPR2I_encoding works for FPR2I
- ins_encode( Push_Reg_FPR(src), DPR2I_encoding(src) );
- ins_pipe( pipe_slow );
-%}
-
-// Convert a float in xmm to an int reg.
-instruct convF2I_reg(eAXRegI dst, eDXRegI tmp, regF src, eFlagsReg cr ) %{
- predicate(UseSSE>=1);
- match(Set dst (ConvF2I src));
- effect( KILL tmp, KILL cr );
- format %{ "CVTTSS2SI $dst, $src\n\t"
- "CMP $dst,0x80000000\n\t"
- "JNE,s fast\n\t"
- "SUB ESP, 4\n\t"
- "MOVSS [ESP], $src\n\t"
- "FLD [ESP]\n\t"
- "ADD ESP, 4\n\t"
- "CALL d2i_wrapper\n"
- "fast:" %}
- ins_encode %{
- Label fast;
- __ cvttss2sil($dst$$Register, $src$$XMMRegister);
- __ cmpl($dst$$Register, 0x80000000);
- __ jccb(Assembler::notEqual, fast);
- __ subptr(rsp, 4);
- __ movflt(Address(rsp, 0), $src$$XMMRegister);
- __ fld_s(Address(rsp, 0));
- __ addptr(rsp, 4);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_wrapper())));
- __ post_call_nop();
- __ bind(fast);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convFPR2L_reg_reg( eADXRegL dst, regFPR src, eFlagsReg cr ) %{
- predicate(UseSSE==0);
- match(Set dst (ConvF2L src));
- effect( KILL cr );
- format %{ "FLD $src\t# Convert float to long\n\t"
- "FLDCW trunc mode\n\t"
- "SUB ESP,8\n\t"
- "FISTp [ESP + #0]\n\t"
- "FLDCW std/24-bit mode\n\t"
- "POP EAX\n\t"
- "POP EDX\n\t"
- "CMP EDX,0x80000000\n\t"
- "JNE,s fast\n\t"
- "TEST EAX,EAX\n\t"
- "JNE,s fast\n\t"
- "FLD $src\n\t"
- "CALL d2l_wrapper\n"
- "fast:" %}
- // DPR2L_encoding works for FPR2L
- ins_encode( Push_Reg_FPR(src), DPR2L_encoding(src) );
- ins_pipe( pipe_slow );
-%}
-
-// XMM lacks a float/double->long conversion, so use the old FPU stack.
-instruct convF2L_reg_reg( eADXRegL dst, regF src, eFlagsReg cr ) %{
- predicate (UseSSE>=1);
- match(Set dst (ConvF2L src));
- effect( KILL cr );
- format %{ "SUB ESP,8\t# Convert float to long\n\t"
- "MOVSS [ESP],$src\n\t"
- "FLD_S [ESP]\n\t"
- "FLDCW trunc mode\n\t"
- "FISTp [ESP + #0]\n\t"
- "FLDCW std/24-bit mode\n\t"
- "POP EAX\n\t"
- "POP EDX\n\t"
- "CMP EDX,0x80000000\n\t"
- "JNE,s fast\n\t"
- "TEST EAX,EAX\n\t"
- "JNE,s fast\n\t"
- "SUB ESP,4\t# Convert float to long\n\t"
- "MOVSS [ESP],$src\n\t"
- "FLD_S [ESP]\n\t"
- "ADD ESP,4\n\t"
- "CALL d2l_wrapper\n"
- "fast:" %}
- ins_encode %{
- Label fast;
- __ subptr(rsp, 8);
- __ movflt(Address(rsp, 0), $src$$XMMRegister);
- __ fld_s(Address(rsp, 0));
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_trunc()));
- __ fistp_d(Address(rsp, 0));
- // Restore the rounding mode, mask the exception
- if (Compile::current()->in_24_bit_fp_mode()) {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
- } else {
- __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
- }
- // Load the converted long, adjust CPU stack
- __ pop(rax);
- __ pop(rdx);
- __ cmpl(rdx, 0x80000000);
- __ jccb(Assembler::notEqual, fast);
- __ testl(rax, rax);
- __ jccb(Assembler::notEqual, fast);
- __ subptr(rsp, 4);
- __ movflt(Address(rsp, 0), $src$$XMMRegister);
- __ fld_s(Address(rsp, 0));
- __ addptr(rsp, 4);
- __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_wrapper())));
- __ post_call_nop();
- __ bind(fast);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convI2DPR_reg(regDPR dst, stackSlotI src) %{
- predicate( UseSSE<=1 );
- match(Set dst (ConvI2D src));
- format %{ "FILD $src\n\t"
- "FSTP $dst" %}
- opcode(0xDB, 0x0); /* DB /0 */
- ins_encode(Push_Mem_I(src), Pop_Reg_DPR(dst));
- ins_pipe( fpu_reg_mem );
-%}
-
-instruct convI2D_reg(regD dst, rRegI src) %{
- predicate( UseSSE>=2 && !UseXmmI2D );
- match(Set dst (ConvI2D src));
- format %{ "CVTSI2SD $dst,$src" %}
- ins_encode %{
- __ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convI2D_mem(regD dst, memory mem) %{
- predicate( UseSSE>=2 );
- match(Set dst (ConvI2D (LoadI mem)));
- format %{ "CVTSI2SD $dst,$mem" %}
- ins_encode %{
- __ cvtsi2sdl ($dst$$XMMRegister, $mem$$Address);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct convXI2D_reg(regD dst, rRegI src)
-%{
- predicate( UseSSE>=2 && UseXmmI2D );
- match(Set dst (ConvI2D src));
-
- format %{ "MOVD $dst,$src\n\t"
- "CVTDQ2PD $dst,$dst\t# i2d" %}
- ins_encode %{
- __ movdl($dst$$XMMRegister, $src$$Register);
- __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister);
- %}
- ins_pipe(pipe_slow); // XXX
-%}
-
-instruct convI2DPR_mem(regDPR dst, memory mem) %{
- predicate( UseSSE<=1 && !Compile::current()->select_24_bit_instr());
- match(Set dst (ConvI2D (LoadI mem)));
- format %{ "FILD $mem\n\t"
- "FSTP $dst" %}
- opcode(0xDB); /* DB /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem),
- Pop_Reg_DPR(dst), ClearInstMark);
- ins_pipe( fpu_reg_mem );
-%}
-
-// Convert a byte to a float; no rounding step needed.
-instruct conv24I2FPR_reg(regFPR dst, stackSlotI src) %{
- predicate( UseSSE==0 && n->in(1)->Opcode() == Op_AndI && n->in(1)->in(2)->is_Con() && n->in(1)->in(2)->get_int() == 255 );
- match(Set dst (ConvI2F src));
- format %{ "FILD $src\n\t"
- "FSTP $dst" %}
-
- opcode(0xDB, 0x0); /* DB /0 */
- ins_encode(Push_Mem_I(src), Pop_Reg_FPR(dst));
- ins_pipe( fpu_reg_mem );
-%}
-
-// In 24-bit mode, force exponent rounding by storing back out
-instruct convI2FPR_SSF(stackSlotF dst, stackSlotI src) %{
- predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (ConvI2F src));
- ins_cost(200);
- format %{ "FILD $src\n\t"
- "FSTP_S $dst" %}
- opcode(0xDB, 0x0); /* DB /0 */
- ins_encode( Push_Mem_I(src),
- Pop_Mem_FPR(dst));
- ins_pipe( fpu_mem_mem );
-%}
-
-// In 24-bit mode, force exponent rounding by storing back out
-instruct convI2FPR_SSF_mem(stackSlotF dst, memory mem) %{
- predicate( UseSSE==0 && Compile::current()->select_24_bit_instr());
- match(Set dst (ConvI2F (LoadI mem)));
- ins_cost(200);
- format %{ "FILD $mem\n\t"
- "FSTP_S $dst" %}
- opcode(0xDB); /* DB /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem),
- Pop_Mem_FPR(dst), ClearInstMark);
- ins_pipe( fpu_mem_mem );
-%}
-
-// This instruction does not round to 24-bits
-instruct convI2FPR_reg(regFPR dst, stackSlotI src) %{
- predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (ConvI2F src));
- format %{ "FILD $src\n\t"
- "FSTP $dst" %}
- opcode(0xDB, 0x0); /* DB /0 */
- ins_encode( Push_Mem_I(src),
- Pop_Reg_FPR(dst));
- ins_pipe( fpu_reg_mem );
-%}
-
-// This instruction does not round to 24-bits
-instruct convI2FPR_mem(regFPR dst, memory mem) %{
- predicate( UseSSE==0 && !Compile::current()->select_24_bit_instr());
- match(Set dst (ConvI2F (LoadI mem)));
- format %{ "FILD $mem\n\t"
- "FSTP $dst" %}
- opcode(0xDB); /* DB /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,mem),
- Pop_Reg_FPR(dst), ClearInstMark);
- ins_pipe( fpu_reg_mem );
-%}
-
-// Convert an int to a float in xmm; no rounding step needed.
-instruct convI2F_reg(regF dst, rRegI src) %{
- predicate( UseSSE==1 || ( UseSSE>=2 && !UseXmmI2F ));
- match(Set dst (ConvI2F src));
- format %{ "CVTSI2SS $dst, $src" %}
- ins_encode %{
- __ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
- instruct convXI2F_reg(regF dst, rRegI src)
-%{
- predicate( UseSSE>=2 && UseXmmI2F );
- match(Set dst (ConvI2F src));
-
- format %{ "MOVD $dst,$src\n\t"
- "CVTDQ2PS $dst,$dst\t# i2f" %}
- ins_encode %{
- __ movdl($dst$$XMMRegister, $src$$Register);
- __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister);
- %}
- ins_pipe(pipe_slow); // XXX
-%}
-
-instruct convI2L_reg( eRegL dst, rRegI src, eFlagsReg cr) %{
- match(Set dst (ConvI2L src));
- effect(KILL cr);
- ins_cost(375);
- format %{ "MOV $dst.lo,$src\n\t"
- "MOV $dst.hi,$src\n\t"
- "SAR $dst.hi,31" %}
- ins_encode(convert_int_long(dst,src));
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Zero-extend convert int to long
-instruct convI2L_reg_zex(eRegL dst, rRegI src, immL_32bits mask, eFlagsReg flags ) %{
- match(Set dst (AndL (ConvI2L src) mask) );
- effect( KILL flags );
- ins_cost(250);
- format %{ "MOV $dst.lo,$src\n\t"
- "XOR $dst.hi,$dst.hi" %}
- opcode(0x33); // XOR
- ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Zero-extend long
-instruct zerox_long(eRegL dst, eRegL src, immL_32bits mask, eFlagsReg flags ) %{
- match(Set dst (AndL src mask) );
- effect( KILL flags );
- ins_cost(250);
- format %{ "MOV $dst.lo,$src.lo\n\t"
- "XOR $dst.hi,$dst.hi\n\t" %}
- opcode(0x33); // XOR
- ins_encode(enc_Copy(dst,src), OpcP, RegReg_Hi2(dst,dst) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-instruct convL2DPR_reg( stackSlotD dst, eRegL src, eFlagsReg cr) %{
- predicate (UseSSE<=1);
- match(Set dst (ConvL2D src));
- effect( KILL cr );
- format %{ "PUSH $src.hi\t# Convert long to double\n\t"
- "PUSH $src.lo\n\t"
- "FILD ST,[ESP + #0]\n\t"
- "ADD ESP,8\n\t"
- "FSTP_D $dst\t# D-round" %}
- opcode(0xDF, 0x5); /* DF /5 */
- ins_encode(convert_long_double(src), Pop_Mem_DPR(dst));
- ins_pipe( pipe_slow );
-%}
-
-instruct convL2D_reg( regD dst, eRegL src, eFlagsReg cr) %{
- predicate (UseSSE>=2);
- match(Set dst (ConvL2D src));
- effect( KILL cr );
- format %{ "PUSH $src.hi\t# Convert long to double\n\t"
- "PUSH $src.lo\n\t"
- "FILD_D [ESP]\n\t"
- "FSTP_D [ESP]\n\t"
- "MOVSD $dst,[ESP]\n\t"
- "ADD ESP,8" %}
- opcode(0xDF, 0x5); /* DF /5 */
- ins_encode(convert_long_double2(src), Push_ResultD(dst));
- ins_pipe( pipe_slow );
-%}
-
-instruct convL2F_reg( regF dst, eRegL src, eFlagsReg cr) %{
- predicate (UseSSE>=1);
- match(Set dst (ConvL2F src));
- effect( KILL cr );
- format %{ "PUSH $src.hi\t# Convert long to single float\n\t"
- "PUSH $src.lo\n\t"
- "FILD_D [ESP]\n\t"
- "FSTP_S [ESP]\n\t"
- "MOVSS $dst,[ESP]\n\t"
- "ADD ESP,8" %}
- opcode(0xDF, 0x5); /* DF /5 */
- ins_encode(convert_long_double2(src), Push_ResultF(dst,0x8));
- ins_pipe( pipe_slow );
-%}
-
-instruct convL2FPR_reg( stackSlotF dst, eRegL src, eFlagsReg cr) %{
- match(Set dst (ConvL2F src));
- effect( KILL cr );
- format %{ "PUSH $src.hi\t# Convert long to single float\n\t"
- "PUSH $src.lo\n\t"
- "FILD ST,[ESP + #0]\n\t"
- "ADD ESP,8\n\t"
- "FSTP_S $dst\t# F-round" %}
- opcode(0xDF, 0x5); /* DF /5 */
- ins_encode(convert_long_double(src), Pop_Mem_FPR(dst));
- ins_pipe( pipe_slow );
-%}
-
-instruct convL2I_reg( rRegI dst, eRegL src ) %{
- match(Set dst (ConvL2I src));
- effect( DEF dst, USE src );
- format %{ "MOV $dst,$src.lo" %}
- ins_encode(enc_CopyL_Lo(dst,src));
- ins_pipe( ialu_reg_reg );
-%}
-
-instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
- match(Set dst (MoveF2I src));
- effect( DEF dst, USE src );
- ins_cost(100);
- format %{ "MOV $dst,$src\t# MoveF2I_stack_reg" %}
- ins_encode %{
- __ movl($dst$$Register, Address(rsp, $src$$disp));
- %}
- ins_pipe( ialu_reg_mem );
-%}
-
-instruct MoveFPR2I_reg_stack(stackSlotI dst, regFPR src) %{
- predicate(UseSSE==0);
- match(Set dst (MoveF2I src));
- effect( DEF dst, USE src );
-
- ins_cost(125);
- format %{ "FST_S $dst,$src\t# MoveF2I_reg_stack" %}
- ins_encode( Pop_Mem_Reg_FPR(dst, src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-instruct MoveF2I_reg_stack_sse(stackSlotI dst, regF src) %{
- predicate(UseSSE>=1);
- match(Set dst (MoveF2I src));
- effect( DEF dst, USE src );
-
- ins_cost(95);
- format %{ "MOVSS $dst,$src\t# MoveF2I_reg_stack_sse" %}
- ins_encode %{
- __ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveF2I_reg_reg_sse(rRegI dst, regF src) %{
- predicate(UseSSE>=2);
- match(Set dst (MoveF2I src));
- effect( DEF dst, USE src );
- ins_cost(85);
- format %{ "MOVD $dst,$src\t# MoveF2I_reg_reg_sse" %}
- ins_encode %{
- __ movdl($dst$$Register, $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
- match(Set dst (MoveI2F src));
- effect( DEF dst, USE src );
-
- ins_cost(100);
- format %{ "MOV $dst,$src\t# MoveI2F_reg_stack" %}
- ins_encode %{
- __ movl(Address(rsp, $dst$$disp), $src$$Register);
- %}
- ins_pipe( ialu_mem_reg );
-%}
-
-
-instruct MoveI2FPR_stack_reg(regFPR dst, stackSlotI src) %{
- predicate(UseSSE==0);
- match(Set dst (MoveI2F src));
- effect(DEF dst, USE src);
-
- ins_cost(125);
- format %{ "FLD_S $src\n\t"
- "FSTP $dst\t# MoveI2F_stack_reg" %}
- opcode(0xD9); /* D9 /0, FLD m32real */
- ins_encode( SetInstMark, OpcP, RMopc_Mem_no_oop(0x00,src),
- Pop_Reg_FPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-instruct MoveI2F_stack_reg_sse(regF dst, stackSlotI src) %{
- predicate(UseSSE>=1);
- match(Set dst (MoveI2F src));
- effect( DEF dst, USE src );
-
- ins_cost(95);
- format %{ "MOVSS $dst,$src\t# MoveI2F_stack_reg_sse" %}
- ins_encode %{
- __ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveI2F_reg_reg_sse(regF dst, rRegI src) %{
- predicate(UseSSE>=2);
- match(Set dst (MoveI2F src));
- effect( DEF dst, USE src );
-
- ins_cost(85);
- format %{ "MOVD $dst,$src\t# MoveI2F_reg_reg_sse" %}
- ins_encode %{
- __ movdl($dst$$XMMRegister, $src$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveD2L_stack_reg(eRegL dst, stackSlotD src) %{
- match(Set dst (MoveD2L src));
- effect(DEF dst, USE src);
-
- ins_cost(250);
- format %{ "MOV $dst.lo,$src\n\t"
- "MOV $dst.hi,$src+4\t# MoveD2L_stack_reg" %}
- opcode(0x8B, 0x8B);
- ins_encode( SetInstMark, OpcP, RegMem(dst,src), OpcS, RegMem_Hi(dst,src), ClearInstMark);
- ins_pipe( ialu_mem_long_reg );
-%}
-
-instruct MoveDPR2L_reg_stack(stackSlotL dst, regDPR src) %{
- predicate(UseSSE<=1);
- match(Set dst (MoveD2L src));
- effect(DEF dst, USE src);
-
- ins_cost(125);
- format %{ "FST_D $dst,$src\t# MoveD2L_reg_stack" %}
- ins_encode( Pop_Mem_Reg_DPR(dst, src) );
- ins_pipe( fpu_mem_reg );
-%}
-
-instruct MoveD2L_reg_stack_sse(stackSlotL dst, regD src) %{
- predicate(UseSSE>=2);
- match(Set dst (MoveD2L src));
- effect(DEF dst, USE src);
- ins_cost(95);
- format %{ "MOVSD $dst,$src\t# MoveD2L_reg_stack_sse" %}
- ins_encode %{
- __ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveD2L_reg_reg_sse(eRegL dst, regD src, regD tmp) %{
- predicate(UseSSE>=2);
- match(Set dst (MoveD2L src));
- effect(DEF dst, USE src, TEMP tmp);
- ins_cost(85);
- format %{ "MOVD $dst.lo,$src\n\t"
- "PSHUFLW $tmp,$src,0x4E\n\t"
- "MOVD $dst.hi,$tmp\t# MoveD2L_reg_reg_sse" %}
- ins_encode %{
- __ movdl($dst$$Register, $src$$XMMRegister);
- __ pshuflw($tmp$$XMMRegister, $src$$XMMRegister, 0x4e);
- __ movdl(HIGH_FROM_LOW($dst$$Register), $tmp$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveL2D_reg_stack(stackSlotD dst, eRegL src) %{
- match(Set dst (MoveL2D src));
- effect(DEF dst, USE src);
-
- ins_cost(200);
- format %{ "MOV $dst,$src.lo\n\t"
- "MOV $dst+4,$src.hi\t# MoveL2D_reg_stack" %}
- opcode(0x89, 0x89);
- ins_encode( SetInstMark, OpcP, RegMem( src, dst ), OpcS, RegMem_Hi( src, dst ), ClearInstMark );
- ins_pipe( ialu_mem_long_reg );
-%}
-
-
-instruct MoveL2DPR_stack_reg(regDPR dst, stackSlotL src) %{
- predicate(UseSSE<=1);
- match(Set dst (MoveL2D src));
- effect(DEF dst, USE src);
- ins_cost(125);
-
- format %{ "FLD_D $src\n\t"
- "FSTP $dst\t# MoveL2D_stack_reg" %}
- opcode(0xDD); /* DD /0, FLD m64real */
- ins_encode( SetInstMark, OpcP, RMopc_Mem_no_oop(0x00,src),
- Pop_Reg_DPR(dst), ClearInstMark );
- ins_pipe( fpu_reg_mem );
-%}
-
-
-instruct MoveL2D_stack_reg_sse(regD dst, stackSlotL src) %{
- predicate(UseSSE>=2 && UseXmmLoadAndClearUpper);
- match(Set dst (MoveL2D src));
- effect(DEF dst, USE src);
-
- ins_cost(95);
- format %{ "MOVSD $dst,$src\t# MoveL2D_stack_reg_sse" %}
- ins_encode %{
- __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveL2D_stack_reg_sse_partial(regD dst, stackSlotL src) %{
- predicate(UseSSE>=2 && !UseXmmLoadAndClearUpper);
- match(Set dst (MoveL2D src));
- effect(DEF dst, USE src);
-
- ins_cost(95);
- format %{ "MOVLPD $dst,$src\t# MoveL2D_stack_reg_sse" %}
- ins_encode %{
- __ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct MoveL2D_reg_reg_sse(regD dst, eRegL src, regD tmp) %{
- predicate(UseSSE>=2);
- match(Set dst (MoveL2D src));
- effect(TEMP dst, USE src, TEMP tmp);
- ins_cost(85);
- format %{ "MOVD $dst,$src.lo\n\t"
- "MOVD $tmp,$src.hi\n\t"
- "PUNPCKLDQ $dst,$tmp\t# MoveL2D_reg_reg_sse" %}
- ins_encode %{
- __ movdl($dst$$XMMRegister, $src$$Register);
- __ movdl($tmp$$XMMRegister, HIGH_FROM_LOW($src$$Register));
- __ punpckldq($dst$$XMMRegister, $tmp$$XMMRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-//----------------------------- CompressBits/ExpandBits ------------------------
-
-instruct compressBitsL_reg(eADXRegL dst, eBCXRegL src, eBDPRegL mask, eSIRegI rtmp, regF xtmp, eFlagsReg cr) %{
- predicate(n->bottom_type()->isa_long());
- match(Set dst (CompressBits src mask));
- effect(TEMP rtmp, TEMP xtmp, KILL cr);
- format %{ "compress_bits $dst, $src, $mask\t! using $rtmp and $xtmp as TEMP" %}
- ins_encode %{
- Label exit, partail_result;
- // Parallely extract both upper and lower 32 bits of source into destination register pair.
- // Merge the results of upper and lower destination registers such that upper destination
- // results are contiguously laid out after the lower destination result.
- __ pextl($dst$$Register, $src$$Register, $mask$$Register);
- __ pextl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($src$$Register), HIGH_FROM_LOW($mask$$Register));
- __ popcntl($rtmp$$Register, $mask$$Register);
- // Skip merging if bit count of lower mask register is equal to 32 (register size).
- __ cmpl($rtmp$$Register, 32);
- __ jccb(Assembler::equal, exit);
- // Due to constraint on number of GPRs on 32 bit target, using XMM register as potential spill slot.
- __ movdl($xtmp$$XMMRegister, $rtmp$$Register);
- // Shift left the contents of upper destination register by true bit count of lower mask register
- // and merge with lower destination register.
- __ shlxl($rtmp$$Register, HIGH_FROM_LOW($dst$$Register), $rtmp$$Register);
- __ orl($dst$$Register, $rtmp$$Register);
- __ movdl($rtmp$$Register, $xtmp$$XMMRegister);
- // Zero out upper destination register if true bit count of lower 32 bit mask is zero
- // since contents of upper destination have already been copied to lower destination
- // register.
- __ cmpl($rtmp$$Register, 0);
- __ jccb(Assembler::greater, partail_result);
- __ movl(HIGH_FROM_LOW($dst$$Register), 0);
- __ jmp(exit);
- __ bind(partail_result);
- // Perform right shift over upper destination register to move out bits already copied
- // to lower destination register.
- __ subl($rtmp$$Register, 32);
- __ negl($rtmp$$Register);
- __ shrxl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($dst$$Register), $rtmp$$Register);
- __ bind(exit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct expandBitsL_reg(eADXRegL dst, eBCXRegL src, eBDPRegL mask, eSIRegI rtmp, regF xtmp, eFlagsReg cr) %{
- predicate(n->bottom_type()->isa_long());
- match(Set dst (ExpandBits src mask));
- effect(TEMP rtmp, TEMP xtmp, KILL cr);
- format %{ "expand_bits $dst, $src, $mask\t! using $rtmp and $xtmp as TEMP" %}
- ins_encode %{
- // Extraction operation sequentially reads the bits from source register starting from LSB
- // and lays them out into destination register at bit locations corresponding to true bits
- // in mask register. Thus number of source bits read are equal to combined true bit count
- // of mask register pair.
- Label exit, mask_clipping;
- __ pdepl($dst$$Register, $src$$Register, $mask$$Register);
- __ pdepl(HIGH_FROM_LOW($dst$$Register), HIGH_FROM_LOW($src$$Register), HIGH_FROM_LOW($mask$$Register));
- __ popcntl($rtmp$$Register, $mask$$Register);
- // If true bit count of lower mask register is 32 then none of bit of lower source register
- // will feed to upper destination register.
- __ cmpl($rtmp$$Register, 32);
- __ jccb(Assembler::equal, exit);
- // Due to constraint on number of GPRs on 32 bit target, using XMM register as potential spill slot.
- __ movdl($xtmp$$XMMRegister, $rtmp$$Register);
- // Shift right the contents of lower source register to remove already consumed bits.
- __ shrxl($rtmp$$Register, $src$$Register, $rtmp$$Register);
- // Extract the bits from lower source register starting from LSB under the influence
- // of upper mask register.
- __ pdepl(HIGH_FROM_LOW($dst$$Register), $rtmp$$Register, HIGH_FROM_LOW($mask$$Register));
- __ movdl($rtmp$$Register, $xtmp$$XMMRegister);
- __ subl($rtmp$$Register, 32);
- __ negl($rtmp$$Register);
- __ movdl($xtmp$$XMMRegister, $mask$$Register);
- __ movl($mask$$Register, HIGH_FROM_LOW($mask$$Register));
- // Clear the set bits in upper mask register which have been used to extract the contents
- // from lower source register.
- __ bind(mask_clipping);
- __ blsrl($mask$$Register, $mask$$Register);
- __ decrementl($rtmp$$Register, 1);
- __ jccb(Assembler::greater, mask_clipping);
- // Starting from LSB extract the bits from upper source register under the influence of
- // remaining set bits in upper mask register.
- __ pdepl($rtmp$$Register, HIGH_FROM_LOW($src$$Register), $mask$$Register);
- // Merge the partial results extracted from lower and upper source register bits.
- __ orl(HIGH_FROM_LOW($dst$$Register), $rtmp$$Register);
- __ movdl($mask$$Register, $xtmp$$XMMRegister);
- __ bind(exit);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// =======================================================================
-// Fast clearing of an array
-// Small non-constant length ClearArray for non-AVX512 targets.
-instruct rep_stos(eCXRegI cnt, eDIRegP base, regD tmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
- predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX <= 2));
- match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
-
- format %{ $$template
- $$emit$$"XOR EAX,EAX\t# ClearArray:\n\t"
- $$emit$$"CMP InitArrayShortSize,rcx\n\t"
- $$emit$$"JG LARGE\n\t"
- $$emit$$"SHL ECX, 1\n\t"
- $$emit$$"DEC ECX\n\t"
- $$emit$$"JS DONE\t# Zero length\n\t"
- $$emit$$"MOV EAX,(EDI,ECX,4)\t# LOOP\n\t"
- $$emit$$"DEC ECX\n\t"
- $$emit$$"JGE LOOP\n\t"
- $$emit$$"JMP DONE\n\t"
- $$emit$$"# LARGE:\n\t"
- if (UseFastStosb) {
- $$emit$$"SHL ECX,3\t# Convert doublewords to bytes\n\t"
- $$emit$$"REP STOSB\t# store EAX into [EDI++] while ECX--\n\t"
- } else if (UseXMMForObjInit) {
- $$emit$$"MOV RDI,RAX\n\t"
- $$emit$$"VPXOR YMM0,YMM0,YMM0\n\t"
- $$emit$$"JMPQ L_zero_64_bytes\n\t"
- $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"VMOVDQU YMM0,0x20(RAX)\n\t"
- $$emit$$"ADD 0x40,RAX\n\t"
- $$emit$$"# L_zero_64_bytes:\n\t"
- $$emit$$"SUB 0x8,RCX\n\t"
- $$emit$$"JGE L_loop\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JL L_tail\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"ADD 0x20,RAX\n\t"
- $$emit$$"SUB 0x4,RCX\n\t"
- $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JLE L_end\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
- $$emit$$"VMOVQ XMM0,(RAX)\n\t"
- $$emit$$"ADD 0x8,RAX\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"JGE L_sloop\n\t"
- $$emit$$"# L_end:\n\t"
- } else {
- $$emit$$"SHL ECX,1\t# Convert doublewords to words\n\t"
- $$emit$$"REP STOS\t# store EAX into [EDI++] while ECX--\n\t"
- }
- $$emit$$"# DONE"
- %}
- ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, false, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Small non-constant length ClearArray for AVX512 targets.
-instruct rep_stos_evex(eCXRegI cnt, eDIRegP base, legRegD tmp, kReg ktmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
- predicate(!((ClearArrayNode*)n)->is_large() && (UseAVX > 2));
- match(Set dummy (ClearArray cnt base));
- ins_cost(125);
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
-
- format %{ $$template
- $$emit$$"XOR EAX,EAX\t# ClearArray:\n\t"
- $$emit$$"CMP InitArrayShortSize,rcx\n\t"
- $$emit$$"JG LARGE\n\t"
- $$emit$$"SHL ECX, 1\n\t"
- $$emit$$"DEC ECX\n\t"
- $$emit$$"JS DONE\t# Zero length\n\t"
- $$emit$$"MOV EAX,(EDI,ECX,4)\t# LOOP\n\t"
- $$emit$$"DEC ECX\n\t"
- $$emit$$"JGE LOOP\n\t"
- $$emit$$"JMP DONE\n\t"
- $$emit$$"# LARGE:\n\t"
- if (UseFastStosb) {
- $$emit$$"SHL ECX,3\t# Convert doublewords to bytes\n\t"
- $$emit$$"REP STOSB\t# store EAX into [EDI++] while ECX--\n\t"
- } else if (UseXMMForObjInit) {
- $$emit$$"MOV RDI,RAX\n\t"
- $$emit$$"VPXOR YMM0,YMM0,YMM0\n\t"
- $$emit$$"JMPQ L_zero_64_bytes\n\t"
- $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"VMOVDQU YMM0,0x20(RAX)\n\t"
- $$emit$$"ADD 0x40,RAX\n\t"
- $$emit$$"# L_zero_64_bytes:\n\t"
- $$emit$$"SUB 0x8,RCX\n\t"
- $$emit$$"JGE L_loop\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JL L_tail\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"ADD 0x20,RAX\n\t"
- $$emit$$"SUB 0x4,RCX\n\t"
- $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JLE L_end\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
- $$emit$$"VMOVQ XMM0,(RAX)\n\t"
- $$emit$$"ADD 0x8,RAX\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"JGE L_sloop\n\t"
- $$emit$$"# L_end:\n\t"
- } else {
- $$emit$$"SHL ECX,1\t# Convert doublewords to words\n\t"
- $$emit$$"REP STOS\t# store EAX into [EDI++] while ECX--\n\t"
- }
- $$emit$$"# DONE"
- %}
- ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, false, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Large non-constant length ClearArray for non-AVX512 targets.
-instruct rep_stos_large(eCXRegI cnt, eDIRegP base, regD tmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
- predicate((UseAVX <= 2) && ((ClearArrayNode*)n)->is_large());
- match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, KILL zero, KILL cr);
- format %{ $$template
- if (UseFastStosb) {
- $$emit$$"XOR EAX,EAX\t# ClearArray:\n\t"
- $$emit$$"SHL ECX,3\t# Convert doublewords to bytes\n\t"
- $$emit$$"REP STOSB\t# store EAX into [EDI++] while ECX--\n\t"
- } else if (UseXMMForObjInit) {
- $$emit$$"MOV RDI,RAX\t# ClearArray:\n\t"
- $$emit$$"VPXOR YMM0,YMM0,YMM0\n\t"
- $$emit$$"JMPQ L_zero_64_bytes\n\t"
- $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"VMOVDQU YMM0,0x20(RAX)\n\t"
- $$emit$$"ADD 0x40,RAX\n\t"
- $$emit$$"# L_zero_64_bytes:\n\t"
- $$emit$$"SUB 0x8,RCX\n\t"
- $$emit$$"JGE L_loop\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JL L_tail\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"ADD 0x20,RAX\n\t"
- $$emit$$"SUB 0x4,RCX\n\t"
- $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JLE L_end\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
- $$emit$$"VMOVQ XMM0,(RAX)\n\t"
- $$emit$$"ADD 0x8,RAX\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"JGE L_sloop\n\t"
- $$emit$$"# L_end:\n\t"
- } else {
- $$emit$$"XOR EAX,EAX\t# ClearArray:\n\t"
- $$emit$$"SHL ECX,1\t# Convert doublewords to words\n\t"
- $$emit$$"REP STOS\t# store EAX into [EDI++] while ECX--\n\t"
- }
- $$emit$$"# DONE"
- %}
- ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, true, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Large non-constant length ClearArray for AVX512 targets.
-instruct rep_stos_large_evex(eCXRegI cnt, eDIRegP base, legRegD tmp, kReg ktmp, eAXRegI zero, Universe dummy, eFlagsReg cr) %{
- predicate((UseAVX > 2) && ((ClearArrayNode*)n)->is_large());
- match(Set dummy (ClearArray cnt base));
- effect(USE_KILL cnt, USE_KILL base, TEMP tmp, TEMP ktmp, KILL zero, KILL cr);
- format %{ $$template
- if (UseFastStosb) {
- $$emit$$"XOR EAX,EAX\t# ClearArray:\n\t"
- $$emit$$"SHL ECX,3\t# Convert doublewords to bytes\n\t"
- $$emit$$"REP STOSB\t# store EAX into [EDI++] while ECX--\n\t"
- } else if (UseXMMForObjInit) {
- $$emit$$"MOV RDI,RAX\t# ClearArray:\n\t"
- $$emit$$"VPXOR YMM0,YMM0,YMM0\n\t"
- $$emit$$"JMPQ L_zero_64_bytes\n\t"
- $$emit$$"# L_loop:\t# 64-byte LOOP\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"VMOVDQU YMM0,0x20(RAX)\n\t"
- $$emit$$"ADD 0x40,RAX\n\t"
- $$emit$$"# L_zero_64_bytes:\n\t"
- $$emit$$"SUB 0x8,RCX\n\t"
- $$emit$$"JGE L_loop\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JL L_tail\n\t"
- $$emit$$"VMOVDQU YMM0,(RAX)\n\t"
- $$emit$$"ADD 0x20,RAX\n\t"
- $$emit$$"SUB 0x4,RCX\n\t"
- $$emit$$"# L_tail:\t# Clearing tail bytes\n\t"
- $$emit$$"ADD 0x4,RCX\n\t"
- $$emit$$"JLE L_end\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"# L_sloop:\t# 8-byte short loop\n\t"
- $$emit$$"VMOVQ XMM0,(RAX)\n\t"
- $$emit$$"ADD 0x8,RAX\n\t"
- $$emit$$"DEC RCX\n\t"
- $$emit$$"JGE L_sloop\n\t"
- $$emit$$"# L_end:\n\t"
- } else {
- $$emit$$"XOR EAX,EAX\t# ClearArray:\n\t"
- $$emit$$"SHL ECX,1\t# Convert doublewords to words\n\t"
- $$emit$$"REP STOS\t# store EAX into [EDI++] while ECX--\n\t"
- }
- $$emit$$"# DONE"
- %}
- ins_encode %{
- __ clear_mem($base$$Register, $cnt$$Register, $zero$$Register,
- $tmp$$XMMRegister, true, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// Small constant length ClearArray for AVX512 targets.
-instruct rep_stos_im(immI cnt, kReg ktmp, eRegP base, regD tmp, rRegI zero, Universe dummy, eFlagsReg cr)
-%{
- predicate(!((ClearArrayNode*)n)->is_large() && (MaxVectorSize >= 32) && VM_Version::supports_avx512vl());
- match(Set dummy (ClearArray cnt base));
- ins_cost(100);
- effect(TEMP tmp, TEMP zero, TEMP ktmp, KILL cr);
- format %{ "clear_mem_imm $base , $cnt \n\t" %}
- ins_encode %{
- __ clear_mem($base$$Register, $cnt$$constant, $zero$$Register, $tmp$$XMMRegister, $ktmp$$KRegister);
- %}
- ins_pipe(pipe_slow);
-%}
-
-instruct string_compareL(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
- eAXRegI result, regD tmp1, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::LL, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareL_evex(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
- eAXRegI result, regD tmp1, kReg ktmp, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::LL, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareU(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
- eAXRegI result, regD tmp1, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::UU, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareU_evex(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
- eAXRegI result, regD tmp1, kReg ktmp, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::UU, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareLU(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
- eAXRegI result, regD tmp1, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::LU, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareLU_evex(eDIRegP str1, eCXRegI cnt1, eSIRegP str2, eDXRegI cnt2,
- eAXRegI result, regD tmp1, kReg ktmp, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::LU, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareUL(eSIRegP str1, eDXRegI cnt1, eDIRegP str2, eCXRegI cnt2,
- eAXRegI result, regD tmp1, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str2$$Register, $str1$$Register,
- $cnt2$$Register, $cnt1$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::UL, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compareUL_evex(eSIRegP str1, eDXRegI cnt1, eDIRegP str2, eCXRegI cnt2,
- eAXRegI result, regD tmp1, kReg ktmp, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw() && ((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL);
- match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP tmp1, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
-
- format %{ "String Compare byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL $tmp1" %}
- ins_encode %{
- __ string_compare($str2$$Register, $str1$$Register,
- $cnt2$$Register, $cnt1$$Register, $result$$Register,
- $tmp1$$XMMRegister, StrIntrinsicNode::UL, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// fast string equals
-instruct string_equals(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
- regD tmp1, regD tmp2, eBXRegI tmp3, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw());
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(TEMP tmp1, TEMP tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
-
- format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
- ins_encode %{
- __ arrays_equals(false, $str1$$Register, $str2$$Register,
- $cnt$$Register, $result$$Register, $tmp3$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
- %}
-
- ins_pipe( pipe_slow );
-%}
-
-instruct string_equals_evex(eDIRegP str1, eSIRegP str2, eCXRegI cnt, eAXRegI result,
- regD tmp1, regD tmp2, kReg ktmp, eBXRegI tmp3, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw());
- match(Set result (StrEquals (Binary str1 str2) cnt));
- effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL tmp3, KILL cr);
-
- format %{ "String Equals $str1,$str2,$cnt -> $result // KILL $tmp1, $tmp2, $tmp3" %}
- ins_encode %{
- __ arrays_equals(false, $str1$$Register, $str2$$Register,
- $cnt$$Register, $result$$Register, $tmp3$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
- %}
-
- ins_pipe( pipe_slow );
-%}
-
-
-// fast search of substring with known size.
-instruct string_indexof_conL(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, immI int_cnt2,
- eBXRegI result, regD vec1, eAXRegI cnt2, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
- match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
- effect(TEMP vec1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
-
- format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $vec1, $cnt1, $cnt2, $tmp" %}
- ins_encode %{
- int icnt2 = (int)$int_cnt2$$constant;
- if (icnt2 >= 16) {
- // IndexOf for constant substrings with size >= 16 elements
- // which don't need to be loaded through stack.
- __ string_indexofC8($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- icnt2, $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
- } else {
- // Small strings are loaded through stack if they cross page boundary.
- __ string_indexof($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- icnt2, $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
- }
- %}
- ins_pipe( pipe_slow );
-%}
-
-// fast search of substring with known size.
-instruct string_indexof_conU(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, immI int_cnt2,
- eBXRegI result, regD vec1, eAXRegI cnt2, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
- match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
- effect(TEMP vec1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
-
- format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $vec1, $cnt1, $cnt2, $tmp" %}
- ins_encode %{
- int icnt2 = (int)$int_cnt2$$constant;
- if (icnt2 >= 8) {
- // IndexOf for constant substrings with size >= 8 elements
- // which don't need to be loaded through stack.
- __ string_indexofC8($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- icnt2, $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
- } else {
- // Small strings are loaded through stack if they cross page boundary.
- __ string_indexof($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- icnt2, $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
- }
- %}
- ins_pipe( pipe_slow );
-%}
-
-// fast search of substring with known size.
-instruct string_indexof_conUL(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, immI int_cnt2,
- eBXRegI result, regD vec1, eAXRegI cnt2, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
- match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
- effect(TEMP vec1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, KILL cnt2, KILL tmp, KILL cr);
-
- format %{ "String IndexOf char[] $str1,$cnt1,$str2,$int_cnt2 -> $result // KILL $vec1, $cnt1, $cnt2, $tmp" %}
- ins_encode %{
- int icnt2 = (int)$int_cnt2$$constant;
- if (icnt2 >= 8) {
- // IndexOf for constant substrings with size >= 8 elements
- // which don't need to be loaded through stack.
- __ string_indexofC8($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- icnt2, $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
- } else {
- // Small strings are loaded through stack if they cross page boundary.
- __ string_indexof($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- icnt2, $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
- }
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_indexofL(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
- eBXRegI result, regD vec1, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL));
- match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP vec1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
-
- format %{ "String IndexOf byte[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
- ins_encode %{
- __ string_indexof($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- (-1), $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::LL);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_indexofU(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
- eBXRegI result, regD vec1, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU));
- match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP vec1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
-
- format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
- ins_encode %{
- __ string_indexof($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- (-1), $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UU);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_indexofUL(eDIRegP str1, eDXRegI cnt1, eSIRegP str2, eAXRegI cnt2,
- eBXRegI result, regD vec1, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL));
- match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
- effect(TEMP vec1, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL tmp, KILL cr);
-
- format %{ "String IndexOf char[] $str1,$cnt1,$str2,$cnt2 -> $result // KILL all" %}
- ins_encode %{
- __ string_indexof($str1$$Register, $str2$$Register,
- $cnt1$$Register, $cnt2$$Register,
- (-1), $result$$Register,
- $vec1$$XMMRegister, $tmp$$Register, StrIntrinsicNode::UL);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_indexof_char(eDIRegP str1, eDXRegI cnt1, eAXRegI ch,
- eBXRegI result, regD vec1, regD vec2, regD vec3, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
- match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
- effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
- format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
- ins_encode %{
- __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
- $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct stringL_indexof_char(eDIRegP str1, eDXRegI cnt1, eAXRegI ch,
- eBXRegI result, regD vec1, regD vec2, regD vec3, eCXRegI tmp, eFlagsReg cr) %{
- predicate(UseSSE42Intrinsics && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
- match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
- effect(TEMP vec1, TEMP vec2, TEMP vec3, USE_KILL str1, USE_KILL cnt1, USE_KILL ch, TEMP tmp, KILL cr);
- format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result // KILL all" %}
- ins_encode %{
- __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
- $vec1$$XMMRegister, $vec2$$XMMRegister, $vec3$$XMMRegister, $tmp$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-
-// fast array equals
-instruct array_equalsB(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
- regD tmp1, regD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
-%{
- predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
- match(Set result (AryEq ary1 ary2));
- effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
- //ins_cost(300);
-
- format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
- ins_encode %{
- __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
- $tmp3$$Register, $result$$Register, $tmp4$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct array_equalsB_evex(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
- regD tmp1, regD tmp2, kReg ktmp, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
-%{
- predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
- match(Set result (AryEq ary1 ary2));
- effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
- //ins_cost(300);
-
- format %{ "Array Equals byte[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
- ins_encode %{
- __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
- $tmp3$$Register, $result$$Register, $tmp4$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, false /* char */, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct array_equalsC(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
- regD tmp1, regD tmp2, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
-%{
- predicate(!VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (AryEq ary1 ary2));
- effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
- //ins_cost(300);
-
- format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
- ins_encode %{
- __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
- $tmp3$$Register, $result$$Register, $tmp4$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct array_equalsC_evex(eDIRegP ary1, eSIRegP ary2, eAXRegI result,
- regD tmp1, regD tmp2, kReg ktmp, eCXRegI tmp3, eBXRegI tmp4, eFlagsReg cr)
-%{
- predicate(VM_Version::supports_avx512vlbw() && ((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
- match(Set result (AryEq ary1 ary2));
- effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL ary1, USE_KILL ary2, KILL tmp3, KILL tmp4, KILL cr);
- //ins_cost(300);
-
- format %{ "Array Equals char[] $ary1,$ary2 -> $result // KILL $tmp1, $tmp2, $tmp3, $tmp4" %}
- ins_encode %{
- __ arrays_equals(true, $ary1$$Register, $ary2$$Register,
- $tmp3$$Register, $result$$Register, $tmp4$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, true /* char */, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct count_positives(eSIRegP ary1, eCXRegI len, eAXRegI result,
- regD tmp1, regD tmp2, eBXRegI tmp3, eFlagsReg cr)
-%{
- predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
- match(Set result (CountPositives ary1 len));
- effect(TEMP tmp1, TEMP tmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
-
- format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
- ins_encode %{
- __ count_positives($ary1$$Register, $len$$Register,
- $result$$Register, $tmp3$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, knoreg, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct count_positives_evex(eSIRegP ary1, eCXRegI len, eAXRegI result,
- regD tmp1, regD tmp2, kReg ktmp1, kReg ktmp2, eBXRegI tmp3, eFlagsReg cr)
-%{
- predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
- match(Set result (CountPositives ary1 len));
- effect(TEMP tmp1, TEMP tmp2, TEMP ktmp1, TEMP ktmp2, USE_KILL ary1, USE_KILL len, KILL tmp3, KILL cr);
-
- format %{ "countPositives byte[] $ary1,$len -> $result // KILL $tmp1, $tmp2, $tmp3" %}
- ins_encode %{
- __ count_positives($ary1$$Register, $len$$Register,
- $result$$Register, $tmp3$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, $ktmp1$$KRegister, $ktmp2$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-
-// fast char[] to byte[] compression
-instruct string_compress(eSIRegP src, eDIRegP dst, eDXRegI len, regD tmp1, regD tmp2,
- regD tmp3, regD tmp4, eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
- match(Set result (StrCompressedCopy src (Binary dst len)));
- effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
-
- format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
- ins_encode %{
- __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
- $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
- knoreg, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_compress_evex(eSIRegP src, eDIRegP dst, eDXRegI len, regD tmp1, regD tmp2,
- regD tmp3, regD tmp4, kReg ktmp1, kReg ktmp2, eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
- match(Set result (StrCompressedCopy src (Binary dst len)));
- effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ktmp1, TEMP ktmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
-
- format %{ "String Compress $src,$dst -> $result // KILL RAX, RCX, RDX" %}
- ins_encode %{
- __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
- $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register,
- $ktmp1$$KRegister, $ktmp2$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// fast byte[] to char[] inflation
-instruct string_inflate(Universe dummy, eSIRegP src, eDIRegP dst, eDXRegI len,
- regD tmp1, eCXRegI tmp2, eFlagsReg cr) %{
- predicate(!VM_Version::supports_avx512vlbw() || !VM_Version::supports_bmi2());
- match(Set dummy (StrInflatedCopy src (Binary dst len)));
- effect(TEMP tmp1, TEMP tmp2, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
-
- format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
- ins_encode %{
- __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
- $tmp1$$XMMRegister, $tmp2$$Register, knoreg);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct string_inflate_evex(Universe dummy, eSIRegP src, eDIRegP dst, eDXRegI len,
- regD tmp1, kReg ktmp, eCXRegI tmp2, eFlagsReg cr) %{
- predicate(VM_Version::supports_avx512vlbw() && VM_Version::supports_bmi2());
- match(Set dummy (StrInflatedCopy src (Binary dst len)));
- effect(TEMP tmp1, TEMP tmp2, TEMP ktmp, USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
-
- format %{ "String Inflate $src,$dst // KILL $tmp1, $tmp2" %}
- ins_encode %{
- __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
- $tmp1$$XMMRegister, $tmp2$$Register, $ktmp$$KRegister);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// encode char[] to byte[] in ISO_8859_1
-instruct encode_iso_array(eSIRegP src, eDIRegP dst, eDXRegI len,
- regD tmp1, regD tmp2, regD tmp3, regD tmp4,
- eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
- predicate(!((EncodeISOArrayNode*)n)->is_ascii());
- match(Set result (EncodeISOArray src (Binary dst len)));
- effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
-
- format %{ "Encode iso array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
- ins_encode %{
- __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
- $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, false);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// encode char[] to byte[] in ASCII
-instruct encode_ascii_array(eSIRegP src, eDIRegP dst, eDXRegI len,
- regD tmp1, regD tmp2, regD tmp3, regD tmp4,
- eCXRegI tmp5, eAXRegI result, eFlagsReg cr) %{
- predicate(((EncodeISOArrayNode*)n)->is_ascii());
- match(Set result (EncodeISOArray src (Binary dst len)));
- effect(TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, USE_KILL src, USE_KILL dst, USE_KILL len, KILL tmp5, KILL cr);
-
- format %{ "Encode ascii array $src,$dst,$len -> $result // KILL ECX, EDX, $tmp1, $tmp2, $tmp3, $tmp4, ESI, EDI " %}
- ins_encode %{
- __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
- $tmp1$$XMMRegister, $tmp2$$XMMRegister, $tmp3$$XMMRegister,
- $tmp4$$XMMRegister, $tmp5$$Register, $result$$Register, true);
- %}
- ins_pipe( pipe_slow );
-%}
-
-//----------Control Flow Instructions------------------------------------------
-// Signed compare Instructions
-instruct compI_eReg(eFlagsReg cr, rRegI op1, rRegI op2) %{
- match(Set cr (CmpI op1 op2));
- effect( DEF cr, USE op1, USE op2 );
- format %{ "CMP $op1,$op2" %}
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( OpcP, RegReg( op1, op2) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-instruct compI_eReg_imm(eFlagsReg cr, rRegI op1, immI op2) %{
- match(Set cr (CmpI op1 op2));
- effect( DEF cr, USE op1 );
- format %{ "CMP $op1,$op2" %}
- opcode(0x81,0x07); /* Opcode 81 /7 */
- // ins_encode( RegImm( op1, op2) ); /* Was CmpImm */
- ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-// Cisc-spilled version of cmpI_eReg
-instruct compI_eReg_mem(eFlagsReg cr, rRegI op1, memory op2) %{
- match(Set cr (CmpI op1 (LoadI op2)));
-
- format %{ "CMP $op1,$op2" %}
- ins_cost(500);
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( SetInstMark, OpcP, RegMem( op1, op2), ClearInstMark );
- ins_pipe( ialu_cr_reg_mem );
-%}
-
-instruct testI_reg( eFlagsReg cr, rRegI src, immI_0 zero ) %{
- match(Set cr (CmpI src zero));
- effect( DEF cr, USE src );
-
- format %{ "TEST $src,$src" %}
- opcode(0x85);
- ins_encode( OpcP, RegReg( src, src ) );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-instruct testI_reg_imm( eFlagsReg cr, rRegI src, immI con, immI_0 zero ) %{
- match(Set cr (CmpI (AndI src con) zero));
-
- format %{ "TEST $src,$con" %}
- opcode(0xF7,0x00);
- ins_encode( OpcP, RegOpc(src), Con32(con) );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-instruct testI_reg_mem( eFlagsReg cr, rRegI src, memory mem, immI_0 zero ) %{
- match(Set cr (CmpI (AndI src mem) zero));
-
- format %{ "TEST $src,$mem" %}
- opcode(0x85);
- ins_encode( SetInstMark, OpcP, RegMem( src, mem ), ClearInstMark );
- ins_pipe( ialu_cr_reg_mem );
-%}
-
-// Unsigned compare Instructions; really, same as signed except they
-// produce an eFlagsRegU instead of eFlagsReg.
-instruct compU_eReg(eFlagsRegU cr, rRegI op1, rRegI op2) %{
- match(Set cr (CmpU op1 op2));
-
- format %{ "CMPu $op1,$op2" %}
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( OpcP, RegReg( op1, op2) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-instruct compU_eReg_imm(eFlagsRegU cr, rRegI op1, immI op2) %{
- match(Set cr (CmpU op1 op2));
-
- format %{ "CMPu $op1,$op2" %}
- opcode(0x81,0x07); /* Opcode 81 /7 */
- ins_encode( OpcSErm( op1, op2 ), Con8or32( op2 ) );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-// // Cisc-spilled version of cmpU_eReg
-instruct compU_eReg_mem(eFlagsRegU cr, rRegI op1, memory op2) %{
- match(Set cr (CmpU op1 (LoadI op2)));
-
- format %{ "CMPu $op1,$op2" %}
- ins_cost(500);
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( SetInstMark, OpcP, RegMem( op1, op2), ClearInstMark );
- ins_pipe( ialu_cr_reg_mem );
-%}
-
-// // Cisc-spilled version of cmpU_eReg
-//instruct compU_mem_eReg(eFlagsRegU cr, memory op1, rRegI op2) %{
-// match(Set cr (CmpU (LoadI op1) op2));
-//
-// format %{ "CMPu $op1,$op2" %}
-// ins_cost(500);
-// opcode(0x39); /* Opcode 39 /r */
-// ins_encode( OpcP, RegMem( op1, op2) );
-//%}
-
-instruct testU_reg( eFlagsRegU cr, rRegI src, immI_0 zero ) %{
- match(Set cr (CmpU src zero));
-
- format %{ "TESTu $src,$src" %}
- opcode(0x85);
- ins_encode( OpcP, RegReg( src, src ) );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-// Unsigned pointer compare Instructions
-instruct compP_eReg(eFlagsRegU cr, eRegP op1, eRegP op2) %{
- match(Set cr (CmpP op1 op2));
-
- format %{ "CMPu $op1,$op2" %}
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( OpcP, RegReg( op1, op2) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-instruct compP_eReg_imm(eFlagsRegU cr, eRegP op1, immP op2) %{
- match(Set cr (CmpP op1 op2));
-
- format %{ "CMPu $op1,$op2" %}
- opcode(0x81,0x07); /* Opcode 81 /7 */
- ins_encode( SetInstMark, OpcSErm( op1, op2 ), Con8or32( op2 ), ClearInstMark );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-// // Cisc-spilled version of cmpP_eReg
-instruct compP_eReg_mem(eFlagsRegU cr, eRegP op1, memory op2) %{
- match(Set cr (CmpP op1 (LoadP op2)));
-
- format %{ "CMPu $op1,$op2" %}
- ins_cost(500);
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( SetInstMark, OpcP, RegMem( op1, op2), ClearInstMark );
- ins_pipe( ialu_cr_reg_mem );
-%}
-
-// // Cisc-spilled version of cmpP_eReg
-//instruct compP_mem_eReg(eFlagsRegU cr, memory op1, eRegP op2) %{
-// match(Set cr (CmpP (LoadP op1) op2));
-//
-// format %{ "CMPu $op1,$op2" %}
-// ins_cost(500);
-// opcode(0x39); /* Opcode 39 /r */
-// ins_encode( OpcP, RegMem( op1, op2) );
-//%}
-
-// Compare raw pointer (used in out-of-heap check).
-// Only works because non-oop pointers must be raw pointers
-// and raw pointers have no anti-dependencies.
-instruct compP_mem_eReg( eFlagsRegU cr, eRegP op1, memory op2 ) %{
- predicate( n->in(2)->in(2)->bottom_type()->reloc() == relocInfo::none );
- match(Set cr (CmpP op1 (LoadP op2)));
-
- format %{ "CMPu $op1,$op2" %}
- opcode(0x3B); /* Opcode 3B /r */
- ins_encode( SetInstMark, OpcP, RegMem( op1, op2), ClearInstMark );
- ins_pipe( ialu_cr_reg_mem );
-%}
-
-//
-// This will generate a signed flags result. This should be ok
-// since any compare to a zero should be eq/neq.
-instruct testP_reg( eFlagsReg cr, eRegP src, immP0 zero ) %{
- match(Set cr (CmpP src zero));
-
- format %{ "TEST $src,$src" %}
- opcode(0x85);
- ins_encode( OpcP, RegReg( src, src ) );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-// Cisc-spilled version of testP_reg
-// This will generate a signed flags result. This should be ok
-// since any compare to a zero should be eq/neq.
-instruct testP_Reg_mem( eFlagsReg cr, memory op, immI_0 zero ) %{
- match(Set cr (CmpP (LoadP op) zero));
-
- format %{ "TEST $op,0xFFFFFFFF" %}
- ins_cost(500);
- opcode(0xF7); /* Opcode F7 /0 */
- ins_encode( SetInstMark, OpcP, RMopc_Mem(0x00,op), Con_d32(0xFFFFFFFF), ClearInstMark );
- ins_pipe( ialu_cr_reg_imm );
-%}
-
-// Yanked all unsigned pointer compare operations.
-// Pointer compares are done with CmpP which is already unsigned.
-
-//----------Max and Min--------------------------------------------------------
-// Min Instructions
-////
-// *** Min and Max using the conditional move are slower than the
-// *** branch version on a Pentium III.
-// // Conditional move for min
-//instruct cmovI_reg_lt( rRegI op2, rRegI op1, eFlagsReg cr ) %{
-// effect( USE_DEF op2, USE op1, USE cr );
-// format %{ "CMOVlt $op2,$op1\t! min" %}
-// opcode(0x4C,0x0F);
-// ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
-// ins_pipe( pipe_cmov_reg );
-//%}
-//
-//// Min Register with Register (P6 version)
-//instruct minI_eReg_p6( rRegI op1, rRegI op2 ) %{
-// predicate(VM_Version::supports_cmov() );
-// match(Set op2 (MinI op1 op2));
-// ins_cost(200);
-// expand %{
-// eFlagsReg cr;
-// compI_eReg(cr,op1,op2);
-// cmovI_reg_lt(op2,op1,cr);
-// %}
-//%}
-
-// Min Register with Register (generic version)
-instruct minI_eReg(rRegI dst, rRegI src, eFlagsReg flags) %{
- match(Set dst (MinI dst src));
- effect(KILL flags);
- ins_cost(300);
-
- format %{ "MIN $dst,$src" %}
- opcode(0xCC);
- ins_encode( min_enc(dst,src) );
- ins_pipe( pipe_slow );
-%}
-
-// Max Register with Register
-// *** Min and Max using the conditional move are slower than the
-// *** branch version on a Pentium III.
-// // Conditional move for max
-//instruct cmovI_reg_gt( rRegI op2, rRegI op1, eFlagsReg cr ) %{
-// effect( USE_DEF op2, USE op1, USE cr );
-// format %{ "CMOVgt $op2,$op1\t! max" %}
-// opcode(0x4F,0x0F);
-// ins_encode( OpcS, OpcP, RegReg( op2, op1 ) );
-// ins_pipe( pipe_cmov_reg );
-//%}
-//
-// // Max Register with Register (P6 version)
-//instruct maxI_eReg_p6( rRegI op1, rRegI op2 ) %{
-// predicate(VM_Version::supports_cmov() );
-// match(Set op2 (MaxI op1 op2));
-// ins_cost(200);
-// expand %{
-// eFlagsReg cr;
-// compI_eReg(cr,op1,op2);
-// cmovI_reg_gt(op2,op1,cr);
-// %}
-//%}
-
-// Max Register with Register (generic version)
-instruct maxI_eReg(rRegI dst, rRegI src, eFlagsReg flags) %{
- match(Set dst (MaxI dst src));
- effect(KILL flags);
- ins_cost(300);
-
- format %{ "MAX $dst,$src" %}
- opcode(0xCC);
- ins_encode( max_enc(dst,src) );
- ins_pipe( pipe_slow );
-%}
-
-// ============================================================================
-// Counted Loop limit node which represents exact final iterator value.
-// Note: the resulting value should fit into integer range since
-// counted loops have limit check on overflow.
-instruct loopLimit_eReg(eAXRegI limit, nadxRegI init, immI stride, eDXRegI limit_hi, nadxRegI tmp, eFlagsReg flags) %{
- match(Set limit (LoopLimit (Binary init limit) stride));
- effect(TEMP limit_hi, TEMP tmp, KILL flags);
- ins_cost(300);
-
- format %{ "loopLimit $init,$limit,$stride # $limit = $init + $stride *( $limit - $init + $stride -1)/ $stride, kills $limit_hi" %}
- ins_encode %{
- int strd = (int)$stride$$constant;
- assert(strd != 1 && strd != -1, "sanity");
- int m1 = (strd > 0) ? 1 : -1;
- // Convert limit to long (EAX:EDX)
- __ cdql();
- // Convert init to long (init:tmp)
- __ movl($tmp$$Register, $init$$Register);
- __ sarl($tmp$$Register, 31);
- // $limit - $init
- __ subl($limit$$Register, $init$$Register);
- __ sbbl($limit_hi$$Register, $tmp$$Register);
- // + ($stride - 1)
- if (strd > 0) {
- __ addl($limit$$Register, (strd - 1));
- __ adcl($limit_hi$$Register, 0);
- __ movl($tmp$$Register, strd);
- } else {
- __ addl($limit$$Register, (strd + 1));
- __ adcl($limit_hi$$Register, -1);
- __ lneg($limit_hi$$Register, $limit$$Register);
- __ movl($tmp$$Register, -strd);
- }
- // signed division: (EAX:EDX) / pos_stride
- __ idivl($tmp$$Register);
- if (strd < 0) {
- // restore sign
- __ negl($tmp$$Register);
- }
- // (EAX) * stride
- __ mull($tmp$$Register);
- // + init (ignore upper bits)
- __ addl($limit$$Register, $init$$Register);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// ============================================================================
-// Branch Instructions
-// Jump Table
-instruct jumpXtnd(rRegI switch_val) %{
- match(Jump switch_val);
- ins_cost(350);
- format %{ "JMP [$constantaddress](,$switch_val,1)\n\t" %}
- ins_encode %{
- // Jump to Address(table_base + switch_reg)
- Address index(noreg, $switch_val$$Register, Address::times_1);
- __ jump(ArrayAddress($constantaddress, index), noreg);
- %}
- ins_pipe(pipe_jmp);
-%}
-
-// Jump Direct - Label defines a relative address from JMP+1
-instruct jmpDir(label labl) %{
- match(Goto);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "JMP $labl" %}
- size(5);
- ins_encode %{
- Label* L = $labl$$label;
- __ jmp(*L, false); // Always long jump
- %}
- ins_pipe( pipe_jmp );
-%}
-
-// Jump Direct Conditional - Label defines a relative address from Jcc+1
-instruct jmpCon(cmpOp cop, eFlagsReg cr, label labl) %{
- match(If cop cr);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop $labl" %}
- size(6);
- ins_encode %{
- Label* L = $labl$$label;
- __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
- %}
- ins_pipe( pipe_jcc );
-%}
-
-// Jump Direct Conditional - Label defines a relative address from Jcc+1
-instruct jmpLoopEnd(cmpOp cop, eFlagsReg cr, label labl) %{
- match(CountedLoopEnd cop cr);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop $labl\t# Loop end" %}
- size(6);
- ins_encode %{
- Label* L = $labl$$label;
- __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
- %}
- ins_pipe( pipe_jcc );
-%}
-
-// Jump Direct Conditional - using unsigned comparison
-instruct jmpConU(cmpOpU cop, eFlagsRegU cmp, label labl) %{
- match(If cop cmp);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop,u $labl" %}
- size(6);
- ins_encode %{
- Label* L = $labl$$label;
- __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
- %}
- ins_pipe(pipe_jcc);
-%}
-
-instruct jmpConUCF(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
- match(If cop cmp);
- effect(USE labl);
-
- ins_cost(200);
- format %{ "J$cop,u $labl" %}
- size(6);
- ins_encode %{
- Label* L = $labl$$label;
- __ jcc((Assembler::Condition)($cop$$cmpcode), *L, false); // Always long jump
- %}
- ins_pipe(pipe_jcc);
-%}
-
-instruct jmpConUCF2(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
- match(If cop cmp);
- effect(USE labl);
-
- ins_cost(200);
- format %{ $$template
- if ($cop$$cmpcode == Assembler::notEqual) {
- $$emit$$"JP,u $labl\n\t"
- $$emit$$"J$cop,u $labl"
- } else {
- $$emit$$"JP,u done\n\t"
- $$emit$$"J$cop,u $labl\n\t"
- $$emit$$"done:"
- }
- %}
- ins_encode %{
- Label* l = $labl$$label;
- if ($cop$$cmpcode == Assembler::notEqual) {
- __ jcc(Assembler::parity, *l, false);
- __ jcc(Assembler::notEqual, *l, false);
- } else if ($cop$$cmpcode == Assembler::equal) {
- Label done;
- __ jccb(Assembler::parity, done);
- __ jcc(Assembler::equal, *l, false);
- __ bind(done);
- } else {
- ShouldNotReachHere();
- }
- %}
- ins_pipe(pipe_jcc);
-%}
-
-// ============================================================================
-// The 2nd slow-half of a subtype check. Scan the subklass's 2ndary superklass
-// array for an instance of the superklass. Set a hidden internal cache on a
-// hit (cache is checked with exposed code in gen_subtype_check()). Return
-// NZ for a miss or zero for a hit. The encoding ALSO sets flags.
-instruct partialSubtypeCheck( eDIRegP result, eSIRegP sub, eAXRegP super, eCXRegI rcx, eFlagsReg cr ) %{
- match(Set result (PartialSubtypeCheck sub super));
- effect( KILL rcx, KILL cr );
-
- ins_cost(1100); // slightly larger than the next version
- format %{ "MOV EDI,[$sub+Klass::secondary_supers]\n\t"
- "MOV ECX,[EDI+ArrayKlass::length]\t# length to scan\n\t"
- "ADD EDI,ArrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
- "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
- "JNE,s miss\t\t# Missed: EDI not-zero\n\t"
- "MOV [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache\n\t"
- "XOR $result,$result\t\t Hit: EDI zero\n\t"
- "miss:\t" %}
-
- opcode(0x1); // Force a XOR of EDI
- ins_encode( enc_PartialSubtypeCheck() );
- ins_pipe( pipe_slow );
-%}
-
-instruct partialSubtypeCheck_vs_Zero( eFlagsReg cr, eSIRegP sub, eAXRegP super, eCXRegI rcx, eDIRegP result, immP0 zero ) %{
- match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
- effect( KILL rcx, KILL result );
-
- ins_cost(1000);
- format %{ "MOV EDI,[$sub+Klass::secondary_supers]\n\t"
- "MOV ECX,[EDI+ArrayKlass::length]\t# length to scan\n\t"
- "ADD EDI,ArrayKlass::base_offset\t# Skip to start of data; set NZ in case count is zero\n\t"
- "REPNE SCASD\t# Scan *EDI++ for a match with EAX while CX-- != 0\n\t"
- "JNE,s miss\t\t# Missed: flags NZ\n\t"
- "MOV [$sub+Klass::secondary_super_cache],$super\t# Hit: update cache, flags Z\n\t"
- "miss:\t" %}
-
- opcode(0x0); // No need to XOR EDI
- ins_encode( enc_PartialSubtypeCheck() );
- ins_pipe( pipe_slow );
-%}
-
-// ============================================================================
-// Branch Instructions -- short offset versions
-//
-// These instructions are used to replace jumps of a long offset (the default
-// match) with jumps of a shorter offset. These instructions are all tagged
-// with the ins_short_branch attribute, which causes the ADLC to suppress the
-// match rules in general matching. Instead, the ADLC generates a conversion
-// method in the MachNode which can be used to do in-place replacement of the
-// long variant with the shorter variant. The compiler will determine if a
-// branch can be taken by the is_short_branch_offset() predicate in the machine
-// specific code section of the file.
-
-// Jump Direct - Label defines a relative address from JMP+1
-instruct jmpDir_short(label labl) %{
- match(Goto);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "JMP,s $labl" %}
- size(2);
- ins_encode %{
- Label* L = $labl$$label;
- __ jmpb(*L);
- %}
- ins_pipe( pipe_jmp );
- ins_short_branch(1);
-%}
-
-// Jump Direct Conditional - Label defines a relative address from Jcc+1
-instruct jmpCon_short(cmpOp cop, eFlagsReg cr, label labl) %{
- match(If cop cr);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop,s $labl" %}
- size(2);
- ins_encode %{
- Label* L = $labl$$label;
- __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
- %}
- ins_pipe( pipe_jcc );
- ins_short_branch(1);
-%}
-
-// Jump Direct Conditional - Label defines a relative address from Jcc+1
-instruct jmpLoopEnd_short(cmpOp cop, eFlagsReg cr, label labl) %{
- match(CountedLoopEnd cop cr);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop,s $labl\t# Loop end" %}
- size(2);
- ins_encode %{
- Label* L = $labl$$label;
- __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
- %}
- ins_pipe( pipe_jcc );
- ins_short_branch(1);
-%}
-
-// Jump Direct Conditional - using unsigned comparison
-instruct jmpConU_short(cmpOpU cop, eFlagsRegU cmp, label labl) %{
- match(If cop cmp);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop,us $labl" %}
- size(2);
- ins_encode %{
- Label* L = $labl$$label;
- __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
- %}
- ins_pipe( pipe_jcc );
- ins_short_branch(1);
-%}
-
-instruct jmpConUCF_short(cmpOpUCF cop, eFlagsRegUCF cmp, label labl) %{
- match(If cop cmp);
- effect(USE labl);
-
- ins_cost(300);
- format %{ "J$cop,us $labl" %}
- size(2);
- ins_encode %{
- Label* L = $labl$$label;
- __ jccb((Assembler::Condition)($cop$$cmpcode), *L);
- %}
- ins_pipe( pipe_jcc );
- ins_short_branch(1);
-%}
-
-instruct jmpConUCF2_short(cmpOpUCF2 cop, eFlagsRegUCF cmp, label labl) %{
- match(If cop cmp);
- effect(USE labl);
-
- ins_cost(300);
- format %{ $$template
- if ($cop$$cmpcode == Assembler::notEqual) {
- $$emit$$"JP,u,s $labl\n\t"
- $$emit$$"J$cop,u,s $labl"
- } else {
- $$emit$$"JP,u,s done\n\t"
- $$emit$$"J$cop,u,s $labl\n\t"
- $$emit$$"done:"
- }
- %}
- size(4);
- ins_encode %{
- Label* l = $labl$$label;
- if ($cop$$cmpcode == Assembler::notEqual) {
- __ jccb(Assembler::parity, *l);
- __ jccb(Assembler::notEqual, *l);
- } else if ($cop$$cmpcode == Assembler::equal) {
- Label done;
- __ jccb(Assembler::parity, done);
- __ jccb(Assembler::equal, *l);
- __ bind(done);
- } else {
- ShouldNotReachHere();
- }
- %}
- ins_pipe(pipe_jcc);
- ins_short_branch(1);
-%}
-
-// ============================================================================
-// Long Compare
-//
-// Currently we hold longs in 2 registers. Comparing such values efficiently
-// is tricky. The flavor of compare used depends on whether we are testing
-// for LT, LE, or EQ. For a simple LT test we can check just the sign bit.
-// The GE test is the negated LT test. The LE test can be had by commuting
-// the operands (yielding a GE test) and then negating; negate again for the
-// GT test. The EQ test is done by ORcc'ing the high and low halves, and the
-// NE test is negated from that.
-
-// Due to a shortcoming in the ADLC, it mixes up expressions like:
-// (foo (CmpI (CmpL X Y) 0)) and (bar (CmpI (CmpL X 0L) 0)). Note the
-// difference between 'Y' and '0L'. The tree-matches for the CmpI sections
-// are collapsed internally in the ADLC's dfa-gen code. The match for
-// (CmpI (CmpL X Y) 0) is silently replaced with (CmpI (CmpL X 0L) 0) and the
-// foo match ends up with the wrong leaf. One fix is to not match both
-// reg-reg and reg-zero forms of long-compare. This is unfortunate because
-// both forms beat the trinary form of long-compare and both are very useful
-// on Intel which has so few registers.
-
-// Manifest a CmpL result in an integer register. Very painful.
-// This is the test to avoid.
-instruct cmpL3_reg_reg(eSIRegI dst, eRegL src1, eRegL src2, eFlagsReg flags ) %{
- match(Set dst (CmpL3 src1 src2));
- effect( KILL flags );
- ins_cost(1000);
- format %{ "XOR $dst,$dst\n\t"
- "CMP $src1.hi,$src2.hi\n\t"
- "JLT,s m_one\n\t"
- "JGT,s p_one\n\t"
- "CMP $src1.lo,$src2.lo\n\t"
- "JB,s m_one\n\t"
- "JEQ,s done\n"
- "p_one:\tINC $dst\n\t"
- "JMP,s done\n"
- "m_one:\tDEC $dst\n"
- "done:" %}
- ins_encode %{
- Label p_one, m_one, done;
- __ xorptr($dst$$Register, $dst$$Register);
- __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register));
- __ jccb(Assembler::less, m_one);
- __ jccb(Assembler::greater, p_one);
- __ cmpl($src1$$Register, $src2$$Register);
- __ jccb(Assembler::below, m_one);
- __ jccb(Assembler::equal, done);
- __ bind(p_one);
- __ incrementl($dst$$Register);
- __ jmpb(done);
- __ bind(m_one);
- __ decrementl($dst$$Register);
- __ bind(done);
- %}
- ins_pipe( pipe_slow );
-%}
-
-//======
-// Manifest a CmpL result in the normal flags. Only good for LT or GE
-// compares. Can be used for LE or GT compares by reversing arguments.
-// NOT GOOD FOR EQ/NE tests.
-instruct cmpL_zero_flags_LTGE( flagsReg_long_LTGE flags, eRegL src, immL0 zero ) %{
- match( Set flags (CmpL src zero ));
- ins_cost(100);
- format %{ "TEST $src.hi,$src.hi" %}
- opcode(0x85);
- ins_encode( OpcP, RegReg_Hi2( src, src ) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-// Manifest a CmpL result in the normal flags. Only good for LT or GE
-// compares. Can be used for LE or GT compares by reversing arguments.
-// NOT GOOD FOR EQ/NE tests.
-instruct cmpL_reg_flags_LTGE( flagsReg_long_LTGE flags, eRegL src1, eRegL src2, rRegI tmp ) %{
- match( Set flags (CmpL src1 src2 ));
- effect( TEMP tmp );
- ins_cost(300);
- format %{ "CMP $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
- "MOV $tmp,$src1.hi\n\t"
- "SBB $tmp,$src2.hi\t! Compute flags for long compare" %}
- ins_encode( long_cmp_flags2( src1, src2, tmp ) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-// Long compares reg < zero/req OR reg >= zero/req.
-// Just a wrapper for a normal branch, plus the predicate test.
-instruct cmpL_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, label labl) %{
- match(If cmp flags);
- effect(USE labl);
- predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge );
- expand %{
- jmpCon(cmp,flags,labl); // JLT or JGE...
- %}
-%}
-
-//======
-// Manifest a CmpUL result in the normal flags. Only good for LT or GE
-// compares. Can be used for LE or GT compares by reversing arguments.
-// NOT GOOD FOR EQ/NE tests.
-instruct cmpUL_zero_flags_LTGE(flagsReg_ulong_LTGE flags, eRegL src, immL0 zero) %{
- match(Set flags (CmpUL src zero));
- ins_cost(100);
- format %{ "TEST $src.hi,$src.hi" %}
- opcode(0x85);
- ins_encode(OpcP, RegReg_Hi2(src, src));
- ins_pipe(ialu_cr_reg_reg);
-%}
-
-// Manifest a CmpUL result in the normal flags. Only good for LT or GE
-// compares. Can be used for LE or GT compares by reversing arguments.
-// NOT GOOD FOR EQ/NE tests.
-instruct cmpUL_reg_flags_LTGE(flagsReg_ulong_LTGE flags, eRegL src1, eRegL src2, rRegI tmp) %{
- match(Set flags (CmpUL src1 src2));
- effect(TEMP tmp);
- ins_cost(300);
- format %{ "CMP $src1.lo,$src2.lo\t! Unsigned long compare; set flags for low bits\n\t"
- "MOV $tmp,$src1.hi\n\t"
- "SBB $tmp,$src2.hi\t! Compute flags for unsigned long compare" %}
- ins_encode(long_cmp_flags2(src1, src2, tmp));
- ins_pipe(ialu_cr_reg_reg);
-%}
-
-// Unsigned long compares reg < zero/req OR reg >= zero/req.
-// Just a wrapper for a normal branch, plus the predicate test.
-instruct cmpUL_LTGE(cmpOpU cmp, flagsReg_ulong_LTGE flags, label labl) %{
- match(If cmp flags);
- effect(USE labl);
- predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge);
- expand %{
- jmpCon(cmp, flags, labl); // JLT or JGE...
- %}
-%}
-
-// Compare 2 longs and CMOVE longs.
-instruct cmovLL_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, eRegL src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- ins_cost(400);
- format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
- "CMOV$cmp $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovLL_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegL dst, load_long_memory src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- ins_cost(500);
- format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
- "CMOV$cmp $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src), ClearInstMark );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovLL_reg_LTGE_U(cmpOpU cmp, flagsReg_ulong_LTGE flags, eRegL dst, eRegL src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- ins_cost(400);
- expand %{
- cmovLL_reg_LTGE(cmp, flags, dst, src);
- %}
-%}
-
-instruct cmovLL_mem_LTGE_U(cmpOpU cmp, flagsReg_ulong_LTGE flags, eRegL dst, load_long_memory src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- ins_cost(500);
- expand %{
- cmovLL_mem_LTGE(cmp, flags, dst, src);
- %}
-%}
-
-// Compare 2 longs and CMOVE ints.
-instruct cmovII_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, rRegI dst, rRegI src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovII_mem_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
- ins_cost(250);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cmp), RegMem( dst, src ), ClearInstMark );
- ins_pipe( pipe_cmov_mem );
-%}
-
-instruct cmovII_reg_LTGE_U(cmpOpU cmp, flagsReg_ulong_LTGE flags, rRegI dst, rRegI src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovII_reg_LTGE(cmp, flags, dst, src);
- %}
-%}
-
-instruct cmovII_mem_LTGE_U(cmpOpU cmp, flagsReg_ulong_LTGE flags, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
- ins_cost(250);
- expand %{
- cmovII_mem_LTGE(cmp, flags, dst, src);
- %}
-%}
-
-// Compare 2 longs and CMOVE ptrs.
-instruct cmovPP_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, eRegP dst, eRegP src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-// Compare 2 unsigned longs and CMOVE ptrs.
-instruct cmovPP_reg_LTGE_U(cmpOpU cmp, flagsReg_ulong_LTGE flags, eRegP dst, eRegP src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovPP_reg_LTGE(cmp,flags,dst,src);
- %}
-%}
-
-// Compare 2 longs and CMOVE doubles
-instruct cmovDDPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovDPR_regS(cmp,flags,dst,src);
- %}
-%}
-
-// Compare 2 longs and CMOVE doubles
-instruct cmovDD_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regD dst, regD src) %{
- predicate( UseSSE>=2 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovD_regS(cmp,flags,dst,src);
- %}
-%}
-
-instruct cmovFFPR_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regFPR dst, regFPR src) %{
- predicate( UseSSE==0 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovFPR_regS(cmp,flags,dst,src);
- %}
-%}
-
-instruct cmovFF_reg_LTGE(cmpOp cmp, flagsReg_long_LTGE flags, regF dst, regF src) %{
- predicate( UseSSE>=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::lt || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ge ));
- match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovF_regS(cmp,flags,dst,src);
- %}
-%}
-
-//======
-// Manifest a CmpL result in the normal flags. Only good for EQ/NE compares.
-instruct cmpL_zero_flags_EQNE( flagsReg_long_EQNE flags, eRegL src, immL0 zero, rRegI tmp ) %{
- match( Set flags (CmpL src zero ));
- effect(TEMP tmp);
- ins_cost(200);
- format %{ "MOV $tmp,$src.lo\n\t"
- "OR $tmp,$src.hi\t! Long is EQ/NE 0?" %}
- ins_encode( long_cmp_flags0( src, tmp ) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Manifest a CmpL result in the normal flags. Only good for EQ/NE compares.
-instruct cmpL_reg_flags_EQNE( flagsReg_long_EQNE flags, eRegL src1, eRegL src2 ) %{
- match( Set flags (CmpL src1 src2 ));
- ins_cost(200+300);
- format %{ "CMP $src1.lo,$src2.lo\t! Long compare; set flags for low bits\n\t"
- "JNE,s skip\n\t"
- "CMP $src1.hi,$src2.hi\n\t"
- "skip:\t" %}
- ins_encode( long_cmp_flags1( src1, src2 ) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-// Long compare reg == zero/reg OR reg != zero/reg
-// Just a wrapper for a normal branch, plus the predicate test.
-instruct cmpL_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, label labl) %{
- match(If cmp flags);
- effect(USE labl);
- predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne );
- expand %{
- jmpCon(cmp,flags,labl); // JEQ or JNE...
- %}
-%}
-
-//======
-// Manifest a CmpUL result in the normal flags. Only good for EQ/NE compares.
-instruct cmpUL_zero_flags_EQNE(flagsReg_ulong_EQNE flags, eRegL src, immL0 zero, rRegI tmp) %{
- match(Set flags (CmpUL src zero));
- effect(TEMP tmp);
- ins_cost(200);
- format %{ "MOV $tmp,$src.lo\n\t"
- "OR $tmp,$src.hi\t! Unsigned long is EQ/NE 0?" %}
- ins_encode(long_cmp_flags0(src, tmp));
- ins_pipe(ialu_reg_reg_long);
-%}
-
-// Manifest a CmpUL result in the normal flags. Only good for EQ/NE compares.
-instruct cmpUL_reg_flags_EQNE(flagsReg_ulong_EQNE flags, eRegL src1, eRegL src2) %{
- match(Set flags (CmpUL src1 src2));
- ins_cost(200+300);
- format %{ "CMP $src1.lo,$src2.lo\t! Unsigned long compare; set flags for low bits\n\t"
- "JNE,s skip\n\t"
- "CMP $src1.hi,$src2.hi\n\t"
- "skip:\t" %}
- ins_encode(long_cmp_flags1(src1, src2));
- ins_pipe(ialu_cr_reg_reg);
-%}
-
-// Unsigned long compare reg == zero/reg OR reg != zero/reg
-// Just a wrapper for a normal branch, plus the predicate test.
-instruct cmpUL_EQNE(cmpOpU cmp, flagsReg_ulong_EQNE flags, label labl) %{
- match(If cmp flags);
- effect(USE labl);
- predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne);
- expand %{
- jmpCon(cmp, flags, labl); // JEQ or JNE...
- %}
-%}
-
-// Compare 2 longs and CMOVE longs.
-instruct cmovLL_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, eRegL src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- ins_cost(400);
- format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
- "CMOV$cmp $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovLL_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegL dst, load_long_memory src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- ins_cost(500);
- format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
- "CMOV$cmp $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src), ClearInstMark );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-// Compare 2 longs and CMOVE ints.
-instruct cmovII_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, rRegI dst, rRegI src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovII_mem_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
- ins_cost(250);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cmp), RegMem( dst, src ), ClearInstMark );
- ins_pipe( pipe_cmov_mem );
-%}
-
-instruct cmovII_reg_EQNE_U(cmpOpU cmp, flagsReg_ulong_EQNE flags, rRegI dst, rRegI src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovII_reg_EQNE(cmp, flags, dst, src);
- %}
-%}
-
-instruct cmovII_mem_EQNE_U(cmpOpU cmp, flagsReg_ulong_EQNE flags, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
- ins_cost(250);
- expand %{
- cmovII_mem_EQNE(cmp, flags, dst, src);
- %}
-%}
-
-// Compare 2 longs and CMOVE ptrs.
-instruct cmovPP_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, eRegP dst, eRegP src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-// Compare 2 unsigned longs and CMOVE ptrs.
-instruct cmovPP_reg_EQNE_U(cmpOpU cmp, flagsReg_ulong_EQNE flags, eRegP dst, eRegP src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovPP_reg_EQNE(cmp,flags,dst,src);
- %}
-%}
-
-// Compare 2 longs and CMOVE doubles
-instruct cmovDDPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovDPR_regS(cmp,flags,dst,src);
- %}
-%}
-
-// Compare 2 longs and CMOVE doubles
-instruct cmovDD_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regD dst, regD src) %{
- predicate( UseSSE>=2 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovD_regS(cmp,flags,dst,src);
- %}
-%}
-
-instruct cmovFFPR_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regFPR dst, regFPR src) %{
- predicate( UseSSE==0 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovFPR_regS(cmp,flags,dst,src);
- %}
-%}
-
-instruct cmovFF_reg_EQNE(cmpOp cmp, flagsReg_long_EQNE flags, regF dst, regF src) %{
- predicate( UseSSE>=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::eq || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::ne ));
- match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovF_regS(cmp,flags,dst,src);
- %}
-%}
-
-//======
-// Manifest a CmpL result in the normal flags. Only good for LE or GT compares.
-// Same as cmpL_reg_flags_LEGT except must negate src
-instruct cmpL_zero_flags_LEGT( flagsReg_long_LEGT flags, eRegL src, immL0 zero, rRegI tmp ) %{
- match( Set flags (CmpL src zero ));
- effect( TEMP tmp );
- ins_cost(300);
- format %{ "XOR $tmp,$tmp\t# Long compare for -$src < 0, use commuted test\n\t"
- "CMP $tmp,$src.lo\n\t"
- "SBB $tmp,$src.hi\n\t" %}
- ins_encode( long_cmp_flags3(src, tmp) );
- ins_pipe( ialu_reg_reg_long );
-%}
-
-// Manifest a CmpL result in the normal flags. Only good for LE or GT compares.
-// Same as cmpL_reg_flags_LTGE except operands swapped. Swapping operands
-// requires a commuted test to get the same result.
-instruct cmpL_reg_flags_LEGT( flagsReg_long_LEGT flags, eRegL src1, eRegL src2, rRegI tmp ) %{
- match( Set flags (CmpL src1 src2 ));
- effect( TEMP tmp );
- ins_cost(300);
- format %{ "CMP $src2.lo,$src1.lo\t! Long compare, swapped operands, use with commuted test\n\t"
- "MOV $tmp,$src2.hi\n\t"
- "SBB $tmp,$src1.hi\t! Compute flags for long compare" %}
- ins_encode( long_cmp_flags2( src2, src1, tmp ) );
- ins_pipe( ialu_cr_reg_reg );
-%}
-
-// Long compares reg < zero/req OR reg >= zero/req.
-// Just a wrapper for a normal branch, plus the predicate test
-instruct cmpL_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, label labl) %{
- match(If cmp flags);
- effect(USE labl);
- predicate( _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le );
- ins_cost(300);
- expand %{
- jmpCon(cmp,flags,labl); // JGT or JLE...
- %}
-%}
-
-//======
-// Manifest a CmpUL result in the normal flags. Only good for LE or GT compares.
-// Same as cmpUL_reg_flags_LEGT except must negate src
-instruct cmpUL_zero_flags_LEGT(flagsReg_ulong_LEGT flags, eRegL src, immL0 zero, rRegI tmp) %{
- match(Set flags (CmpUL src zero));
- effect(TEMP tmp);
- ins_cost(300);
- format %{ "XOR $tmp,$tmp\t# Unsigned long compare for -$src < 0, use commuted test\n\t"
- "CMP $tmp,$src.lo\n\t"
- "SBB $tmp,$src.hi\n\t" %}
- ins_encode(long_cmp_flags3(src, tmp));
- ins_pipe(ialu_reg_reg_long);
-%}
-
-// Manifest a CmpUL result in the normal flags. Only good for LE or GT compares.
-// Same as cmpUL_reg_flags_LTGE except operands swapped. Swapping operands
-// requires a commuted test to get the same result.
-instruct cmpUL_reg_flags_LEGT(flagsReg_ulong_LEGT flags, eRegL src1, eRegL src2, rRegI tmp) %{
- match(Set flags (CmpUL src1 src2));
- effect(TEMP tmp);
- ins_cost(300);
- format %{ "CMP $src2.lo,$src1.lo\t! Unsigned long compare, swapped operands, use with commuted test\n\t"
- "MOV $tmp,$src2.hi\n\t"
- "SBB $tmp,$src1.hi\t! Compute flags for unsigned long compare" %}
- ins_encode(long_cmp_flags2( src2, src1, tmp));
- ins_pipe(ialu_cr_reg_reg);
-%}
-
-// Unsigned long compares reg < zero/req OR reg >= zero/req.
-// Just a wrapper for a normal branch, plus the predicate test
-instruct cmpUL_LEGT(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, label labl) %{
- match(If cmp flags);
- effect(USE labl);
- predicate(_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt || _kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le);
- ins_cost(300);
- expand %{
- jmpCon(cmp, flags, labl); // JGT or JLE...
- %}
-%}
-
-// Compare 2 longs and CMOVE longs.
-instruct cmovLL_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, eRegL src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- ins_cost(400);
- format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
- "CMOV$cmp $dst.hi,$src.hi" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg_Lo2( dst, src ), enc_cmov(cmp), RegReg_Hi2( dst, src ) );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovLL_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegL dst, load_long_memory src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- ins_cost(500);
- format %{ "CMOV$cmp $dst.lo,$src.lo\n\t"
- "CMOV$cmp $dst.hi,$src.hi+4" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cmp), RegMem(dst, src), enc_cmov(cmp), RegMem_Hi(dst, src), ClearInstMark );
- ins_pipe( pipe_cmov_reg_long );
-%}
-
-instruct cmovLL_reg_LEGT_U(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, eRegL dst, eRegL src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst src)));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- ins_cost(400);
- expand %{
- cmovLL_reg_LEGT(cmp, flags, dst, src);
- %}
-%}
-
-instruct cmovLL_mem_LEGT_U(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, eRegL dst, load_long_memory src) %{
- match(Set dst (CMoveL (Binary cmp flags) (Binary dst (LoadL src))));
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- ins_cost(500);
- expand %{
- cmovLL_mem_LEGT(cmp, flags, dst, src);
- %}
-%}
-
-// Compare 2 longs and CMOVE ints.
-instruct cmovII_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, rRegI dst, rRegI src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-instruct cmovII_mem_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
- ins_cost(250);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( SetInstMark, enc_cmov(cmp), RegMem( dst, src ), ClearInstMark );
- ins_pipe( pipe_cmov_mem );
-%}
-
-instruct cmovII_reg_LEGT_U(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, rRegI dst, rRegI src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovII_reg_LEGT(cmp, flags, dst, src);
- %}
-%}
-
-instruct cmovII_mem_LEGT_U(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, rRegI dst, memory src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveI (Binary cmp flags) (Binary dst (LoadI src))));
- ins_cost(250);
- expand %{
- cmovII_mem_LEGT(cmp, flags, dst, src);
- %}
-%}
-
-// Compare 2 longs and CMOVE ptrs.
-instruct cmovPP_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, eRegP dst, eRegP src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- format %{ "CMOV$cmp $dst,$src" %}
- opcode(0x0F,0x40);
- ins_encode( enc_cmov(cmp), RegReg( dst, src ) );
- ins_pipe( pipe_cmov_reg );
-%}
-
-// Compare 2 unsigned longs and CMOVE ptrs.
-instruct cmovPP_reg_LEGT_U(cmpOpU_commute cmp, flagsReg_ulong_LEGT flags, eRegP dst, eRegP src) %{
- predicate(VM_Version::supports_cmov() && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveP (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- cmovPP_reg_LEGT(cmp,flags,dst,src);
- %}
-%}
-
-// Compare 2 longs and CMOVE doubles
-instruct cmovDDPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regDPR dst, regDPR src) %{
- predicate( UseSSE<=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovDPR_regS(cmp,flags,dst,src);
- %}
-%}
-
-// Compare 2 longs and CMOVE doubles
-instruct cmovDD_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regD dst, regD src) %{
- predicate( UseSSE>=2 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveD (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovD_regS(cmp,flags,dst,src);
- %}
-%}
-
-instruct cmovFFPR_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regFPR dst, regFPR src) %{
- predicate( UseSSE==0 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovFPR_regS(cmp,flags,dst,src);
- %}
-%}
-
-
-instruct cmovFF_reg_LEGT(cmpOp_commute cmp, flagsReg_long_LEGT flags, regF dst, regF src) %{
- predicate( UseSSE>=1 && ( _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::le || _kids[0]->_kids[0]->_leaf->as_Bool()->_test._test == BoolTest::gt ));
- match(Set dst (CMoveF (Binary cmp flags) (Binary dst src)));
- ins_cost(200);
- expand %{
- fcmovF_regS(cmp,flags,dst,src);
- %}
-%}
-
-
-// ============================================================================
-// Procedure Call/Return Instructions
-// Call Java Static Instruction
-// Note: If this code changes, the corresponding ret_addr_offset() and
-// compute_padding() functions will have to be adjusted.
-instruct CallStaticJavaDirect(method meth) %{
- match(CallStaticJava);
- effect(USE meth);
-
- ins_cost(300);
- format %{ "CALL,static " %}
- opcode(0xE8); /* E8 cd */
- ins_encode( pre_call_resets,
- Java_Static_Call( meth ),
- call_epilog,
- post_call_FPU );
- ins_pipe( pipe_slow );
- ins_alignment(4);
-%}
-
-// Call Java Dynamic Instruction
-// Note: If this code changes, the corresponding ret_addr_offset() and
-// compute_padding() functions will have to be adjusted.
-instruct CallDynamicJavaDirect(method meth) %{
- match(CallDynamicJava);
- effect(USE meth);
-
- ins_cost(300);
- format %{ "MOV EAX,(oop)-1\n\t"
- "CALL,dynamic" %}
- opcode(0xE8); /* E8 cd */
- ins_encode( pre_call_resets,
- Java_Dynamic_Call( meth ),
- call_epilog,
- post_call_FPU );
- ins_pipe( pipe_slow );
- ins_alignment(4);
-%}
-
-// Call Runtime Instruction
-instruct CallRuntimeDirect(method meth) %{
- match(CallRuntime );
- effect(USE meth);
-
- ins_cost(300);
- format %{ "CALL,runtime " %}
- opcode(0xE8); /* E8 cd */
- // Use FFREEs to clear entries in float stack
- ins_encode( pre_call_resets,
- FFree_Float_Stack_All,
- Java_To_Runtime( meth ),
- post_call_FPU );
- ins_pipe( pipe_slow );
-%}
-
-// Call runtime without safepoint
-instruct CallLeafDirect(method meth) %{
- match(CallLeaf);
- effect(USE meth);
-
- ins_cost(300);
- format %{ "CALL_LEAF,runtime " %}
- opcode(0xE8); /* E8 cd */
- ins_encode( pre_call_resets,
- FFree_Float_Stack_All,
- Java_To_Runtime( meth ),
- Verify_FPU_For_Leaf, post_call_FPU );
- ins_pipe( pipe_slow );
-%}
-
-instruct CallLeafNoFPDirect(method meth) %{
- match(CallLeafNoFP);
- effect(USE meth);
-
- ins_cost(300);
- format %{ "CALL_LEAF_NOFP,runtime " %}
- opcode(0xE8); /* E8 cd */
- ins_encode(pre_call_resets, Java_To_Runtime(meth));
- ins_pipe( pipe_slow );
-%}
-
-
-// Return Instruction
-// Remove the return address & jump to it.
-instruct Ret() %{
- match(Return);
- format %{ "RET" %}
- opcode(0xC3);
- ins_encode(OpcP);
- ins_pipe( pipe_jmp );
-%}
-
-// Tail Call; Jump from runtime stub to Java code.
-// Also known as an 'interprocedural jump'.
-// Target of jump will eventually return to caller.
-// TailJump below removes the return address.
-// Don't use ebp for 'jump_target' because a MachEpilogNode has already been
-// emitted just above the TailCall which has reset ebp to the caller state.
-instruct TailCalljmpInd(eRegP_no_EBP jump_target, eBXRegP method_ptr) %{
- match(TailCall jump_target method_ptr);
- ins_cost(300);
- format %{ "JMP $jump_target \t# EBX holds method" %}
- opcode(0xFF, 0x4); /* Opcode FF /4 */
- ins_encode( OpcP, RegOpc(jump_target) );
- ins_pipe( pipe_jmp );
-%}
-
-
-// Tail Jump; remove the return address; jump to target.
-// TailCall above leaves the return address around.
-instruct tailjmpInd(eRegP_no_EBP jump_target, eAXRegP ex_oop) %{
- match( TailJump jump_target ex_oop );
- ins_cost(300);
- format %{ "POP EDX\t# pop return address into dummy\n\t"
- "JMP $jump_target " %}
- opcode(0xFF, 0x4); /* Opcode FF /4 */
- ins_encode( enc_pop_rdx,
- OpcP, RegOpc(jump_target) );
- ins_pipe( pipe_jmp );
-%}
-
-// Forward exception.
-instruct ForwardExceptionjmp()
-%{
- match(ForwardException);
-
- format %{ "JMP forward_exception_stub" %}
- ins_encode %{
- __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()), noreg);
- %}
- ins_pipe(pipe_jmp);
-%}
-
-// Create exception oop: created by stack-crawling runtime code.
-// Created exception is now available to this handler, and is setup
-// just prior to jumping to this handler. No code emitted.
-instruct CreateException( eAXRegP ex_oop )
-%{
- match(Set ex_oop (CreateEx));
-
- size(0);
- // use the following format syntax
- format %{ "# exception oop is in EAX; no code emitted" %}
- ins_encode();
- ins_pipe( empty );
-%}
-
-
-// Rethrow exception:
-// The exception oop will come in the first argument position.
-// Then JUMP (not call) to the rethrow stub code.
-instruct RethrowException()
-%{
- match(Rethrow);
-
- // use the following format syntax
- format %{ "JMP rethrow_stub" %}
- ins_encode(enc_rethrow);
- ins_pipe( pipe_jmp );
-%}
-
-// inlined locking and unlocking
-
-instruct cmpFastLock(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI tmp, eRegP scr, eRegP thread) %{
- predicate(LockingMode != LM_LIGHTWEIGHT);
- match(Set cr (FastLock object box));
- effect(TEMP tmp, TEMP scr, USE_KILL box, TEMP thread);
- ins_cost(300);
- format %{ "FASTLOCK $object,$box\t! kills $box,$tmp,$scr" %}
- ins_encode %{
- __ get_thread($thread$$Register);
- __ fast_lock($object$$Register, $box$$Register, $tmp$$Register,
- $scr$$Register, noreg, noreg, $thread$$Register, nullptr);
- %}
- ins_pipe(pipe_slow);
-%}
-
-instruct cmpFastUnlock(eFlagsReg cr, eRegP object, eAXRegP box, eRegP tmp ) %{
- predicate(LockingMode != LM_LIGHTWEIGHT);
- match(Set cr (FastUnlock object box));
- effect(TEMP tmp, USE_KILL box);
- ins_cost(300);
- format %{ "FASTUNLOCK $object,$box\t! kills $box,$tmp" %}
- ins_encode %{
- __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
- %}
- ins_pipe(pipe_slow);
-%}
-
-instruct cmpFastLockLightweight(eFlagsReg cr, eRegP object, eBXRegP box, eAXRegI eax_reg, eRegP tmp, eRegP thread) %{
- predicate(LockingMode == LM_LIGHTWEIGHT);
- match(Set cr (FastLock object box));
- effect(TEMP eax_reg, TEMP tmp, USE_KILL box, TEMP thread);
- ins_cost(300);
- format %{ "FASTLOCK $object,$box\t! kills $box,$eax_reg,$tmp" %}
- ins_encode %{
- __ get_thread($thread$$Register);
- __ fast_lock_lightweight($object$$Register, $box$$Register, $eax_reg$$Register, $tmp$$Register, $thread$$Register);
- %}
- ins_pipe(pipe_slow);
-%}
-
-instruct cmpFastUnlockLightweight(eFlagsReg cr, eRegP object, eAXRegP eax_reg, eRegP tmp, eRegP thread) %{
- predicate(LockingMode == LM_LIGHTWEIGHT);
- match(Set cr (FastUnlock object eax_reg));
- effect(TEMP tmp, USE_KILL eax_reg, TEMP thread);
- ins_cost(300);
- format %{ "FASTUNLOCK $object,$eax_reg\t! kills $eax_reg,$tmp" %}
- ins_encode %{
- __ get_thread($thread$$Register);
- __ fast_unlock_lightweight($object$$Register, $eax_reg$$Register, $tmp$$Register, $thread$$Register);
- %}
- ins_pipe(pipe_slow);
-%}
-
-instruct mask_all_evexL_LT32(kReg dst, eRegL src) %{
- predicate(Matcher::vector_length(n) <= 32);
- match(Set dst (MaskAll src));
- format %{ "mask_all_evexL_LE32 $dst, $src \t" %}
- ins_encode %{
- int mask_len = Matcher::vector_length(this);
- __ vector_maskall_operation($dst$$KRegister, $src$$Register, mask_len);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct mask_all_evexL_GT32(kReg dst, eRegL src, kReg ktmp) %{
- predicate(Matcher::vector_length(n) > 32);
- match(Set dst (MaskAll src));
- effect(TEMP ktmp);
- format %{ "mask_all_evexL_GT32 $dst, $src \t! using $ktmp as TEMP " %}
- ins_encode %{
- int mask_len = Matcher::vector_length(this);
- __ vector_maskall_operation32($dst$$KRegister, $src$$Register, $ktmp$$KRegister, mask_len);
- %}
- ins_pipe( pipe_slow );
-%}
-
-instruct mask_all_evexI_GT32(kReg dst, rRegI src, kReg ktmp) %{
- predicate(Matcher::vector_length(n) > 32);
- match(Set dst (MaskAll src));
- effect(TEMP ktmp);
- format %{ "mask_all_evexI_GT32 $dst, $src \t! using $ktmp as TEMP" %}
- ins_encode %{
- int mask_len = Matcher::vector_length(this);
- __ vector_maskall_operation32($dst$$KRegister, $src$$Register, $ktmp$$KRegister, mask_len);
- %}
- ins_pipe( pipe_slow );
-%}
-
-// ============================================================================
-// Safepoint Instruction
-instruct safePoint_poll_tls(eFlagsReg cr, eRegP_no_EBP poll) %{
- match(SafePoint poll);
- effect(KILL cr, USE poll);
-
- format %{ "TSTL #EAX,[$poll]\t! Safepoint: poll for GC" %}
- ins_cost(125);
- // EBP would need size(3)
- size(2); /* setting an explicit size will cause debug builds to assert if size is incorrect */
- ins_encode %{
- __ set_inst_mark();
- __ relocate(relocInfo::poll_type);
- __ clear_inst_mark();
- address pre_pc = __ pc();
- __ testl(rax, Address($poll$$Register, 0));
- address post_pc = __ pc();
- guarantee(pre_pc[0] == 0x85, "must emit test-ax [reg]");
- %}
- ins_pipe(ialu_reg_mem);
-%}
-
-
-// ============================================================================
-// This name is KNOWN by the ADLC and cannot be changed.
-// The ADLC forces a 'TypeRawPtr::BOTTOM' output type
-// for this guy.
-instruct tlsLoadP(eRegP dst, eFlagsReg cr) %{
- match(Set dst (ThreadLocal));
- effect(DEF dst, KILL cr);
-
- format %{ "MOV $dst, Thread::current()" %}
- ins_encode %{
- Register dstReg = as_Register($dst$$reg);
- __ get_thread(dstReg);
- %}
- ins_pipe( ialu_reg_fat );
-%}
-
-
-
-//----------PEEPHOLE RULES-----------------------------------------------------
-// These must follow all instruction definitions as they use the names
-// defined in the instructions definitions.
-//
-// peepmatch ( root_instr_name [preceding_instruction]* );
-//
-// peepconstraint %{
-// (instruction_number.operand_name relational_op instruction_number.operand_name
-// [, ...] );
-// // instruction numbers are zero-based using left to right order in peepmatch
-//
-// peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
-// // provide an instruction_number.operand_name for each operand that appears
-// // in the replacement instruction's match rule
-//
-// ---------VM FLAGS---------------------------------------------------------
-//
-// All peephole optimizations can be turned off using -XX:-OptoPeephole
-//
-// Each peephole rule is given an identifying number starting with zero and
-// increasing by one in the order seen by the parser. An individual peephole
-// can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
-// on the command-line.
-//
-// ---------CURRENT LIMITATIONS----------------------------------------------
-//
-// Only match adjacent instructions in same basic block
-// Only equality constraints
-// Only constraints between operands, not (0.dest_reg == EAX_enc)
-// Only one replacement instruction
-//
-// ---------EXAMPLE----------------------------------------------------------
-//
-// // pertinent parts of existing instructions in architecture description
-// instruct movI(rRegI dst, rRegI src) %{
-// match(Set dst (CopyI src));
-// %}
-//
-// instruct incI_eReg(rRegI dst, immI_1 src, eFlagsReg cr) %{
-// match(Set dst (AddI dst src));
-// effect(KILL cr);
-// %}
-//
-// // Change (inc mov) to lea
-// peephole %{
-// // increment preceded by register-register move
-// peepmatch ( incI_eReg movI );
-// // require that the destination register of the increment
-// // match the destination register of the move
-// peepconstraint ( 0.dst == 1.dst );
-// // construct a replacement instruction that sets
-// // the destination to ( move's source register + one )
-// peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
-// %}
-//
-// Implementation no longer uses movX instructions since
-// machine-independent system no longer uses CopyX nodes.
-//
-// peephole %{
-// peepmatch ( incI_eReg movI );
-// peepconstraint ( 0.dst == 1.dst );
-// peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
-// %}
-//
-// peephole %{
-// peepmatch ( decI_eReg movI );
-// peepconstraint ( 0.dst == 1.dst );
-// peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
-// %}
-//
-// peephole %{
-// peepmatch ( addI_eReg_imm movI );
-// peepconstraint ( 0.dst == 1.dst );
-// peepreplace ( leaI_eReg_immI( 0.dst 1.src 0.src ) );
-// %}
-//
-// peephole %{
-// peepmatch ( addP_eReg_imm movP );
-// peepconstraint ( 0.dst == 1.dst );
-// peepreplace ( leaP_eReg_immI( 0.dst 1.src 0.src ) );
-// %}
-
-// // Change load of spilled value to only a spill
-// instruct storeI(memory mem, rRegI src) %{
-// match(Set mem (StoreI mem src));
-// %}
-//
-// instruct loadI(rRegI dst, memory mem) %{
-// match(Set dst (LoadI mem));
-// %}
-//
-peephole %{
- peepmatch ( loadI storeI );
- peepconstraint ( 1.src == 0.dst, 1.mem == 0.mem );
- peepreplace ( storeI( 1.mem 1.mem 1.src ) );
-%}
-
-//----------SMARTSPILL RULES---------------------------------------------------
-// These must follow all instruction definitions as they use the names
-// defined in the instructions definitions.
diff --git a/src/hotspot/cpu/x86/x86_64.ad b/src/hotspot/cpu/x86/x86_64.ad
index 8cc4a970bfd..5781ffce4bd 100644
--- a/src/hotspot/cpu/x86/x86_64.ad
+++ b/src/hotspot/cpu/x86/x86_64.ad
@@ -818,7 +818,7 @@ void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
st->print("# stack alignment check");
#endif
}
- if (C->stub_function() != nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
+ if (C->stub_function() != nullptr) {
st->print("\n\t");
st->print("cmpl [r15_thread + #disarmed_guard_value_offset], #disarmed_guard_value\t");
st->print("\n\t");
@@ -845,7 +845,7 @@ void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
Register klass = rscratch1;
__ mov_metadata(klass, C->method()->holder()->constant_encoding());
- __ clinit_barrier(klass, r15_thread, &L_skip_barrier /*L_fast_path*/);
+ __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
__ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
@@ -943,7 +943,7 @@ void MachEpilogNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
code_stub = &stub->entry();
}
__ relocate(relocInfo::poll_return_type);
- __ safepoint_poll(*code_stub, r15_thread, true /* at_return */, true /* in_nmethod */);
+ __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
}
}
@@ -1547,7 +1547,11 @@ void BoxLockNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
uint BoxLockNode::size(PhaseRegAlloc *ra_) const
{
int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
- return (offset < 0x80) ? 5 : 8; // REX
+ if (ra_->get_encode(this) > 15) {
+ return (offset < 0x80) ? 6 : 9; // REX2
+ } else {
+ return (offset < 0x80) ? 5 : 8; // REX
+ }
}
//=============================================================================
@@ -4377,16 +4381,21 @@ instruct loadNKlassCompactHeaders(rRegN dst, memory mem, rFlagsReg cr)
predicate(UseCompactObjectHeaders);
match(Set dst (LoadNKlass mem));
effect(KILL cr);
- ins_cost(125); // XXX
+ ins_cost(125);
format %{
"movl $dst, $mem\t# compressed klass ptr, shifted\n\t"
"shrl $dst, markWord::klass_shift_at_offset"
%}
ins_encode %{
- __ movl($dst$$Register, $mem$$Address);
- __ shrl($dst$$Register, markWord::klass_shift_at_offset);
+ if (UseAPX) {
+ __ eshrl($dst$$Register, $mem$$Address, markWord::klass_shift_at_offset, false);
+ }
+ else {
+ __ movl($dst$$Register, $mem$$Address);
+ __ shrl($dst$$Register, markWord::klass_shift_at_offset);
+ }
%}
- ins_pipe(ialu_reg_mem); // XXX
+ ins_pipe(ialu_reg_mem);
%}
// Load Float
@@ -6136,6 +6145,7 @@ instruct cmovI_imm_01(rRegI dst, immI_1 src, rFlagsReg cr, cmpOp cop)
instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6146,6 +6156,19 @@ instruct cmovI_reg(rRegI dst, rRegI src, rFlagsReg cr, cmpOp cop)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovI_reg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr, cmpOp cop)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
%{
predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
@@ -6161,6 +6184,7 @@ instruct cmovI_imm_01U(rRegI dst, immI_1 src, rFlagsRegU cr, cmpOpU cop)
%}
instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6171,6 +6195,18 @@ instruct cmovI_regU(cmpOpU cop, rFlagsRegU cr, rRegI dst, rRegI src) %{
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovI_regU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, rRegI src2) %{
+ predicate(UseAPX);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
%{
predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_int() == 0);
@@ -6186,6 +6222,7 @@ instruct cmovI_imm_01UCF(rRegI dst, immI_1 src, rFlagsRegUCF cr, cmpOpUCF cop)
%}
instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
ins_cost(200);
expand %{
@@ -6193,8 +6230,19 @@ instruct cmovI_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
%}
%}
+instruct cmovI_regUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, rRegI src2) %{
+ predicate(UseAPX);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
+ ins_cost(200);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
- predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveI (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6207,10 +6255,24 @@ instruct cmovI_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovI_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
+ predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovpl $dst, $src1, $src2\n\t"
+ "ecmovnel $dst, $src1, $src2" %}
+ ins_encode %{
+ __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
+ __ ecmovl(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src) %{
- predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveI (Binary cop cr) (Binary src dst)));
ins_cost(200); // XXX
@@ -6223,8 +6285,25 @@ instruct cmovI_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src)
ins_pipe(pipe_cmov_reg);
%}
+// We need this special handling for only eq / neq comparison since NaN == NaN is false,
+// and parity flag bit is set if any of the operand is a NaN.
+instruct cmovI_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegI dst, rRegI src1, rRegI src2) %{
+ predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovpl $dst, $src1, $src2\n\t"
+ "ecmovnel $dst, $src1, $src2" %}
+ ins_encode %{
+ __ ecmovl(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
+ __ ecmovl(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
// Conditional move
instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
ins_cost(250); // XXX
@@ -6235,9 +6314,24 @@ instruct cmovI_mem(cmpOp cop, rFlagsReg cr, rRegI dst, memory src) %{
ins_pipe(pipe_cmov_mem);
%}
+// Conditional move
+instruct cmovI_rReg_rReg_mem_ndd(rRegI dst, cmpOp cop, rFlagsReg cr, rRegI src1, memory src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
+
+ ins_cost(250);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, int ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
+ %}
+ ins_pipe(pipe_cmov_mem);
+%}
+
// Conditional move
instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
ins_cost(250); // XXX
@@ -6249,6 +6343,7 @@ instruct cmovI_memU(cmpOpU cop, rFlagsRegU cr, rRegI dst, memory src)
%}
instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveI (Binary cop cr) (Binary dst (LoadI src))));
ins_cost(250);
expand %{
@@ -6256,9 +6351,35 @@ instruct cmovI_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegI dst, memory src) %{
%}
%}
+instruct cmovI_rReg_rReg_memU_ndd(rRegI dst, cmpOpU cop, rFlagsRegU cr, rRegI src1, memory src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
+
+ ins_cost(250);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
+ %}
+ ins_pipe(pipe_cmov_mem);
+%}
+
+instruct cmovI_rReg_rReg_memUCF_ndd(rRegI dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegI src1, memory src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveI (Binary cop cr) (Binary src1 (LoadI src2))));
+ ins_cost(250);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, int ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
+ %}
+ ins_pipe(pipe_cmov_mem);
+%}
+
// Conditional move
instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6269,9 +6390,24 @@ instruct cmovN_reg(rRegN dst, rRegN src, rFlagsReg cr, cmpOp cop)
ins_pipe(pipe_cmov_reg);
%}
+// Conditional move ndd
+instruct cmovN_reg_ndd(rRegN dst, rRegN src1, rRegN src2, rFlagsReg cr, cmpOp cop)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# signed, compressed ptr ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
// Conditional move
instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6283,6 +6419,7 @@ instruct cmovN_regU(cmpOpU cop, rFlagsRegU cr, rRegN dst, rRegN src)
%}
instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
ins_cost(200);
expand %{
@@ -6290,6 +6427,31 @@ instruct cmovN_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
%}
%}
+// Conditional move ndd
+instruct cmovN_regU_ndd(rRegN dst, cmpOpU cop, rFlagsRegU cr, rRegN src1, rRegN src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
+instruct cmovN_regUCF_ndd(rRegN dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegN src1, rRegN src2) %{
+ predicate(UseAPX);
+ match(Set dst (CMoveN (Binary cop cr) (Binary src1 src2)));
+ ins_cost(200);
+ format %{ "ecmovl$cop $dst, $src1, $src2\t# unsigned, compressed ptr ndd" %}
+ ins_encode %{
+ __ ecmovl((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovN_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src) %{
predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveN (Binary cop cr) (Binary dst src)));
@@ -6323,6 +6485,7 @@ instruct cmovN_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegN dst, rRegN src)
// Conditional move
instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6333,9 +6496,24 @@ instruct cmovP_reg(rRegP dst, rRegP src, rFlagsReg cr, cmpOp cop)
ins_pipe(pipe_cmov_reg); // XXX
%}
+// Conditional move ndd
+instruct cmovP_reg_ndd(rRegP dst, rRegP src1, rRegP src2, rFlagsReg cr, cmpOp cop)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, ptr ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
// Conditional move
instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6346,6 +6524,20 @@ instruct cmovP_regU(cmpOpU cop, rFlagsRegU cr, rRegP dst, rRegP src)
ins_pipe(pipe_cmov_reg); // XXX
%}
+// Conditional move ndd
+instruct cmovP_regU_ndd(rRegP dst, cmpOpU cop, rFlagsRegU cr, rRegP src1, rRegP src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
ins_cost(200);
@@ -6354,8 +6546,18 @@ instruct cmovP_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
%}
%}
+instruct cmovP_regUCF_ndd(rRegP dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegP src1, rRegP src2) %{
+ match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, ptr ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
- predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveP (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6368,10 +6570,24 @@ instruct cmovP_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovP_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
+ predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovpq $dst, $src1, $src2\n\t"
+ "ecmovneq $dst, $src1, $src2" %}
+ ins_encode %{
+ __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
+ __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src) %{
- predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveP (Binary cop cr) (Binary src dst)));
ins_cost(200); // XXX
@@ -6384,6 +6600,20 @@ instruct cmovP_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovP_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegP dst, rRegP src1, rRegP src2) %{
+ predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ match(Set dst (CMoveP (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovpq $dst, $src1, $src2\n\t"
+ "ecmovneq $dst, $src1, $src2" %}
+ ins_encode %{
+ __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
+ __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
%{
predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
@@ -6400,6 +6630,7 @@ instruct cmovL_imm_01(rRegL dst, immL1 src, rFlagsReg cr, cmpOp cop)
instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6410,8 +6641,22 @@ instruct cmovL_reg(cmpOp cop, rFlagsReg cr, rRegL dst, rRegL src)
ins_pipe(pipe_cmov_reg); // XXX
%}
+instruct cmovL_reg_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, rRegL src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
ins_cost(200); // XXX
@@ -6422,6 +6667,19 @@ instruct cmovL_mem(cmpOp cop, rFlagsReg cr, rRegL dst, memory src)
ins_pipe(pipe_cmov_mem); // XXX
%}
+instruct cmovL_rReg_rReg_mem_ndd(rRegL dst, cmpOp cop, rFlagsReg cr, rRegL src1, memory src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
+
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# signed, long ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
+ %}
+ ins_pipe(pipe_cmov_mem);
+%}
+
instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
%{
predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
@@ -6438,6 +6696,7 @@ instruct cmovL_imm_01U(rRegL dst, immL1 src, rFlagsRegU cr, cmpOpU cop)
instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6448,6 +6707,19 @@ instruct cmovL_regU(cmpOpU cop, rFlagsRegU cr, rRegL dst, rRegL src)
ins_pipe(pipe_cmov_reg); // XXX
%}
+instruct cmovL_regU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, rRegL src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
%{
predicate(n->in(2)->in(2)->is_Con() && n->in(2)->in(2)->get_long() == 0);
@@ -6463,6 +6735,7 @@ instruct cmovL_imm_01UCF(rRegL dst, immL1 src, rFlagsRegUCF cr, cmpOpUCF cop)
%}
instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
ins_cost(200);
expand %{
@@ -6470,8 +6743,20 @@ instruct cmovL_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
%}
%}
+instruct cmovL_regUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, rRegL src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
- predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
match(Set dst (CMoveL (Binary cop cr) (Binary dst src)));
ins_cost(200); // XXX
@@ -6484,10 +6769,24 @@ instruct cmovL_regUCF2_ne(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovL_regUCF2_ne_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
+ predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::ne);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovpq $dst, $src1, $src2\n\t"
+ "ecmovneq $dst, $src1, $src2" %}
+ ins_encode %{
+ __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
+ __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
// Since (x == y) == !(x != y), we can flip the sense of the test by flipping the
// inputs of the CMove
instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src) %{
- predicate(n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ predicate(!UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
match(Set dst (CMoveL (Binary cop cr) (Binary src dst)));
ins_cost(200); // XXX
@@ -6500,8 +6799,23 @@ instruct cmovL_regUCF2_eq(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovL_regUCF2_eq_ndd(cmpOpUCF2 cop, rFlagsRegUCF cr, rRegL dst, rRegL src1, rRegL src2) %{
+ predicate(UseAPX && n->in(1)->in(1)->as_Bool()->_test._test == BoolTest::eq);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 src2)));
+
+ ins_cost(200);
+ format %{ "ecmovpq $dst, $src1, $src2\n\t"
+ "ecmovneq $dst, $src1, $src2" %}
+ ins_encode %{
+ __ ecmovq(Assembler::parity, $dst$$Register, $src1$$Register, $src2$$Register);
+ __ ecmovq(Assembler::notEqual, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
+
instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
%{
+ predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
ins_cost(200); // XXX
@@ -6513,6 +6827,7 @@ instruct cmovL_memU(cmpOpU cop, rFlagsRegU cr, rRegL dst, memory src)
%}
instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
+ predicate(!UseAPX);
match(Set dst (CMoveL (Binary cop cr) (Binary dst (LoadL src))));
ins_cost(200);
expand %{
@@ -6520,6 +6835,31 @@ instruct cmovL_memUCF(cmpOpUCF cop, rFlagsRegUCF cr, rRegL dst, memory src) %{
%}
%}
+instruct cmovL_rReg_rReg_memU_ndd(rRegL dst, cmpOpU cop, rFlagsRegU cr, rRegL src1, memory src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
+
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
+ %}
+ ins_pipe(pipe_cmov_mem);
+%}
+
+instruct cmovL_rReg_rReg_memUCF_ndd(rRegL dst, cmpOpUCF cop, rFlagsRegUCF cr, rRegL src1, memory src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (CMoveL (Binary cop cr) (Binary src1 (LoadL src2))));
+ ins_cost(200);
+ format %{ "ecmovq$cop $dst, $src1, $src2\t# unsigned, long ndd" %}
+ ins_encode %{
+ __ ecmovq((Assembler::Condition)($cop$$cmpcode), $dst$$Register, $src1$$Register, $src2$$Address);
+ %}
+ ins_pipe(pipe_cmov_mem);
+%}
+
instruct cmovF_reg(cmpOp cop, rFlagsReg cr, regF dst, regF src)
%{
match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
@@ -6613,6 +6953,7 @@ instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AddI dst src));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -6623,8 +6964,23 @@ instruct addI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+instruct addI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eaddl($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AddI dst src));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -6636,8 +6992,37 @@ instruct addI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
ins_pipe( ialu_reg );
%}
+instruct addI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eaddl($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe( ialu_reg );
+%}
+
+instruct addI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eaddl($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe( ialu_reg );
+%}
+
instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AddI dst (LoadI src)));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -6650,6 +7035,36 @@ instruct addI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct addI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eaddl($dst$$Register, $src1$$Address, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct addI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddI src1 (LoadI src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "eaddl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eaddl($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
instruct addI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
%{
match(Set dst (StoreI dst (AddI (LoadI dst) src)));
@@ -6681,7 +7096,7 @@ instruct addI_mem_imm(memory dst, immI src, rFlagsReg cr)
instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
%{
- predicate(UseIncDec);
+ predicate(!UseAPX && UseIncDec);
match(Set dst (AddI dst src));
effect(KILL cr);
@@ -6692,6 +7107,32 @@ instruct incI_rReg(rRegI dst, immI_1 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct incI_rReg_ndd(rRegI dst, rRegI src, immI_1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddI src val));
+ effect(KILL cr);
+
+ format %{ "eincl $dst, $src\t# int ndd" %}
+ ins_encode %{
+ __ eincl($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct incI_rReg_mem_ndd(rRegI dst, memory src, immI_1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddI (LoadI src) val));
+ effect(KILL cr);
+
+ format %{ "eincl $dst, $src\t# int ndd" %}
+ ins_encode %{
+ __ eincl($dst$$Register, $src$$Address, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
%{
predicate(UseIncDec);
@@ -6709,7 +7150,7 @@ instruct incI_mem(memory dst, immI_1 src, rFlagsReg cr)
// XXX why does that use AddI
instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
%{
- predicate(UseIncDec);
+ predicate(!UseAPX && UseIncDec);
match(Set dst (AddI dst src));
effect(KILL cr);
@@ -6720,6 +7161,32 @@ instruct decI_rReg(rRegI dst, immI_M1 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct decI_rReg_ndd(rRegI dst, rRegI src, immI_M1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddI src val));
+ effect(KILL cr);
+
+ format %{ "edecl $dst, $src\t# int ndd" %}
+ ins_encode %{
+ __ edecl($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct decI_rReg_mem_ndd(rRegI dst, memory src, immI_M1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddI (LoadI src) val));
+ effect(KILL cr);
+
+ format %{ "edecl $dst, $src\t# int ndd" %}
+ ins_encode %{
+ __ edecl($dst$$Register, $src$$Address, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// XXX why does that use AddI
instruct decI_mem(memory dst, immI_M1 src, rFlagsReg cr)
%{
@@ -6788,6 +7255,7 @@ instruct leaI_rReg_rReg_immI2_immI(rRegI dst, rRegI base, rRegI index, immI2 sca
instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AddL dst src));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -6799,8 +7267,23 @@ instruct addL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+instruct addL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eaddq($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AddL dst src));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -6812,8 +7295,37 @@ instruct addL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
ins_pipe( ialu_reg );
%}
+instruct addL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eaddq($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe( ialu_reg );
+%}
+
+instruct addL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eaddq($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe( ialu_reg );
+%}
+
instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AddL dst (LoadL src)));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -6826,6 +7338,36 @@ instruct addL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct addL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddL src1 (LoadL src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eaddq($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct addL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AddL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "eaddq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eaddq($dst$$Register, $src1$$Address, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
instruct addL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
%{
match(Set dst (StoreL dst (AddL (LoadL dst) src)));
@@ -6856,7 +7398,7 @@ instruct addL_mem_imm(memory dst, immL32 src, rFlagsReg cr)
instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
%{
- predicate(UseIncDec);
+ predicate(!UseAPX && UseIncDec);
match(Set dst (AddL dst src));
effect(KILL cr);
@@ -6867,6 +7409,32 @@ instruct incL_rReg(rRegI dst, immL1 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct incL_rReg_ndd(rRegI dst, rRegI src, immL1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddL src val));
+ effect(KILL cr);
+
+ format %{ "eincq $dst, $src\t# long ndd" %}
+ ins_encode %{
+ __ eincq($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct incL_rReg_mem_ndd(rRegI dst, memory src, immL1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddL (LoadL src) val));
+ effect(KILL cr);
+
+ format %{ "eincq $dst, $src\t# long ndd" %}
+ ins_encode %{
+ __ eincq($dst$$Register, $src$$Address, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
%{
predicate(UseIncDec);
@@ -6884,7 +7452,7 @@ instruct incL_mem(memory dst, immL1 src, rFlagsReg cr)
// XXX why does that use AddL
instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
%{
- predicate(UseIncDec);
+ predicate(!UseAPX && UseIncDec);
match(Set dst (AddL dst src));
effect(KILL cr);
@@ -6895,6 +7463,32 @@ instruct decL_rReg(rRegL dst, immL_M1 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct decL_rReg_ndd(rRegL dst, rRegL src, immL_M1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddL src val));
+ effect(KILL cr);
+
+ format %{ "edecq $dst, $src\t# long ndd" %}
+ ins_encode %{
+ __ edecq($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct decL_rReg_mem_ndd(rRegL dst, memory src, immL_M1 val, rFlagsReg cr)
+%{
+ predicate(UseAPX && UseIncDec);
+ match(Set dst (AddL (LoadL src) val));
+ effect(KILL cr);
+
+ format %{ "edecq $dst, $src\t# long ndd" %}
+ ins_encode %{
+ __ edecq($dst$$Register, $src$$Address, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// XXX why does that use AddL
instruct decL_mem(memory dst, immL_M1 src, rFlagsReg cr)
%{
@@ -7530,6 +8124,7 @@ instruct absL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
// Integer Subtraction Instructions
instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (SubI dst src));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -7541,8 +8136,51 @@ instruct subI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+instruct subI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ esubl($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
+instruct subI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ esubl($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
+instruct subI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ esubl($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (SubI dst (LoadI src)));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -7555,6 +8193,36 @@ instruct subI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct subI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubI src1 (LoadI src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ esubl($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct subI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "esubl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ esubl($dst$$Register, $src1$$Address, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
%{
match(Set dst (StoreI dst (SubI (LoadI dst) src)));
@@ -7571,6 +8239,7 @@ instruct subI_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (SubL dst src));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -7582,8 +8251,51 @@ instruct subL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+instruct subL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ esubq($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
+instruct subL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ esubq($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
+instruct subL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ esubq($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (SubL dst (LoadL src)));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
@@ -7596,6 +8308,36 @@ instruct subL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct subL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubL src1 (LoadL src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ esubq($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct subL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_carry_flag, PD::Flag_sets_parity_flag);
+
+ ins_cost(150);
+ format %{ "esubq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ esubq($dst$$Register, $src1$$Address, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
instruct subL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
%{
match(Set dst (StoreL dst (SubL (LoadL dst) src)));
@@ -7626,6 +8368,7 @@ instruct subP_rReg(rRegP dst, rRegI src, immI_0 zero, rFlagsReg cr)
instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (SubI zero dst));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
@@ -7637,8 +8380,23 @@ instruct negI_rReg(rRegI dst, immI_0 zero, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct negI_rReg_ndd(rRegI dst, rRegI src, immI_0 zero, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubI zero src));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "enegl $dst, $src\t# int ndd" %}
+ ins_encode %{
+ __ enegl($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (NegI dst));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
@@ -7650,6 +8408,20 @@ instruct negI_rReg_2(rRegI dst, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct negI_rReg_2_ndd(rRegI dst, rRegI src, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (NegI src));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "enegl $dst, $src\t# int ndd" %}
+ ins_encode %{
+ __ enegl($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
%{
match(Set dst (StoreI dst (SubI zero (LoadI dst))));
@@ -7665,6 +8437,7 @@ instruct negI_mem(memory dst, immI_0 zero, rFlagsReg cr)
instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (SubL zero dst));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
@@ -7676,8 +8449,23 @@ instruct negL_rReg(rRegL dst, immL0 zero, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct negL_rReg_ndd(rRegL dst, rRegL src, immL0 zero, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (SubL zero src));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "enegq $dst, $src\t# long ndd" %}
+ ins_encode %{
+ __ enegq($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (NegL dst));
effect(KILL cr);
flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
@@ -7689,6 +8477,20 @@ instruct negL_rReg_2(rRegL dst, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct negL_rReg_2_ndd(rRegL dst, rRegL src, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (NegL src));
+ effect(KILL cr);
+ flag(PD::Flag_sets_overflow_flag, PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag);
+
+ format %{ "enegq $dst, $src\t# long ndd" %}
+ ins_encode %{
+ __ enegq($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
%{
match(Set dst (StoreL dst (SubL zero (LoadL dst))));
@@ -7708,6 +8510,7 @@ instruct negL_mem(memory dst, immL0 zero, rFlagsReg cr)
instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulI dst src));
effect(KILL cr);
@@ -7719,8 +8522,23 @@ instruct mulI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(ialu_reg_reg_alu0);
%}
+instruct mulI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulI src1 src2));
+ effect(KILL cr);
+
+ ins_cost(300);
+ format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eimull($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg_alu0);
+%}
+
instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulI src imm));
effect(KILL cr);
@@ -7732,8 +8550,23 @@ instruct mulI_rReg_imm(rRegI dst, rRegI src, immI imm, rFlagsReg cr)
ins_pipe(ialu_reg_reg_alu0);
%}
+instruct mulI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulI src1 src2));
+ effect(KILL cr);
+
+ ins_cost(300);
+ format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eimull($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_reg_alu0);
+%}
+
instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulI dst (LoadI src)));
effect(KILL cr);
@@ -7745,8 +8578,23 @@ instruct mulI_mem(rRegI dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem_alu0);
%}
+instruct mulI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulI src1 (LoadI src2)));
+ effect(KILL cr);
+
+ ins_cost(350);
+ format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eimull($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem_alu0);
+%}
+
instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulI (LoadI src) imm));
effect(KILL cr);
@@ -7758,6 +8606,20 @@ instruct mulI_mem_imm(rRegI dst, memory src, immI imm, rFlagsReg cr)
ins_pipe(ialu_reg_mem_alu0);
%}
+instruct mulI_rReg_mem_imm(rRegI dst, memory src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulI (LoadI src1) src2));
+ effect(KILL cr);
+
+ ins_cost(300);
+ format %{ "eimull $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eimull($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_mem_alu0);
+%}
+
instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg cr)
%{
match(Set dst (MulAddS2I (Binary dst src1) (Binary src2 src3)));
@@ -7770,6 +8632,7 @@ instruct mulAddS2I_rReg(rRegI dst, rRegI src1, rRegI src2, rRegI src3, rFlagsReg
instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulL dst src));
effect(KILL cr);
@@ -7781,8 +8644,23 @@ instruct mulL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
ins_pipe(ialu_reg_reg_alu0);
%}
+instruct mulL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulL src1 src2));
+ effect(KILL cr);
+
+ ins_cost(300);
+ format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eimulq($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg_alu0);
+%}
+
instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulL src imm));
effect(KILL cr);
@@ -7794,8 +8672,23 @@ instruct mulL_rReg_imm(rRegL dst, rRegL src, immL32 imm, rFlagsReg cr)
ins_pipe(ialu_reg_reg_alu0);
%}
+instruct mulL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulL src1 src2));
+ effect(KILL cr);
+
+ ins_cost(300);
+ format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eimulq($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_reg_alu0);
+%}
+
instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulL dst (LoadL src)));
effect(KILL cr);
@@ -7807,8 +8700,23 @@ instruct mulL_mem(rRegL dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem_alu0);
%}
+instruct mulL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulL src1 (LoadL src2)));
+ effect(KILL cr);
+
+ ins_cost(350);
+ format %{ "eimulq $dst, $src1, $src2 \t# long" %}
+ ins_encode %{
+ __ eimulq($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem_alu0);
+%}
+
instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (MulL (LoadL src) imm));
effect(KILL cr);
@@ -7820,6 +8728,20 @@ instruct mulL_mem_imm(rRegL dst, memory src, immL32 imm, rFlagsReg cr)
ins_pipe(ialu_reg_mem_alu0);
%}
+instruct mulL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (MulL (LoadL src1) src2));
+ effect(KILL cr);
+
+ ins_cost(300);
+ format %{ "eimulq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eimulq($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg_mem_alu0);
+%}
+
instruct mulHiL_rReg(rdx_RegL dst, rRegL src, rax_RegL rax, rFlagsReg cr)
%{
match(Set dst (MulHiL src rax));
@@ -8055,6 +8977,35 @@ instruct umodL_rReg(rdx_RegL rdx, rax_RegL rax, no_rax_rdx_RegL div, rFlagsReg c
// Shift Left by one, two, three
instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
%{
+ predicate(!UseAPX);
+ match(Set dst (LShiftI dst shift));
+ effect(KILL cr);
+
+ format %{ "sall $dst, $shift" %}
+ ins_encode %{
+ __ sall($dst$$Register, $shift$$constant);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Shift Left by one, two, three
+instruct salI_rReg_immI2_ndd(rRegI dst, rRegI src, immI2 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (LShiftI src shift));
+ effect(KILL cr);
+
+ format %{ "esall $dst, $src, $shift\t# int(ndd)" %}
+ ins_encode %{
+ __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Shift Left by 8-bit immediate
+instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
+%{
+ predicate(!UseAPX);
match(Set dst (LShiftI dst shift));
effect(KILL cr);
@@ -8066,14 +9017,28 @@ instruct salI_rReg_immI2(rRegI dst, immI2 shift, rFlagsReg cr)
%}
// Shift Left by 8-bit immediate
-instruct salI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
+instruct salI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
%{
- match(Set dst (LShiftI dst shift));
+ predicate(UseAPX);
+ match(Set dst (LShiftI src shift));
effect(KILL cr);
- format %{ "sall $dst, $shift" %}
+ format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
ins_encode %{
- __ sall($dst$$Register, $shift$$constant);
+ __ esall($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct salI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (LShiftI (LoadI src) shift));
+ effect(KILL cr);
+
+ format %{ "esall $dst, $src, $shift\t# int (ndd)" %}
+ ins_encode %{
+ __ esall($dst$$Register, $src$$Address, $shift$$constant, false);
%}
ins_pipe(ialu_reg);
%}
@@ -8146,6 +9111,7 @@ instruct salI_mem_rReg(rRegI dst, memory src, rRegI shift)
// Arithmetic Shift Right by 8-bit immediate
instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (RShiftI dst shift));
effect(KILL cr);
@@ -8156,6 +9122,33 @@ instruct sarI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
ins_pipe(ialu_mem_imm);
%}
+// Arithmetic Shift Right by 8-bit immediate
+instruct sarI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (RShiftI src shift));
+ effect(KILL cr);
+
+ format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
+ ins_encode %{
+ __ esarl($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_mem_imm);
+%}
+
+instruct sarI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (RShiftI (LoadI src) shift));
+ effect(KILL cr);
+
+ format %{ "esarl $dst, $src, $shift\t# int (ndd)" %}
+ ins_encode %{
+ __ esarl($dst$$Register, $src$$Address, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_mem_imm);
+%}
+
// Arithmetic Shift Right by 8-bit immediate
instruct sarI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
%{
@@ -8224,6 +9217,7 @@ instruct sarI_mem_rReg(rRegI dst, memory src, rRegI shift)
// Logical Shift Right by 8-bit immediate
instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (URShiftI dst shift));
effect(KILL cr);
@@ -8234,6 +9228,33 @@ instruct shrI_rReg_imm(rRegI dst, immI8 shift, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+// Logical Shift Right by 8-bit immediate
+instruct shrI_rReg_imm_ndd(rRegI dst, rRegI src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (URShiftI src shift));
+ effect(KILL cr);
+
+ format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
+ ins_encode %{
+ __ eshrl($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct shrI_rReg_mem_imm_ndd(rRegI dst, memory src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (URShiftI (LoadI src) shift));
+ effect(KILL cr);
+
+ format %{ "eshrl $dst, $src, $shift\t # int (ndd)" %}
+ ins_encode %{
+ __ eshrl($dst$$Register, $src$$Address, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Logical Shift Right by 8-bit immediate
instruct shrI_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
%{
@@ -8303,6 +9324,35 @@ instruct shrI_mem_rReg(rRegI dst, memory src, rRegI shift)
// Shift Left by one, two, three
instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
%{
+ predicate(!UseAPX);
+ match(Set dst (LShiftL dst shift));
+ effect(KILL cr);
+
+ format %{ "salq $dst, $shift" %}
+ ins_encode %{
+ __ salq($dst$$Register, $shift$$constant);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Shift Left by one, two, three
+instruct salL_rReg_immI2_ndd(rRegL dst, rRegL src, immI2 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (LShiftL src shift));
+ effect(KILL cr);
+
+ format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
+ ins_encode %{
+ __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Shift Left by 8-bit immediate
+instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
+%{
+ predicate(!UseAPX);
match(Set dst (LShiftL dst shift));
effect(KILL cr);
@@ -8314,14 +9364,28 @@ instruct salL_rReg_immI2(rRegL dst, immI2 shift, rFlagsReg cr)
%}
// Shift Left by 8-bit immediate
-instruct salL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
+instruct salL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
%{
- match(Set dst (LShiftL dst shift));
+ predicate(UseAPX);
+ match(Set dst (LShiftL src shift));
effect(KILL cr);
- format %{ "salq $dst, $shift" %}
+ format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
ins_encode %{
- __ salq($dst$$Register, $shift$$constant);
+ __ esalq($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct salL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (LShiftL (LoadL src) shift));
+ effect(KILL cr);
+
+ format %{ "esalq $dst, $src, $shift\t# long (ndd)" %}
+ ins_encode %{
+ __ esalq($dst$$Register, $src$$Address, $shift$$constant, false);
%}
ins_pipe(ialu_reg);
%}
@@ -8394,6 +9458,7 @@ instruct salL_mem_rReg(rRegL dst, memory src, rRegI shift)
// Arithmetic Shift Right by 8-bit immediate
instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (RShiftL dst shift));
effect(KILL cr);
@@ -8404,6 +9469,33 @@ instruct sarL_rReg_imm(rRegL dst, immI shift, rFlagsReg cr)
ins_pipe(ialu_mem_imm);
%}
+// Arithmetic Shift Right by 8-bit immediate
+instruct sarL_rReg_imm_ndd(rRegL dst, rRegL src, immI shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (RShiftL src shift));
+ effect(KILL cr);
+
+ format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
+ ins_encode %{
+ __ esarq($dst$$Register, $src$$Register, (unsigned char)($shift$$constant & 0x3F), false);
+ %}
+ ins_pipe(ialu_mem_imm);
+%}
+
+instruct sarL_rReg_mem_imm_ndd(rRegL dst, memory src, immI shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (RShiftL (LoadL src) shift));
+ effect(KILL cr);
+
+ format %{ "esarq $dst, $src, $shift\t# long (ndd)" %}
+ ins_encode %{
+ __ esarq($dst$$Register, $src$$Address, (unsigned char)($shift$$constant & 0x3F), false);
+ %}
+ ins_pipe(ialu_mem_imm);
+%}
+
// Arithmetic Shift Right by 8-bit immediate
instruct sarL_mem_imm(memory dst, immI shift, rFlagsReg cr)
%{
@@ -8472,6 +9564,7 @@ instruct sarL_mem_rReg(rRegL dst, memory src, rRegI shift)
// Logical Shift Right by 8-bit immediate
instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (URShiftL dst shift));
effect(KILL cr);
@@ -8482,6 +9575,33 @@ instruct shrL_rReg_imm(rRegL dst, immI8 shift, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+// Logical Shift Right by 8-bit immediate
+instruct shrL_rReg_imm_ndd(rRegL dst, rRegL src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (URShiftL src shift));
+ effect(KILL cr);
+
+ format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
+ ins_encode %{
+ __ eshrq($dst$$Register, $src$$Register, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct shrL_rReg_mem_imm_ndd(rRegL dst, memory src, immI8 shift, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (URShiftL (LoadL src) shift));
+ effect(KILL cr);
+
+ format %{ "eshrq $dst, $src, $shift\t# long (ndd)" %}
+ ins_encode %{
+ __ eshrq($dst$$Register, $src$$Address, $shift$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Logical Shift Right by 8-bit immediate
instruct shrL_mem_imm(memory dst, immI8 shift, rFlagsReg cr)
%{
@@ -8590,7 +9710,7 @@ instruct rolI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
instruct rolI_immI8(rRegI dst, rRegI src, immI8 shift)
%{
- predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
+ predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
match(Set dst (RotateLeft src shift));
format %{ "rolxl $dst, $src, $shift" %}
ins_encode %{
@@ -8616,7 +9736,7 @@ instruct rolI_mem_immI8(rRegI dst, memory src, immI8 shift)
// Rotate Left by variable
instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
%{
- predicate(n->bottom_type()->basic_type() == T_INT);
+ predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
match(Set dst (RotateLeft dst shift));
effect(KILL cr);
format %{ "roll $dst, $shift" %}
@@ -8626,6 +9746,20 @@ instruct rolI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Rotate Left by variable
+instruct rolI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
+%{
+ predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
+ match(Set dst (RotateLeft src shift));
+ effect(KILL cr);
+
+ format %{ "eroll $dst, $src, $shift\t# rotate left (int ndd)" %}
+ ins_encode %{
+ __ eroll($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Rotate Right by constant.
instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
%{
@@ -8642,7 +9776,7 @@ instruct rorI_immI8_legacy(rRegI dst, immI8 shift, rFlagsReg cr)
// Rotate Right by constant.
instruct rorI_immI8(rRegI dst, rRegI src, immI8 shift)
%{
- predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
+ predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_INT);
match(Set dst (RotateRight src shift));
format %{ "rorxl $dst, $src, $shift" %}
ins_encode %{
@@ -8666,7 +9800,7 @@ instruct rorI_mem_immI8(rRegI dst, memory src, immI8 shift)
// Rotate Right by variable
instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
%{
- predicate(n->bottom_type()->basic_type() == T_INT);
+ predicate(!UseAPX && n->bottom_type()->basic_type() == T_INT);
match(Set dst (RotateRight dst shift));
effect(KILL cr);
format %{ "rorl $dst, $shift" %}
@@ -8676,6 +9810,20 @@ instruct rorI_rReg_Var(rRegI dst, rcx_RegI shift, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Rotate Right by variable
+instruct rorI_rReg_Var_ndd(rRegI dst, rRegI src, rcx_RegI shift, rFlagsReg cr)
+%{
+ predicate(UseAPX && n->bottom_type()->basic_type() == T_INT);
+ match(Set dst (RotateRight src shift));
+ effect(KILL cr);
+
+ format %{ "erorl $dst, $src, $shift\t# rotate right(int ndd)" %}
+ ins_encode %{
+ __ erorl($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Rotate Left by constant.
instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
%{
@@ -8691,7 +9839,7 @@ instruct rolL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
instruct rolL_immI8(rRegL dst, rRegL src, immI8 shift)
%{
- predicate(VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
+ predicate(!UseAPX && VM_Version::supports_bmi2() && n->bottom_type()->basic_type() == T_LONG);
match(Set dst (RotateLeft src shift));
format %{ "rolxq $dst, $src, $shift" %}
ins_encode %{
@@ -8717,7 +9865,7 @@ instruct rolL_mem_immI8(rRegL dst, memory src, immI8 shift)
// Rotate Left by variable
instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
%{
- predicate(n->bottom_type()->basic_type() == T_LONG);
+ predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
match(Set dst (RotateLeft dst shift));
effect(KILL cr);
format %{ "rolq $dst, $shift" %}
@@ -8727,6 +9875,20 @@ instruct rolL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Rotate Left by variable
+instruct rolL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
+%{
+ predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
+ match(Set dst (RotateLeft src shift));
+ effect(KILL cr);
+
+ format %{ "erolq $dst, $src, $shift\t# rotate left(long ndd)" %}
+ ins_encode %{
+ __ erolq($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Rotate Right by constant.
instruct rorL_immI8_legacy(rRegL dst, immI8 shift, rFlagsReg cr)
%{
@@ -8767,7 +9929,7 @@ instruct rorL_mem_immI8(rRegL dst, memory src, immI8 shift)
// Rotate Right by variable
instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
%{
- predicate(n->bottom_type()->basic_type() == T_LONG);
+ predicate(!UseAPX && n->bottom_type()->basic_type() == T_LONG);
match(Set dst (RotateRight dst shift));
effect(KILL cr);
format %{ "rorq $dst, $shift" %}
@@ -8777,6 +9939,20 @@ instruct rorL_rReg_Var(rRegL dst, rcx_RegI shift, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Rotate Right by variable
+instruct rorL_rReg_Var_ndd(rRegL dst, rRegL src, rcx_RegI shift, rFlagsReg cr)
+%{
+ predicate(UseAPX && n->bottom_type()->basic_type() == T_LONG);
+ match(Set dst (RotateRight src shift));
+ effect(KILL cr);
+
+ format %{ "erorq $dst, $src, $shift\t# rotate right(long ndd)" %}
+ ins_encode %{
+ __ erorq($dst$$Register, $src$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
//----------------------------- CompressBits/ExpandBits ------------------------
instruct compressBitsL_reg(rRegL dst, rRegL src, rRegL mask) %{
@@ -8828,6 +10004,7 @@ instruct expandBitsL_mem(rRegL dst, rRegL src, memory mask) %{
// And Register with Register
instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AndI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -8839,6 +10016,22 @@ instruct andI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// And Register with Register using New Data Destination (NDD)
+instruct andI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eandl($dst$$Register, $src1$$Register, $src2$$Register, false);
+
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// And Register with Immediate 255
instruct andI_rReg_imm255(rRegI dst, rRegI src, immI_255 mask)
%{
@@ -8905,6 +10098,7 @@ instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rR
// And Register with Immediate
instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AndI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -8916,9 +10110,38 @@ instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct andI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eandl($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct andI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eandl($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// And Register with Memory
instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AndI dst (LoadI src)));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -8931,6 +10154,21 @@ instruct andI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct andI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndI src1 (LoadI src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "eandl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eandl($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
// And Memory with Register
instruct andB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
%{
@@ -9103,6 +10341,7 @@ instruct blsrI_rReg_mem(rRegI dst, memory src, immI_M1 minus_1, rFlagsReg cr)
// Or Register with Register
instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (OrI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9114,9 +10353,25 @@ instruct orI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Or Register with Register using New Data Destination (NDD)
+instruct orI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eorl($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Or Register with Immediate
instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (OrI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9128,9 +10383,52 @@ instruct orI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct orI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eorl($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct orI_rReg_imm_rReg_ndd(rRegI dst, immI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorl $dst, $src2, $src1\t# int ndd" %}
+ ins_encode %{
+ __ eorl($dst$$Register, $src2$$Register, $src1$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct orI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eorl($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Or Register with Memory
instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (OrI dst (LoadI src)));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9143,6 +10441,21 @@ instruct orI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct orI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrI src1 (LoadI src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "eorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ eorl($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
// Or Memory with Register
instruct orB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
%{
@@ -9191,6 +10504,7 @@ instruct orI_mem_imm(memory dst, immI src, rFlagsReg cr)
// Xor Register with Register
instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (XorI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9202,20 +10516,50 @@ instruct xorI_rReg(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Xor Register with Register using New Data Destination (NDD)
+instruct xorI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ exorl($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Xor Register with Immediate -1
-instruct xorI_rReg_im1(rRegI dst, immI_M1 imm) %{
+instruct xorI_rReg_im1(rRegI dst, immI_M1 imm)
+%{
+ predicate(!UseAPX);
match(Set dst (XorI dst imm));
- format %{ "not $dst" %}
+ format %{ "notl $dst" %}
ins_encode %{
__ notl($dst$$Register);
%}
ins_pipe(ialu_reg);
%}
+instruct xorI_rReg_im1_ndd(rRegI dst, rRegI src, immI_M1 imm)
+%{
+ match(Set dst (XorI src imm));
+ predicate(UseAPX);
+
+ format %{ "enotl $dst, $src" %}
+ ins_encode %{
+ __ enotl($dst$$Register, $src$$Register);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Xor Register with Immediate
instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (XorI dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9227,9 +10571,39 @@ instruct xorI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct xorI_rReg_rReg_imm_ndd(rRegI dst, rRegI src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorI src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ exorl($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Xor Memory with Immediate
+instruct xorI_rReg_mem_imm_ndd(rRegI dst, memory src1, immI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ exorl($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Xor Register with Memory
instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (XorI dst (LoadI src)));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9242,6 +10616,36 @@ instruct xorI_rReg_mem(rRegI dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct xorI_rReg_rReg_mem_ndd(rRegI dst, rRegI src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorI src1 (LoadI src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ exorl($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct xorI_rReg_mem_rReg_ndd(rRegI dst, memory src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorI (LoadI src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "exorl $dst, $src1, $src2\t# int ndd" %}
+ ins_encode %{
+ __ exorl($dst$$Register, $src1$$Address, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
// Xor Memory with Register
instruct xorB_mem_rReg(memory dst, rRegI src, rFlagsReg cr)
%{
@@ -9293,6 +10697,7 @@ instruct xorI_mem_imm(memory dst, immI src, rFlagsReg cr)
// And Register with Register
instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AndL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9304,6 +10709,22 @@ instruct andL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// And Register with Register using New Data Destination (NDD)
+instruct andL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eandq($dst$$Register, $src1$$Register, $src2$$Register, false);
+
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// And Register with Immediate 255
instruct andL_rReg_imm255(rRegL dst, rRegL src, immL_255 mask)
%{
@@ -9333,6 +10754,7 @@ instruct andL_rReg_imm65535(rRegL dst, rRegL src, immL_65535 mask)
// And Register with Immediate
instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AndL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9344,9 +10766,38 @@ instruct andL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct andL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eandq($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct andL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eandq($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// And Register with Memory
instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (AndL dst (LoadL src)));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9359,6 +10810,36 @@ instruct andL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct andL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndL src1 (LoadL src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eandq($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct andL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (AndL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "eandq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eandq($dst$$Register, $src1$$Address, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
// And Memory with Register
instruct andL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
%{
@@ -9534,6 +11015,7 @@ instruct blsrL_rReg_mem(rRegL dst, memory src, immL_M1 minus_1, rFlagsReg cr)
// Or Register with Register
instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (OrL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9545,6 +11027,22 @@ instruct orL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Or Register with Register using New Data Destination (NDD)
+instruct orL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
+
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Use any_RegP to match R15 (TLS register) without spilling.
instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
match(Set dst (OrL dst (CastP2X src)));
@@ -9558,10 +11056,22 @@ instruct orL_rReg_castP2X(rRegL dst, any_RegP src, rFlagsReg cr) %{
ins_pipe(ialu_reg_reg);
%}
+instruct orL_rReg_castP2X_ndd(rRegL dst, any_RegP src1, any_RegP src2, rFlagsReg cr) %{
+ match(Set dst (OrL src1 (CastP2X src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eorq($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
// Or Register with Immediate
instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (OrL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9573,9 +11083,53 @@ instruct orL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct orL_rReg_rReg_imm_ndd(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eorq($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+instruct orL_rReg_imm_rReg_ndd(rRegL dst, immL32 src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorq $dst, $src2, $src1\t# long ndd" %}
+ ins_encode %{
+ __ eorq($dst$$Register, $src2$$Register, $src1$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Or Memory with Immediate
+instruct orL_rReg_mem_imm_ndd(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eorq($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Or Register with Memory
instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (OrL dst (LoadL src)));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9588,6 +11142,21 @@ instruct orL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct orL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (OrL src1 (LoadL src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "eorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ eorq($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
// Or Memory with Register
instruct orL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
%{
@@ -9639,6 +11208,7 @@ instruct btsL_mem_imm(memory dst, immL_Pow2 con, rFlagsReg cr)
// Xor Register with Register
instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (XorL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9650,8 +11220,25 @@ instruct xorL_rReg(rRegL dst, rRegL src, rFlagsReg cr)
ins_pipe(ialu_reg_reg);
%}
+// Xor Register with Register using New Data Destination (NDD)
+instruct xorL_rReg_ndd(rRegL dst, rRegL src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ exorq($dst$$Register, $src1$$Register, $src2$$Register, false);
+ %}
+ ins_pipe(ialu_reg_reg);
+%}
+
// Xor Register with Immediate -1
-instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
+instruct xorL_rReg_im1(rRegL dst, immL_M1 imm)
+%{
+ predicate(!UseAPX);
match(Set dst (XorL dst imm));
format %{ "notq $dst" %}
@@ -9661,9 +11248,22 @@ instruct xorL_rReg_im1(rRegL dst, immL_M1 imm) %{
ins_pipe(ialu_reg);
%}
+instruct xorL_rReg_im1_ndd(rRegL dst,rRegL src, immL_M1 imm)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorL src imm));
+
+ format %{ "enotq $dst, $src" %}
+ ins_encode %{
+ __ enotq($dst$$Register, $src$$Register);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Xor Register with Immediate
instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (XorL dst src));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9675,9 +11275,39 @@ instruct xorL_rReg_imm(rRegL dst, immL32 src, rFlagsReg cr)
ins_pipe(ialu_reg);
%}
+instruct xorL_rReg_rReg_imm(rRegL dst, rRegL src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorL src1 src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ exorq($dst$$Register, $src1$$Register, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
+// Xor Memory with Immediate
+instruct xorL_rReg_mem_imm(rRegL dst, memory src1, immL32 src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ exorq($dst$$Register, $src1$$Address, $src2$$constant, false);
+ %}
+ ins_pipe(ialu_reg);
+%}
+
// Xor Register with Memory
instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
match(Set dst (XorL dst (LoadL src)));
effect(KILL cr);
flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
@@ -9690,6 +11320,36 @@ instruct xorL_rReg_mem(rRegL dst, memory src, rFlagsReg cr)
ins_pipe(ialu_reg_mem);
%}
+instruct xorL_rReg_rReg_mem_ndd(rRegL dst, rRegL src1, memory src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorL src1 (LoadL src2)));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ exorq($dst$$Register, $src1$$Register, $src2$$Address, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
+instruct xorL_rReg_mem_rReg_ndd(rRegL dst, memory src1, rRegL src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ match(Set dst (XorL (LoadL src1) src2));
+ effect(KILL cr);
+ flag(PD::Flag_sets_sign_flag, PD::Flag_sets_zero_flag, PD::Flag_sets_parity_flag, PD::Flag_clears_overflow_flag, PD::Flag_clears_carry_flag);
+
+ ins_cost(150);
+ format %{ "exorq $dst, $src1, $src2\t# long ndd" %}
+ ins_encode %{
+ __ exorq($dst$$Register, $src1$$Address, $src1$$Register, false);
+ %}
+ ins_pipe(ialu_reg_mem);
+%}
+
// Xor Memory with Register
instruct xorL_mem_rReg(memory dst, rRegL src, rFlagsReg cr)
%{
@@ -12003,6 +13663,7 @@ instruct testB_mem_imm(rFlagsReg cr, memory mem, immI8 imm, immI_0 zero)
instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
effect(USE_DEF dst, USE src, USE cr);
format %{ "cmovlgt $dst, $src\t# min" %}
@@ -12012,9 +13673,21 @@ instruct cmovI_reg_g(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovI_reg_g_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ effect(DEF dst, USE src1, USE src2, USE cr);
+
+ format %{ "ecmovlgt $dst, $src1, $src2\t# min ndd" %}
+ ins_encode %{
+ __ ecmovl(Assembler::greater, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
instruct minI_rReg(rRegI dst, rRegI src)
%{
+ predicate(!UseAPX);
match(Set dst (MinI dst src));
ins_cost(200);
@@ -12025,8 +13698,23 @@ instruct minI_rReg(rRegI dst, rRegI src)
%}
%}
+instruct minI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (MinI src1 src2));
+ effect(DEF dst, USE src1, USE src2);
+
+ ins_cost(200);
+ expand %{
+ rFlagsReg cr;
+ compI_rReg(cr, src1, src2);
+ cmovI_reg_g_ndd(dst, src1, src2, cr);
+ %}
+%}
+
instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
%{
+ predicate(!UseAPX);
effect(USE_DEF dst, USE src, USE cr);
format %{ "cmovllt $dst, $src\t# max" %}
@@ -12036,9 +13724,21 @@ instruct cmovI_reg_l(rRegI dst, rRegI src, rFlagsReg cr)
ins_pipe(pipe_cmov_reg);
%}
+instruct cmovI_reg_l_ndd(rRegI dst, rRegI src1, rRegI src2, rFlagsReg cr)
+%{
+ predicate(UseAPX);
+ effect(DEF dst, USE src1, USE src2, USE cr);
+
+ format %{ "ecmovllt $dst, $src1, $src2\t# max ndd" %}
+ ins_encode %{
+ __ ecmovl(Assembler::less, $dst$$Register, $src1$$Register, $src2$$Register);
+ %}
+ ins_pipe(pipe_cmov_reg);
+%}
instruct maxI_rReg(rRegI dst, rRegI src)
%{
+ predicate(!UseAPX);
match(Set dst (MaxI dst src));
ins_cost(200);
@@ -12049,6 +13749,20 @@ instruct maxI_rReg(rRegI dst, rRegI src)
%}
%}
+instruct maxI_rReg_ndd(rRegI dst, rRegI src1, rRegI src2)
+%{
+ predicate(UseAPX);
+ match(Set dst (MaxI src1 src2));
+ effect(DEF dst, USE src1, USE src2);
+
+ ins_cost(200);
+ expand %{
+ rFlagsReg cr;
+ compI_rReg(cr, src1, src2);
+ cmovI_reg_l_ndd(dst, src1, src2, cr);
+ %}
+%}
+
// ============================================================================
// Branch Instructions
diff --git a/src/hotspot/cpu/zero/vmStructs_zero.hpp b/src/hotspot/cpu/zero/vmStructs_zero.hpp
index 64a9300e25c..9100c765e7e 100644
--- a/src/hotspot/cpu/zero/vmStructs_zero.hpp
+++ b/src/hotspot/cpu/zero/vmStructs_zero.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
* Copyright 2007 Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -30,12 +30,12 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
+#define VM_STRUCTS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field)
-#define VM_TYPES_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
+#define VM_TYPES_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type)
-#define VM_INT_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_INT_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant)
-#define VM_LONG_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_LONG_CONSTANTS_CPU(declare_constant, declare_preprocessor_constant)
#endif // CPU_ZERO_VMSTRUCTS_ZERO_HPP
diff --git a/src/hotspot/os/aix/attachListener_aix.cpp b/src/hotspot/os/aix/attachListener_aix.cpp
index 218ee04fdcc..e5101814f97 100644
--- a/src/hotspot/os/aix/attachListener_aix.cpp
+++ b/src/hotspot/os/aix/attachListener_aix.cpp
@@ -73,16 +73,7 @@ class AixAttachListener: AllStatic {
static bool _atexit_registered;
- // reads a request from the given connected socket
- static AixAttachOperation* read_request(int s);
-
public:
- enum {
- ATTACH_PROTOCOL_VER = 1 // protocol version
- };
- enum {
- ATTACH_ERROR_BADVERSION = 101 // error codes
- };
static void set_path(char* path) {
if (path == nullptr) {
@@ -107,25 +98,65 @@ class AixAttachListener: AllStatic {
static void set_shutdown(bool shutdown) { _shutdown = shutdown; }
static bool is_shutdown() { return _shutdown; }
- // write the given buffer to a socket
- static int write_fully(int s, char* buf, size_t len);
-
static AixAttachOperation* dequeue();
};
+class SocketChannel : public AttachOperation::RequestReader, public AttachOperation::ReplyWriter {
+private:
+ int _socket;
+public:
+ SocketChannel(int socket) : _socket(socket) {}
+ ~SocketChannel() {
+ close();
+ }
+
+ bool opened() const {
+ return _socket != -1;
+ }
+
+ void close() {
+ if (opened()) {
+ // SHUT_RDWR is not available
+ ::shutdown(_socket, 2);
+ ::close(_socket);
+ _socket = -1;
+ }
+ }
+
+ // RequestReader
+ int read(void* buffer, int size) override {
+ ssize_t n;
+ RESTARTABLE(::read(_socket, buffer, (size_t)size), n);
+ return checked_cast(n);
+ }
+
+ // ReplyWriter
+ int write(const void* buffer, int size) override {
+ ssize_t n;
+ RESTARTABLE(::write(_socket, buffer, size), n);
+ return checked_cast(n);
+ }
+
+ void flush() override {
+ }
+};
+
class AixAttachOperation: public AttachOperation {
private:
// the connection to the client
- int _socket;
+ SocketChannel _socket_channel;
public:
- void complete(jint res, bufferedStream* st);
+ AixAttachOperation(int socket) : AttachOperation(), _socket_channel(socket) {}
- void set_socket(int s) { _socket = s; }
- int socket() const { return _socket; }
+ void complete(jint res, bufferedStream* st) override;
- AixAttachOperation(char* name) : AttachOperation(name) {
- set_socket(-1);
+ ReplyWriter* get_reply_writer() override {
+ return &_socket_channel;
+ }
+
+ bool read_request() {
+ return _socket_channel.read_request(this, &_socket_channel);
}
};
@@ -137,34 +168,6 @@ bool AixAttachListener::_atexit_registered = false;
// Shutdown marker to prevent accept blocking during clean-up
volatile bool AixAttachListener::_shutdown = false;
-// Supporting class to help split a buffer into individual components
-class ArgumentIterator : public StackObj {
- private:
- char* _pos;
- char* _end;
- public:
- ArgumentIterator(char* arg_buffer, size_t arg_size) {
- _pos = arg_buffer;
- _end = _pos + arg_size - 1;
- }
- char* next() {
- if (*_pos == '\0') {
- // advance the iterator if possible (null arguments)
- if (_pos < _end) {
- _pos += 1;
- }
- return nullptr;
- }
- char* res = _pos;
- char* next_pos = strchr(_pos, '\0');
- if (next_pos < _end) {
- next_pos++;
- }
- _pos = next_pos;
- return res;
- }
-};
-
// On AIX if sockets block until all data has been transmitted
// successfully in some communication domains a socket "close" may
// never complete. We have to take care that after the socket shutdown
@@ -258,106 +261,6 @@ int AixAttachListener::init() {
return 0;
}
-// Given a socket that is connected to a peer we read the request and
-// create an AttachOperation. As the socket is blocking there is potential
-// for a denial-of-service if the peer does not response. However this happens
-// after the peer credentials have been checked and in the worst case it just
-// means that the attach listener thread is blocked.
-//
-AixAttachOperation* AixAttachListener::read_request(int s) {
- char ver_str[8];
- os::snprintf_checked(ver_str, sizeof(ver_str), "%d", ATTACH_PROTOCOL_VER);
-
- // The request is a sequence of strings so we first figure out the
- // expected count and the maximum possible length of the request.
- // The request is:
- // 00000
- // where is the protocol version (1), is the command
- // name ("load", "datadump", ...), and is an argument
- int expected_str_count = 2 + AttachOperation::arg_count_max;
- const size_t max_len = (sizeof(ver_str) + 1) + (AttachOperation::name_length_max + 1) +
- AttachOperation::arg_count_max*(AttachOperation::arg_length_max + 1);
-
- char buf[max_len];
- int str_count = 0;
-
- // Read until all (expected) strings have been read, the buffer is
- // full, or EOF.
-
- size_t off = 0;
- size_t left = max_len;
-
- do {
- ssize_t n;
- // Don't block on interrupts because this will
- // hang in the clean-up when shutting down.
- n = read(s, buf+off, left);
- assert(n <= checked_cast(left), "buffer was too small, impossible!");
- buf[max_len - 1] = '\0';
- if (n == -1) {
- return nullptr; // reset by peer or other error
- }
- if (n == 0) {
- break;
- }
- for (int i=0; i so check it now to
- // check for protocol mismatch
- if (str_count == 1) {
- if ((strlen(buf) != strlen(ver_str)) ||
- (atoi(buf) != ATTACH_PROTOCOL_VER)) {
- char msg[32];
- os::snprintf_checked(msg, sizeof(msg), "%d\n", ATTACH_ERROR_BADVERSION);
- write_fully(s, msg, strlen(msg));
- return nullptr;
- }
- }
- }
- }
- off += n;
- left -= n;
- } while (left > 0 && str_count < expected_str_count);
-
- if (str_count != expected_str_count) {
- return nullptr; // incomplete request
- }
-
- // parse request
-
- ArgumentIterator args(buf, (max_len)-left);
-
- // version already checked
- char* v = args.next();
-
- char* name = args.next();
- if (name == nullptr || strlen(name) > AttachOperation::name_length_max) {
- return nullptr;
- }
-
- AixAttachOperation* op = new AixAttachOperation(name);
-
- for (int i=0; iset_arg(i, nullptr);
- } else {
- if (strlen(arg) > AttachOperation::arg_length_max) {
- delete op;
- return nullptr;
- }
- op->set_arg(i, arg);
- }
- }
-
- op->set_socket(s);
- return op;
-}
-
-
// Dequeue an operation
//
// In the Aix implementation there is only a single operation and clients
@@ -402,9 +305,9 @@ AixAttachOperation* AixAttachListener::dequeue() {
}
// peer credential look okay so we read the request
- AixAttachOperation* op = read_request(s);
- if (op == nullptr) {
- ::close(s);
+ AixAttachOperation* op = new AixAttachOperation(s);
+ if (!op->read_request()) {
+ delete op;
continue;
} else {
return op;
@@ -412,21 +315,6 @@ AixAttachOperation* AixAttachListener::dequeue() {
}
}
-// write the given buffer to the socket
-int AixAttachListener::write_fully(int s, char* buf, size_t len) {
- do {
- ssize_t n = ::write(s, buf, len);
- if (n == -1) {
- if (errno != EINTR) return -1;
- } else {
- buf += n;
- len -= n;
- }
- }
- while (len > 0);
- return 0;
-}
-
// Complete an operation by sending the operation result and any result
// output to the client. At this time the socket is in blocking mode so
// potentially we can block if there is a lot of data and the client is
@@ -436,24 +324,6 @@ int AixAttachListener::write_fully(int s, char* buf, size_t len) {
// socket could be made non-blocking and a timeout could be used.
void AixAttachOperation::complete(jint result, bufferedStream* st) {
- JavaThread* thread = JavaThread::current();
- ThreadBlockInVM tbivm(thread);
-
- // write operation result
- char msg[32];
- os::snprintf_checked(msg, sizeof(msg), "%d\n", result);
- int rc = AixAttachListener::write_fully(this->socket(), msg, strlen(msg));
-
- // write any result data
- if (rc == 0) {
- // Shutdown the socket in the cleanup function to enable more than
- // one agent attach in a sequence (see comments to listener_cleanup()).
- AixAttachListener::write_fully(this->socket(), (char*) st->base(), st->size());
- }
-
- // done
- ::close(this->socket());
-
delete this;
}
@@ -493,6 +363,7 @@ void AttachListener::vm_start() {
}
int AttachListener::pd_init() {
+ AttachListener::set_supported_version(ATTACH_API_V2);
JavaThread* thread = JavaThread::current();
ThreadBlockInVM tbivm(thread);
diff --git a/src/hotspot/os/aix/loadlib_aix.cpp b/src/hotspot/os/aix/loadlib_aix.cpp
index 2c38e1b637c..90a7271ad6d 100644
--- a/src/hotspot/os/aix/loadlib_aix.cpp
+++ b/src/hotspot/os/aix/loadlib_aix.cpp
@@ -42,6 +42,9 @@
// For loadquery()
#include
+// For getargs()
+#include
+
// Use raw malloc instead of os::malloc - this code gets used for error reporting.
// A class to "intern" eternal strings.
@@ -205,6 +208,22 @@ static bool reload_table() {
trcVerbose("loadquery buffer size is %zu.", buflen);
+ // the entry for the executable itself does not contain a path.
+ // instead we retrieve the path of the executable with the getargs API.
+ static char pgmpath[PATH_MAX+1] = "";
+ static char* pgmbase = nullptr;
+ if (pgmpath[0] == 0) {
+ procentry64 PInfo;
+ PInfo.pi_pid = ::getpid();
+ if (0 == ::getargs(&PInfo, sizeof(PInfo), (char*)pgmpath, PATH_MAX) && *pgmpath) {
+ pgmpath[PATH_MAX] = '\0';
+ pgmbase = strrchr(pgmpath, '/');
+ if (pgmbase != nullptr) {
+ pgmbase += 1;
+ }
+ }
+ }
+
// Iterate over the loadquery result. For details see sys/ldr.h on AIX.
ldi = (struct ld_info*) buffer;
@@ -223,7 +242,12 @@ static bool reload_table() {
lm->data = ldi->ldinfo_dataorg;
lm->data_len = ldi->ldinfo_datasize;
- lm->path = g_stringlist.add(ldi->ldinfo_filename);
+ if (pgmbase != nullptr && 0 == strcmp(pgmbase, ldi->ldinfo_filename)) {
+ lm->path = g_stringlist.add(pgmpath);
+ } else {
+ lm->path = g_stringlist.add(ldi->ldinfo_filename);
+ }
+
if (!lm->path) {
log_warning(os)("OOM.");
free(lm);
diff --git a/src/hotspot/os/aix/osThread_aix.cpp b/src/hotspot/os/aix/osThread_aix.cpp
index 204b271ceee..bbc7cf41e52 100644
--- a/src/hotspot/os/aix/osThread_aix.cpp
+++ b/src/hotspot/os/aix/osThread_aix.cpp
@@ -37,11 +37,9 @@ OSThread::OSThread()
_siginfo(nullptr),
_ucontext(nullptr),
_expanding_stack(0),
- _alt_sig_stack(nullptr),
- _startThread_lock(new Monitor(Mutex::event, "startThread_lock")) {
+ _alt_sig_stack(nullptr) {
sigemptyset(&_caller_sigmask);
}
OSThread::~OSThread() {
- delete _startThread_lock;
}
diff --git a/src/hotspot/os/aix/osThread_aix.hpp b/src/hotspot/os/aix/osThread_aix.hpp
index 771c2c19e45..81c0eafa0f7 100644
--- a/src/hotspot/os/aix/osThread_aix.hpp
+++ b/src/hotspot/os/aix/osThread_aix.hpp
@@ -114,15 +114,6 @@ class OSThread : public OSThreadBase {
void set_alt_sig_stack(address val) { _alt_sig_stack = val; }
address alt_sig_stack(void) { return _alt_sig_stack; }
- private:
- Monitor* _startThread_lock; // sync parent and child in thread creation
-
- public:
-
- Monitor* startThread_lock() const {
- return _startThread_lock;
- }
-
// Printing
uintx thread_id_for_printing() const override {
return (uintx)_thread_id;
diff --git a/src/hotspot/os/aix/os_aix.cpp b/src/hotspot/os/aix/os_aix.cpp
index e452bfdfd7c..aee15e4c55a 100644
--- a/src/hotspot/os/aix/os_aix.cpp
+++ b/src/hotspot/os/aix/os_aix.cpp
@@ -796,6 +796,8 @@ bool os::create_thread(Thread* thread, ThreadType thr_type,
// OSThread::thread_id is the pthread id.
osthread->set_thread_id(tid);
+ // child thread synchronization is not done here on AIX, a thread is started in suspended state
+
return true;
}
@@ -837,13 +839,6 @@ bool os::create_attached_thread(JavaThread* thread) {
thread->set_osthread(osthread);
- if (UseNUMA) {
- int lgrp_id = os::numa_get_group_id();
- if (lgrp_id != -1) {
- thread->set_lgrp_id(lgrp_id);
- }
- }
-
// initialize signal mask for this thread
// and save the caller's signal mask
PosixSignals::hotspot_sigmask(thread);
diff --git a/src/hotspot/os/aix/vmStructs_aix.hpp b/src/hotspot/os/aix/vmStructs_aix.hpp
index fb4b6409aaa..c79c9477fff 100644
--- a/src/hotspot/os/aix/vmStructs_aix.hpp
+++ b/src/hotspot/os/aix/vmStructs_aix.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,14 +29,14 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field) \
\
/******************************/ \
/* Threads (NOTE: incomplete) */ \
/******************************/ \
nonstatic_field(OSThread, _thread_id, pthread_t) \
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type) \
\
/**********************/ \
/* Posix Thread IDs */ \
@@ -44,9 +44,9 @@
\
declare_unsigned_integer_type(pthread_t)
-#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
-#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
#define VM_ADDRESSES_OS(declare_address, declare_preprocessor_address, declare_function)
diff --git a/src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp b/src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp
index 3cd9338f1d6..ac723483637 100644
--- a/src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp
+++ b/src/hotspot/os/bsd/gc/z/zNUMA_bsd.cpp
@@ -26,10 +26,7 @@
void ZNUMA::pd_initialize() {
_enabled = false;
-}
-
-uint32_t ZNUMA::count() {
- return 1;
+ _count = 1;
}
uint32_t ZNUMA::id() {
diff --git a/src/hotspot/os/bsd/os_bsd.cpp b/src/hotspot/os/bsd/os_bsd.cpp
index c538c54e86f..faa3efa2384 100644
--- a/src/hotspot/os/bsd/os_bsd.cpp
+++ b/src/hotspot/os/bsd/os_bsd.cpp
@@ -242,16 +242,6 @@ static char cpu_arch[] = "ppc";
#error Add appropriate cpu_arch setting
#endif
-// JVM variant
-#if defined(ZERO)
- #define JVM_VARIANT "zero"
-#elif defined(COMPILER2)
- #define JVM_VARIANT "server"
-#else
- #define JVM_VARIANT "client"
-#endif
-
-
void os::Bsd::initialize_system_info() {
int mib[2];
size_t len;
@@ -1558,7 +1548,7 @@ void os::jvm_path(char *buf, jint buflen) {
// Add the appropriate JVM variant subdir
len = strlen(buf);
jrelib_p = buf + len;
- snprintf(jrelib_p, buflen-len, "/%s", JVM_VARIANT);
+ snprintf(jrelib_p, buflen-len, "/%s", Abstract_VM_Version::vm_variant());
if (0 != access(buf, F_OK)) {
snprintf(jrelib_p, buflen-len, "%s", "");
}
diff --git a/src/hotspot/os/bsd/vmStructs_bsd.hpp b/src/hotspot/os/bsd/vmStructs_bsd.hpp
index b29067a9024..e07f1cd7dd0 100644
--- a/src/hotspot/os/bsd/vmStructs_bsd.hpp
+++ b/src/hotspot/os/bsd/vmStructs_bsd.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,7 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field) \
\
/******************************/ \
/* Threads (NOTE: incomplete) */ \
@@ -39,7 +39,7 @@
nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
nonstatic_field(OSThread, _unique_thread_id, uint64_t)
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type) \
\
/**********************/ \
/* Thread IDs */ \
@@ -47,9 +47,9 @@
\
declare_unsigned_integer_type(OSThread::thread_id_t)
-#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
-#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
#define VM_ADDRESSES_OS(declare_address, declare_preprocessor_address, declare_function) \
declare_preprocessor_address("RTLD_DEFAULT", RTLD_DEFAULT)
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
index bb51d4f3b8e..612cb9a9302 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.cpp
@@ -25,6 +25,7 @@
#include
#include
#include
+#include
#include "cgroupSubsystem_linux.hpp"
#include "cgroupV1Subsystem_linux.hpp"
#include "cgroupV2Subsystem_linux.hpp"
@@ -36,8 +37,26 @@
#include "runtime/os.hpp"
#include "utilities/globalDefinitions.hpp"
+// Inlined from for portability.
+#ifndef CGROUP2_SUPER_MAGIC
+# define CGROUP2_SUPER_MAGIC 0x63677270
+#endif
+
// controller names have to match the *_IDX indices
static const char* cg_controller_name[] = { "cpuset", "cpu", "cpuacct", "memory", "pids" };
+static inline int cg_v2_controller_index(const char* name) {
+ if (strcmp(name, "cpuset") == 0) {
+ return CPUSET_IDX;
+ } else if (strcmp(name, "cpu") == 0) {
+ return CPU_IDX;
+ } else if (strcmp(name, "memory") == 0) {
+ return MEMORY_IDX;
+ } else if (strcmp(name, "pids") == 0) {
+ return PIDS_IDX;
+ } else {
+ return -1;
+ }
+}
CgroupSubsystem* CgroupSubsystemFactory::create() {
CgroupV1MemoryController* memory = nullptr;
@@ -48,10 +67,25 @@ CgroupSubsystem* CgroupSubsystemFactory::create() {
CgroupInfo cg_infos[CG_INFO_LENGTH];
u1 cg_type_flags = INVALID_CGROUPS_GENERIC;
const char* proc_cgroups = "/proc/cgroups";
+ const char* sys_fs_cgroup_cgroup_controllers = "/sys/fs/cgroup/cgroup.controllers";
+ const char* controllers_file = proc_cgroups;
const char* proc_self_cgroup = "/proc/self/cgroup";
const char* proc_self_mountinfo = "/proc/self/mountinfo";
+ const char* sys_fs_cgroup = "/sys/fs/cgroup";
+ struct statfs fsstat = {};
+ bool cgroups_v2_enabled = false;
- bool valid_cgroup = determine_type(cg_infos, proc_cgroups, proc_self_cgroup, proc_self_mountinfo, &cg_type_flags);
+ // Assume cgroups v2 is usable by the JDK iff /sys/fs/cgroup has the cgroup v2
+ // file system magic. If it does not then heuristics are required to determine
+ // if cgroups v1 is usable or not.
+ if (statfs(sys_fs_cgroup, &fsstat) != -1) {
+ cgroups_v2_enabled = (fsstat.f_type == CGROUP2_SUPER_MAGIC);
+ if (cgroups_v2_enabled) {
+ controllers_file = sys_fs_cgroup_cgroup_controllers;
+ }
+ }
+
+ bool valid_cgroup = determine_type(cg_infos, cgroups_v2_enabled, controllers_file, proc_self_cgroup, proc_self_mountinfo, &cg_type_flags);
if (!valid_cgroup) {
// Could not detect cgroup type
@@ -216,84 +250,118 @@ static inline bool match_mount_info_line(char* line,
}
bool CgroupSubsystemFactory::determine_type(CgroupInfo* cg_infos,
- const char* proc_cgroups,
+ bool cgroups_v2_enabled,
+ const char* controllers_file,
const char* proc_self_cgroup,
const char* proc_self_mountinfo,
u1* flags) {
FILE *mntinfo = nullptr;
- FILE *cgroups = nullptr;
+ FILE* controllers = nullptr;
FILE *cgroup = nullptr;
char buf[MAXPATHLEN+1];
char *p;
- bool is_cgroupsV2;
// true iff all required controllers, memory, cpu, cpuacct are enabled
// at the kernel level.
// pids might not be enabled on older Linux distros (SLES 12.1, RHEL 7.1)
// cpuset might not be enabled on newer Linux distros (Fedora 41)
- bool all_required_controllers_enabled;
+ bool all_required_controllers_enabled = true;
- /*
- * Read /proc/cgroups so as to be able to distinguish cgroups v2 vs cgroups v1.
- *
- * For cgroups v1 hierarchy (hybrid or legacy), cpu, cpuacct, cpuset, memory controllers
- * must have non-zero for the hierarchy ID field and relevant controllers mounted.
- * Conversely, for cgroups v2 (unified hierarchy), cpu, cpuacct, cpuset, memory
- * controllers must have hierarchy ID 0 and the unified controller mounted.
- */
- cgroups = os::fopen(proc_cgroups, "r");
- if (cgroups == nullptr) {
- log_debug(os, container)("Can't open %s, %s", proc_cgroups, os::strerror(errno));
+ // If cgroups v2 is enabled, open /sys/fs/cgroup/cgroup.controllers. If not, open /proc/cgroups.
+ controllers = os::fopen(controllers_file, "r");
+ if (controllers == nullptr) {
+ log_debug(os, container)("Can't open %s, %s", controllers_file, os::strerror(errno));
*flags = INVALID_CGROUPS_GENERIC;
return false;
}
- while ((p = fgets(buf, MAXPATHLEN, cgroups)) != nullptr) {
- char name[MAXPATHLEN+1];
- int hierarchy_id;
- int enabled;
-
- // Format of /proc/cgroups documented via man 7 cgroups
- if (sscanf(p, "%s %d %*d %d", name, &hierarchy_id, &enabled) != 3) {
- continue;
+ if (cgroups_v2_enabled) {
+ /*
+ * cgroups v2 is enabled. For cgroups v2 (unified hierarchy), the cpu and memory
+ * controllers must be enabled.
+ */
+ if ((p = fgets(buf, MAXPATHLEN, controllers)) != nullptr) {
+ char* controller = nullptr;
+ #define ISSPACE_CHARS " \n\t\r\f\v"
+ while ((controller = strsep(&p, ISSPACE_CHARS)) != nullptr) {
+ int i;
+ if ((i = cg_v2_controller_index(controller)) != -1) {
+ cg_infos[i]._name = os::strdup(controller);
+ cg_infos[i]._enabled = true;
+ if (i == PIDS_IDX || i == CPUSET_IDX) {
+ log_debug(os, container)("Detected optional %s controller entry in %s",
+ controller, controllers_file);
+ }
+ }
+ }
+ #undef ISSPACE_CHARS
+ } else {
+ log_debug(os, container)("Can't read %s, %s", controllers_file, os::strerror(errno));
+ *flags = INVALID_CGROUPS_V2;
+ return false;
}
- if (strcmp(name, "memory") == 0) {
- cg_infos[MEMORY_IDX]._name = os::strdup(name);
- cg_infos[MEMORY_IDX]._hierarchy_id = hierarchy_id;
- cg_infos[MEMORY_IDX]._enabled = (enabled == 1);
- } else if (strcmp(name, "cpuset") == 0) {
- log_debug(os, container)("Detected optional cpuset controller entry in %s", proc_cgroups);
- cg_infos[CPUSET_IDX]._name = os::strdup(name);
- cg_infos[CPUSET_IDX]._hierarchy_id = hierarchy_id;
- cg_infos[CPUSET_IDX]._enabled = (enabled == 1);
- } else if (strcmp(name, "cpu") == 0) {
- cg_infos[CPU_IDX]._name = os::strdup(name);
- cg_infos[CPU_IDX]._hierarchy_id = hierarchy_id;
- cg_infos[CPU_IDX]._enabled = (enabled == 1);
- } else if (strcmp(name, "cpuacct") == 0) {
- cg_infos[CPUACCT_IDX]._name = os::strdup(name);
- cg_infos[CPUACCT_IDX]._hierarchy_id = hierarchy_id;
- cg_infos[CPUACCT_IDX]._enabled = (enabled == 1);
- } else if (strcmp(name, "pids") == 0) {
- log_debug(os, container)("Detected optional pids controller entry in %s", proc_cgroups);
- cg_infos[PIDS_IDX]._name = os::strdup(name);
- cg_infos[PIDS_IDX]._hierarchy_id = hierarchy_id;
- cg_infos[PIDS_IDX]._enabled = (enabled == 1);
- }
- }
- fclose(cgroups);
-
- is_cgroupsV2 = true;
- all_required_controllers_enabled = true;
- for (int i = 0; i < CG_INFO_LENGTH; i++) {
- // pids and cpuset controllers are optional. All other controllers are required
- if (i != PIDS_IDX && i != CPUSET_IDX) {
- is_cgroupsV2 = is_cgroupsV2 && cg_infos[i]._hierarchy_id == 0;
- all_required_controllers_enabled = all_required_controllers_enabled && cg_infos[i]._enabled;
- }
- if (log_is_enabled(Debug, os, container) && !cg_infos[i]._enabled) {
- log_debug(os, container)("controller %s is not enabled\n", cg_controller_name[i]);
+ for (int i = 0; i < CG_INFO_LENGTH; i++) {
+ // cgroups v2 does not have cpuacct.
+ if (i == CPUACCT_IDX) {
+ continue;
+ }
+ // For cgroups v2, cpuacct is rolled into cpu, and the pids and cpuset controllers
+ // are optional; the remaining controllers, cpu and memory, are required.
+ if (i == CPU_IDX || i == MEMORY_IDX) {
+ all_required_controllers_enabled = all_required_controllers_enabled && cg_infos[i]._enabled;
+ }
+ if (log_is_enabled(Debug, os, container) && !cg_infos[i]._enabled) {
+ log_debug(os, container)("controller %s is not enabled", cg_controller_name[i]);
+ }
+ }
+ } else {
+ /*
+ * The /sys/fs/cgroup filesystem magic hint suggests we have cg v1. Read /proc/cgroups; for
+ * cgroups v1 hierarchy (hybrid or legacy), cpu, cpuacct, cpuset, and memory controllers must
+ * have non-zero for the hierarchy ID field and relevant controllers mounted.
+ */
+ while ((p = fgets(buf, MAXPATHLEN, controllers)) != nullptr) {
+ char name[MAXPATHLEN+1];
+ int hierarchy_id;
+ int enabled;
+
+ // Format of /proc/cgroups documented via man 7 cgroups
+ if (sscanf(p, "%s %d %*d %d", name, &hierarchy_id, &enabled) != 3) {
+ continue;
+ }
+ if (strcmp(name, "memory") == 0) {
+ cg_infos[MEMORY_IDX]._name = os::strdup(name);
+ cg_infos[MEMORY_IDX]._hierarchy_id = hierarchy_id;
+ cg_infos[MEMORY_IDX]._enabled = (enabled == 1);
+ } else if (strcmp(name, "cpuset") == 0) {
+ cg_infos[CPUSET_IDX]._name = os::strdup(name);
+ cg_infos[CPUSET_IDX]._hierarchy_id = hierarchy_id;
+ cg_infos[CPUSET_IDX]._enabled = (enabled == 1);
+ } else if (strcmp(name, "cpu") == 0) {
+ cg_infos[CPU_IDX]._name = os::strdup(name);
+ cg_infos[CPU_IDX]._hierarchy_id = hierarchy_id;
+ cg_infos[CPU_IDX]._enabled = (enabled == 1);
+ } else if (strcmp(name, "cpuacct") == 0) {
+ cg_infos[CPUACCT_IDX]._name = os::strdup(name);
+ cg_infos[CPUACCT_IDX]._hierarchy_id = hierarchy_id;
+ cg_infos[CPUACCT_IDX]._enabled = (enabled == 1);
+ } else if (strcmp(name, "pids") == 0) {
+ log_debug(os, container)("Detected optional pids controller entry in %s", controllers_file);
+ cg_infos[PIDS_IDX]._name = os::strdup(name);
+ cg_infos[PIDS_IDX]._hierarchy_id = hierarchy_id;
+ cg_infos[PIDS_IDX]._enabled = (enabled == 1);
+ }
+ }
+ for (int i = 0; i < CG_INFO_LENGTH; i++) {
+ // pids controller is optional. All other controllers are required
+ if (i != PIDS_IDX) {
+ all_required_controllers_enabled = all_required_controllers_enabled && cg_infos[i]._enabled;
+ }
+ if (log_is_enabled(Debug, os, container) && !cg_infos[i]._enabled) {
+ log_debug(os, container)("controller %s is not enabled\n", cg_controller_name[i]);
+ }
}
}
+ fclose(controllers);
if (!all_required_controllers_enabled) {
// one or more required controllers disabled, disable container support
@@ -335,7 +403,7 @@ bool CgroupSubsystemFactory::determine_type(CgroupInfo* cg_infos,
continue;
}
- while (!is_cgroupsV2 && (token = strsep(&controllers, ",")) != nullptr) {
+ while (!cgroups_v2_enabled && (token = strsep(&controllers, ",")) != nullptr) {
if (strcmp(token, "memory") == 0) {
assert(hierarchy_id == cg_infos[MEMORY_IDX]._hierarchy_id, "/proc/cgroups and /proc/self/cgroup hierarchy mismatch for memory");
cg_infos[MEMORY_IDX]._cgroup_path = os::strdup(cgroup_path);
@@ -346,7 +414,7 @@ bool CgroupSubsystemFactory::determine_type(CgroupInfo* cg_infos,
assert(hierarchy_id == cg_infos[CPU_IDX]._hierarchy_id, "/proc/cgroups and /proc/self/cgroup hierarchy mismatch for cpu");
cg_infos[CPU_IDX]._cgroup_path = os::strdup(cgroup_path);
} else if (strcmp(token, "cpuacct") == 0) {
- assert(hierarchy_id == cg_infos[CPUACCT_IDX]._hierarchy_id, "/proc/cgroups and /proc/self/cgroup hierarchy mismatch for cpuacc");
+ assert(hierarchy_id == cg_infos[CPUACCT_IDX]._hierarchy_id, "/proc/cgroups and /proc/self/cgroup hierarchy mismatch for cpuacct");
cg_infos[CPUACCT_IDX]._cgroup_path = os::strdup(cgroup_path);
} else if (strcmp(token, "pids") == 0) {
assert(hierarchy_id == cg_infos[PIDS_IDX]._hierarchy_id, "/proc/cgroups (%d) and /proc/self/cgroup (%d) hierarchy mismatch for pids",
@@ -354,7 +422,7 @@ bool CgroupSubsystemFactory::determine_type(CgroupInfo* cg_infos,
cg_infos[PIDS_IDX]._cgroup_path = os::strdup(cgroup_path);
}
}
- if (is_cgroupsV2) {
+ if (cgroups_v2_enabled) {
// On some systems we have mixed cgroups v1 and cgroups v2 controllers (e.g. freezer on cg1 and
// all relevant controllers on cg2). Only set the cgroup path when we see a hierarchy id of 0.
if (hierarchy_id != 0) {
@@ -390,14 +458,14 @@ bool CgroupSubsystemFactory::determine_type(CgroupInfo* cg_infos,
char *cptr = tmpcgroups;
char *token;
- /* Cgroup v2 relevant info. We only look for the _mount_path iff is_cgroupsV2 so
+ /* Cgroup v2 relevant info. We only look for the _mount_path iff cgroups_v2_enabled so
* as to avoid memory stomping of the _mount_path pointer later on in the cgroup v1
* block in the hybrid case.
*
* We collect the read only mount option in the cgroup infos so as to have that
* info ready when determining is_containerized().
*/
- if (is_cgroupsV2 && match_mount_info_line(p,
+ if (cgroups_v2_enabled && match_mount_info_line(p,
tmproot,
tmpmount,
mount_opts,
@@ -476,7 +544,7 @@ bool CgroupSubsystemFactory::determine_type(CgroupInfo* cg_infos,
return false;
}
- if (is_cgroupsV2) {
+ if (cgroups_v2_enabled) {
if (!cgroupv2_mount_point_found) {
log_trace(os, container)("Mount point for cgroupv2 not found in /proc/self/mountinfo");
cleanup(cg_infos);
diff --git a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
index 3a1a93a2573..53b178397f8 100644
--- a/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
+++ b/src/hotspot/os/linux/cgroupSubsystem_linux.hpp
@@ -324,7 +324,8 @@ class CgroupSubsystemFactory: AllStatic {
// Determine the cgroup type (version 1 or version 2), given
// relevant paths to files. Sets 'flags' accordingly.
static bool determine_type(CgroupInfo* cg_infos,
- const char* proc_cgroups,
+ bool cgroups_v2_enabled,
+ const char* controllers_file,
const char* proc_self_cgroup,
const char* proc_self_mountinfo,
u1* flags);
diff --git a/src/hotspot/os/linux/cgroupUtil_linux.cpp b/src/hotspot/os/linux/cgroupUtil_linux.cpp
index bc0e018d6be..b52ef87dcae 100644
--- a/src/hotspot/os/linux/cgroupUtil_linux.cpp
+++ b/src/hotspot/os/linux/cgroupUtil_linux.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, Red Hat, Inc.
+ * Copyright (c) 2024, 2025, Red Hat, Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -49,12 +49,18 @@ int CgroupUtil::processor_count(CgroupCpuController* cpu_ctrl, int host_cpus) {
}
void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
+ assert(mem->cgroup_path() != nullptr, "invariant");
+ if (strstr(mem->cgroup_path(), "../") != nullptr) {
+ log_warning(os, container)("Cgroup memory controller path at '%s' seems to have moved to '%s', detected limits won't be accurate",
+ mem->mount_point(), mem->cgroup_path());
+ mem->set_subsystem_path("/");
+ return;
+ }
if (!mem->needs_hierarchy_adjustment()) {
// nothing to do
return;
}
log_trace(os, container)("Adjusting controller path for memory: %s", mem->subsystem_path());
- assert(mem->cgroup_path() != nullptr, "invariant");
char* orig = os::strdup(mem->cgroup_path());
char* cg_path = os::strdup(orig);
char* last_slash;
@@ -62,7 +68,8 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
julong phys_mem = os::Linux::physical_memory();
char* limit_cg_path = nullptr;
jlong limit = mem->read_memory_limit_in_bytes(phys_mem);
- jlong lowest_limit = phys_mem;
+ jlong lowest_limit = limit < 0 ? phys_mem : limit;
+ julong orig_limit = ((julong)lowest_limit) != phys_mem ? lowest_limit : phys_mem;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
// update to shortened path and try again
@@ -83,7 +90,7 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
limit_cg_path = os::strdup("/");
}
assert(lowest_limit >= 0, "limit must be positive");
- if ((julong)lowest_limit != phys_mem) {
+ if ((julong)lowest_limit != orig_limit) {
// we've found a lower limit anywhere in the hierarchy,
// set the path to the limit path
assert(limit_cg_path != nullptr, "limit path must be set");
@@ -93,6 +100,7 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
mem->subsystem_path(),
lowest_limit);
} else {
+ log_trace(os, container)("Lowest limit was: " JLONG_FORMAT, lowest_limit);
log_trace(os, container)("No lower limit found for memory in hierarchy %s, "
"adjusting to original path %s",
mem->mount_point(), orig);
@@ -104,19 +112,26 @@ void CgroupUtil::adjust_controller(CgroupMemoryController* mem) {
}
void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
+ assert(cpu->cgroup_path() != nullptr, "invariant");
+ if (strstr(cpu->cgroup_path(), "../") != nullptr) {
+ log_warning(os, container)("Cgroup cpu controller path at '%s' seems to have moved to '%s', detected limits won't be accurate",
+ cpu->mount_point(), cpu->cgroup_path());
+ cpu->set_subsystem_path("/");
+ return;
+ }
if (!cpu->needs_hierarchy_adjustment()) {
// nothing to do
return;
}
log_trace(os, container)("Adjusting controller path for cpu: %s", cpu->subsystem_path());
- assert(cpu->cgroup_path() != nullptr, "invariant");
char* orig = os::strdup(cpu->cgroup_path());
char* cg_path = os::strdup(orig);
char* last_slash;
assert(cg_path[0] == '/', "cgroup path must start with '/'");
int host_cpus = os::Linux::active_processor_count();
int cpus = CgroupUtil::processor_count(cpu, host_cpus);
- int lowest_limit = host_cpus;
+ int lowest_limit = cpus < host_cpus ? cpus: host_cpus;
+ int orig_limit = lowest_limit != host_cpus ? lowest_limit : host_cpus;
char* limit_cg_path = nullptr;
while ((last_slash = strrchr(cg_path, '/')) != cg_path) {
*last_slash = '\0'; // strip path
@@ -138,7 +153,7 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
limit_cg_path = os::strdup(cg_path);
}
assert(lowest_limit >= 0, "limit must be positive");
- if (lowest_limit != host_cpus) {
+ if (lowest_limit != orig_limit) {
// we've found a lower limit anywhere in the hierarchy,
// set the path to the limit path
assert(limit_cg_path != nullptr, "limit path must be set");
@@ -148,6 +163,7 @@ void CgroupUtil::adjust_controller(CgroupCpuController* cpu) {
cpu->subsystem_path(),
lowest_limit);
} else {
+ log_trace(os, container)("Lowest limit was: %d", lowest_limit);
log_trace(os, container)("No lower limit found for cpu in hierarchy %s, "
"adjusting to original path %s",
cpu->mount_point(), orig);
diff --git a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
index a6ac2822b25..8d9c3edb72a 100644
--- a/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV1Subsystem_linux.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,6 +37,47 @@
/*
* Set directory to subsystem specific files based
* on the contents of the mountinfo and cgroup files.
+ *
+ * The method determines whether it runs in
+ * - host mode
+ * - container mode
+ *
+ * In the host mode, _root is equal to "/" and
+ * the subsystem path is equal to the _mount_point path
+ * joined with cgroup_path.
+ *
+ * In the container mode, it can be two possibilities:
+ * - private namespace (cgroupns=private)
+ * - host namespace (cgroupns=host, default mode in cgroup V1 hosts)
+ *
+ * Private namespace is equivalent to the host mode, i.e.
+ * the subsystem path is set by concatenating
+ * _mount_point and cgroup_path.
+ *
+ * In the host namespace, _root is equal to host's cgroup path
+ * of the control group to which the containerized process
+ * belongs to at the moment of creation. The mountinfo and
+ * cgroup files are mirrored from the host, while the subsystem
+ * specific files are mapped directly at _mount_point, i.e.
+ * at /sys/fs/cgroup//, the subsystem path is
+ * then set equal to _mount_point.
+ *
+ * A special case of the subsystem path is when a cgroup path
+ * includes a subgroup, when a containerized process was associated
+ * with an existing cgroup, that is different from cgroup
+ * in which the process has been created.
+ * Here, the _root is equal to the host's initial cgroup path,
+ * cgroup_path will be equal to host's new cgroup path.
+ * As host cgroup hierarchies are not accessible in the container,
+ * it needs to be determined which part of cgroup path
+ * is accessible inside container, i.e. mapped under
+ * /sys/fs/cgroup//.
+ * In Docker default setup, host's cgroup path can be
+ * of the form: /docker//,
+ * from which only is mapped.
+ * The method trims cgroup path from left, until the subgroup
+ * component is found. The subsystem path will be set to
+ * the _mount_point joined with the subgroup path.
*/
void CgroupV1Controller::set_subsystem_path(const char* cgroup_path) {
if (_cgroup_path != nullptr) {
@@ -49,28 +90,36 @@ void CgroupV1Controller::set_subsystem_path(const char* cgroup_path) {
_cgroup_path = os::strdup(cgroup_path);
stringStream ss;
if (_root != nullptr && cgroup_path != nullptr) {
+ ss.print_raw(_mount_point);
if (strcmp(_root, "/") == 0) {
- ss.print_raw(_mount_point);
+ // host processes and containers with cgroupns=private
if (strcmp(cgroup_path,"/") != 0) {
ss.print_raw(cgroup_path);
}
- _path = os::strdup(ss.base());
} else {
- if (strcmp(_root, cgroup_path) == 0) {
- ss.print_raw(_mount_point);
- _path = os::strdup(ss.base());
- } else {
- char *p = strstr((char*)cgroup_path, _root);
- if (p != nullptr && p == _root) {
- if (strlen(cgroup_path) > strlen(_root)) {
- ss.print_raw(_mount_point);
- const char* cg_path_sub = cgroup_path + strlen(_root);
- ss.print_raw(cg_path_sub);
- _path = os::strdup(ss.base());
+ // containers with cgroupns=host, default setting is _root==cgroup_path
+ if (strcmp(_root, cgroup_path) != 0) {
+ if (*cgroup_path != '\0' && strcmp(cgroup_path, "/") != 0) {
+ // When moved to a subgroup, between subgroups, the path suffix will change.
+ const char *suffix = cgroup_path;
+ while (suffix != nullptr) {
+ stringStream pp;
+ pp.print_raw(_mount_point);
+ pp.print_raw(suffix);
+ if (os::file_exists(pp.base())) {
+ ss.print_raw(suffix);
+ if (suffix != cgroup_path) {
+ log_trace(os, container)("set_subsystem_path: cgroup v1 path reduced to: %s.", suffix);
+ }
+ break;
+ }
+ log_trace(os, container)("set_subsystem_path: skipped non-existent directory: %s.", suffix);
+ suffix = strchr(suffix + 1, '/');
}
}
}
}
+ _path = os::strdup(ss.base());
}
}
diff --git a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
index 62e8cac3a62..cbadbb9db02 100644
--- a/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
+++ b/src/hotspot/os/linux/cgroupV2Subsystem_linux.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2022, Red Hat Inc.
+ * Copyright (c) 2020, 2025, Red Hat Inc.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -292,6 +292,10 @@ jlong memory_swap_limit_value(CgroupV2Controller* ctrl) {
}
void CgroupV2Controller::set_subsystem_path(const char* cgroup_path) {
+ if (_cgroup_path != nullptr) {
+ os::free(_cgroup_path);
+ }
+ _cgroup_path = os::strdup(cgroup_path);
if (_path != nullptr) {
os::free(_path);
}
diff --git a/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp b/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp
index 3ae0c6ab719..5a5db428548 100644
--- a/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp
+++ b/src/hotspot/os/linux/gc/z/zNUMA_linux.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,15 +32,9 @@
void ZNUMA::pd_initialize() {
_enabled = UseNUMA;
-}
-
-uint32_t ZNUMA::count() {
- if (!_enabled) {
- // NUMA support not enabled
- return 1;
- }
-
- return os::Linux::numa_max_node() + 1;
+ _count = UseNUMA
+ ? os::Linux::numa_max_node() + 1
+ : 1;
}
uint32_t ZNUMA::id() {
@@ -65,7 +59,7 @@ uint32_t ZNUMA::memory_id(uintptr_t addr) {
fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string());
}
- assert(id < count(), "Invalid NUMA id");
+ assert(id < _count, "Invalid NUMA id");
return id;
}
diff --git a/src/hotspot/os/linux/os_linux.cpp b/src/hotspot/os/linux/os_linux.cpp
index 57b8a37baf2..35455b7b4ad 100644
--- a/src/hotspot/os/linux/os_linux.cpp
+++ b/src/hotspot/os/linux/os_linux.cpp
@@ -3775,10 +3775,9 @@ static bool hugetlbfs_sanity_check(size_t page_size) {
munmap(p, page_size);
return true;
} else {
- log_info(pagesize)("Large page size (%zu%s) failed sanity check, "
+ log_info(pagesize)("Large page size (" EXACTFMT ") failed sanity check, "
"checking if smaller large page sizes are usable",
- byte_size_in_exact_unit(page_size),
- exact_unit_for_byte_size(page_size));
+ EXACTFMTARGS(page_size));
for (size_t page_size_ = page_sizes.next_smaller(page_size);
page_size_ > os::vm_page_size();
page_size_ = page_sizes.next_smaller(page_size_)) {
@@ -3787,9 +3786,8 @@ static bool hugetlbfs_sanity_check(size_t page_size) {
if (p != MAP_FAILED) {
// Mapping succeeded, sanity check passed.
munmap(p, page_size_);
- log_info(pagesize)("Large page size (%zu%s) passed sanity check",
- byte_size_in_exact_unit(page_size_),
- exact_unit_for_byte_size(page_size_));
+ log_info(pagesize)("Large page size (" EXACTFMT ") passed sanity check",
+ EXACTFMTARGS(page_size_));
return true;
}
}
@@ -3986,26 +3984,21 @@ void os::Linux::large_page_init() {
LargePageSizeInBytes == 0 ||
LargePageSizeInBytes == default_large_page_size) {
large_page_size = default_large_page_size;
- log_info(pagesize)("Using the default large page size: %zu%s",
- byte_size_in_exact_unit(large_page_size),
- exact_unit_for_byte_size(large_page_size));
+ log_info(pagesize)("Using the default large page size: " EXACTFMT,
+ EXACTFMTARGS(large_page_size));
} else {
if (all_large_pages.contains(LargePageSizeInBytes)) {
large_page_size = LargePageSizeInBytes;
- log_info(pagesize)("Overriding default large page size (%zu%s) "
- "using LargePageSizeInBytes: %zu%s",
- byte_size_in_exact_unit(default_large_page_size),
- exact_unit_for_byte_size(default_large_page_size),
- byte_size_in_exact_unit(large_page_size),
- exact_unit_for_byte_size(large_page_size));
+ log_info(pagesize)("Overriding default large page size (" EXACTFMT ") "
+ "using LargePageSizeInBytes: " EXACTFMT,
+ EXACTFMTARGS(default_large_page_size),
+ EXACTFMTARGS(large_page_size));
} else {
large_page_size = default_large_page_size;
- log_info(pagesize)("LargePageSizeInBytes is not a valid large page size (%zu%s) "
- "using the default large page size: %zu%s",
- byte_size_in_exact_unit(LargePageSizeInBytes),
- exact_unit_for_byte_size(LargePageSizeInBytes),
- byte_size_in_exact_unit(large_page_size),
- exact_unit_for_byte_size(large_page_size));
+ log_info(pagesize)("LargePageSizeInBytes is not a valid large page size (" EXACTFMT ") "
+ "using the default large page size: " EXACTFMT,
+ EXACTFMTARGS(LargePageSizeInBytes),
+ EXACTFMTARGS(large_page_size));
}
}
@@ -4046,9 +4039,8 @@ static void log_on_commit_special_failure(char* req_addr, size_t bytes,
assert(error == ENOMEM, "Only expect to fail if no memory is available");
log_info(pagesize)("Failed to reserve and commit memory with given page size. req_addr: " PTR_FORMAT
- " size: %zu%s, page size: %zu%s, (errno = %d)",
- p2i(req_addr), byte_size_in_exact_unit(bytes), exact_unit_for_byte_size(bytes),
- byte_size_in_exact_unit(page_size), exact_unit_for_byte_size(page_size), error);
+ " size: " EXACTFMT ", page size: " EXACTFMT ", (errno = %d)",
+ p2i(req_addr), EXACTFMTARGS(bytes), EXACTFMTARGS(page_size), error);
}
static bool commit_memory_special(size_t bytes,
@@ -4075,11 +4067,8 @@ static bool commit_memory_special(size_t bytes,
return false;
}
- log_debug(pagesize)("Commit special mapping: " PTR_FORMAT ", size=%zu%s, page size=%zu%s",
- p2i(addr), byte_size_in_exact_unit(bytes),
- exact_unit_for_byte_size(bytes),
- byte_size_in_exact_unit(page_size),
- exact_unit_for_byte_size(page_size));
+ log_debug(pagesize)("Commit special mapping: " PTR_FORMAT ", size=" EXACTFMT ", page size=" EXACTFMT,
+ p2i(addr), EXACTFMTARGS(bytes), EXACTFMTARGS(page_size));
assert(is_aligned(addr, page_size), "Must be");
return true;
}
diff --git a/src/hotspot/os/linux/vmStructs_linux.hpp b/src/hotspot/os/linux/vmStructs_linux.hpp
index 2696a891f56..90bc6517a3f 100644
--- a/src/hotspot/os/linux/vmStructs_linux.hpp
+++ b/src/hotspot/os/linux/vmStructs_linux.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -31,7 +31,7 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field) \
\
/******************************/ \
/* Threads (NOTE: incomplete) */ \
@@ -39,7 +39,7 @@
nonstatic_field(OSThread, _thread_id, pid_t) \
nonstatic_field(OSThread, _pthread_id, pthread_t)
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type) \
\
/**********************/ \
/* Posix Thread IDs */ \
@@ -48,9 +48,9 @@
declare_integer_type(pid_t) \
declare_unsigned_integer_type(pthread_t)
-#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
-#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
#define VM_ADDRESSES_OS(declare_address, declare_preprocessor_address, declare_function) \
declare_preprocessor_address("RTLD_DEFAULT", RTLD_DEFAULT)
diff --git a/src/hotspot/os/posix/attachListener_posix.cpp b/src/hotspot/os/posix/attachListener_posix.cpp
index 27728d0ca1f..a4bc49c6bf3 100644
--- a/src/hotspot/os/posix/attachListener_posix.cpp
+++ b/src/hotspot/os/posix/attachListener_posix.cpp
@@ -114,6 +114,7 @@ public:
void close() {
if (opened()) {
+ ::shutdown(_socket, SHUT_RDWR);
::close(_socket);
_socket = -1;
}
@@ -132,9 +133,8 @@ public:
RESTARTABLE(::write(_socket, buffer, size), n);
return checked_cast(n);
}
- // called after writing all data
+
void flush() override {
- ::shutdown(_socket, SHUT_RDWR);
}
};
@@ -144,13 +144,16 @@ class PosixAttachOperation: public AttachOperation {
SocketChannel _socket_channel;
public:
+ PosixAttachOperation(int socket) : AttachOperation(), _socket_channel(socket) {}
+
void complete(jint res, bufferedStream* st) override;
- PosixAttachOperation(int socket) : AttachOperation(), _socket_channel(socket) {
+ ReplyWriter* get_reply_writer() override {
+ return &_socket_channel;
}
bool read_request() {
- return AttachOperation::read_request(&_socket_channel, &_socket_channel);
+ return _socket_channel.read_request(this, &_socket_channel);
}
};
@@ -318,11 +321,6 @@ PosixAttachOperation* PosixAttachListener::dequeue() {
// socket could be made non-blocking and a timeout could be used.
void PosixAttachOperation::complete(jint result, bufferedStream* st) {
- JavaThread* thread = JavaThread::current();
- ThreadBlockInVM tbivm(thread);
-
- write_reply(&_socket_channel, result, st);
-
delete this;
}
diff --git a/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp b/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp
index a177fe2b636..a103f764c98 100644
--- a/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp
+++ b/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp
@@ -32,7 +32,7 @@ void ZVirtualMemoryManager::pd_initialize_before_reserve() {
// Does nothing
}
-void ZVirtualMemoryManager::pd_initialize_after_reserve() {
+void ZVirtualMemoryManager::pd_register_callbacks(ZMemoryManager* manager) {
// Does nothing
}
diff --git a/src/hotspot/os/posix/os_posix.cpp b/src/hotspot/os/posix/os_posix.cpp
index 6f756fbf648..a36d6b87641 100644
--- a/src/hotspot/os/posix/os_posix.cpp
+++ b/src/hotspot/os/posix/os_posix.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -551,7 +551,7 @@ void os::Posix::print_uptime_info(outputStream* st) {
setutxent();
while ((ent = getutxent())) {
if (!strcmp("system boot", ent->ut_line)) {
- bootsec = ent->ut_tv.tv_sec;
+ bootsec = (int)ent->ut_tv.tv_sec;
break;
}
}
diff --git a/src/hotspot/os/posix/signals_posix.cpp b/src/hotspot/os/posix/signals_posix.cpp
index 2c0ab6732c1..555ac832aae 100644
--- a/src/hotspot/os/posix/signals_posix.cpp
+++ b/src/hotspot/os/posix/signals_posix.cpp
@@ -49,6 +49,13 @@
#include
+#define SEGV_BNDERR_value 3
+
+#if defined(SEGV_BNDERR)
+STATIC_ASSERT(SEGV_BNDERR == SEGV_BNDERR_value);
+#else
+#define SEGV_BNDERR SEGV_BNDERR_value
+#endif
static const char* get_signal_name(int sig, char* out, size_t outlen);
@@ -969,6 +976,9 @@ static bool get_signal_code_description(const siginfo_t* si, enum_sigcode_desc_t
{ SIGFPE, FPE_FLTSUB, "FPE_FLTSUB", "Subscript out of range." },
{ SIGSEGV, SEGV_MAPERR, "SEGV_MAPERR", "Address not mapped to object." },
{ SIGSEGV, SEGV_ACCERR, "SEGV_ACCERR", "Invalid permissions for mapped object." },
+#if defined(LINUX)
+ { SIGSEGV, SEGV_BNDERR, "SEGV_BNDERR", "Failed address bound checks." },
+#endif
#if defined(AIX)
// no explanation found what keyerr would be
{ SIGSEGV, SEGV_KEYERR, "SEGV_KEYERR", "key error" },
diff --git a/src/hotspot/os/windows/attachListener_windows.cpp b/src/hotspot/os/windows/attachListener_windows.cpp
index 4e6f39b8f81..2cc9d192ce5 100644
--- a/src/hotspot/os/windows/attachListener_windows.cpp
+++ b/src/hotspot/os/windows/attachListener_windows.cpp
@@ -92,6 +92,8 @@ public:
void close() {
if (opened()) {
+ ThreadBlockInVM tbivm(JavaThread::current());
+ FlushFileBuffers(_hPipe);
CloseHandle(_hPipe);
_hPipe = INVALID_HANDLE_VALUE;
}
@@ -123,15 +125,13 @@ public:
&written,
nullptr); // not overlapped
if (!fSuccess) {
- log_error(attach)("pipe write error (%d)", GetLastError());
- return -1;
+ log_error(attach)("pipe write error (%d)", GetLastError());
+ return -1;
}
return (int)written;
}
void flush() override {
- assert(opened(), "must be");
- FlushFileBuffers(_hPipe);
}
};
@@ -151,11 +151,15 @@ public:
}
bool read_request() {
- return AttachOperation::read_request(&_pipe, &_pipe);
+ return _pipe.read_request(this, &_pipe);
}
public:
void complete(jint result, bufferedStream* result_stream) override;
+
+ ReplyWriter* get_reply_writer() override {
+ return &_pipe;
+ }
};
@@ -432,11 +436,6 @@ Win32AttachOperation* Win32AttachListener::dequeue() {
}
void Win32AttachOperation::complete(jint result, bufferedStream* result_stream) {
- JavaThread* thread = JavaThread::current();
- ThreadBlockInVM tbivm(thread);
-
- write_reply(&_pipe, result, result_stream);
-
delete this;
}
diff --git a/src/hotspot/os/windows/gc/z/zMapper_windows.cpp b/src/hotspot/os/windows/gc/z/zMapper_windows.cpp
index eadabcbb030..d3d0a9766fd 100644
--- a/src/hotspot/os/windows/gc/z/zMapper_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zMapper_windows.cpp
@@ -78,7 +78,7 @@ void ZMapper::unreserve(zaddress_unsafe addr, size_t size) {
const bool res = ZSyscall::VirtualFreeEx(
GetCurrentProcess(), // hProcess
(void*)untype(addr), // lpAddress
- size, // dwSize
+ 0, // dwSize
MEM_RELEASE // dwFreeType
);
diff --git a/src/hotspot/os/windows/gc/z/zNUMA_windows.cpp b/src/hotspot/os/windows/gc/z/zNUMA_windows.cpp
index 8a93b66f389..afe8f18c392 100644
--- a/src/hotspot/os/windows/gc/z/zNUMA_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zNUMA_windows.cpp
@@ -25,10 +25,7 @@
void ZNUMA::pd_initialize() {
_enabled = false;
-}
-
-uint32_t ZNUMA::count() {
- return 1;
+ _count = 1;
}
uint32_t ZNUMA::id() {
diff --git a/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp b/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp
index 392b16e38a3..ac5be56a0c0 100644
--- a/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp
@@ -33,7 +33,7 @@
class ZVirtualMemoryManagerImpl : public CHeapObj {
public:
virtual void initialize_before_reserve() {}
- virtual void initialize_after_reserve(ZMemoryManager* manager) {}
+ virtual void register_callbacks(ZMemoryManager* manager) {}
virtual bool reserve(zaddress_unsafe addr, size_t size) = 0;
virtual void unreserve(zaddress_unsafe addr, size_t size) = 0;
};
@@ -47,7 +47,7 @@ public:
class ZVirtualMemoryManagerSmallPages : public ZVirtualMemoryManagerImpl {
private:
class PlaceholderCallbacks : public AllStatic {
- public:
+ private:
static void split_placeholder(zoffset start, size_t size) {
ZMapper::split_placeholder(ZOffset::address_unsafe(start), size);
}
@@ -79,99 +79,93 @@ private:
}
}
- // Called when a memory area is returned to the memory manager but can't
- // be merged with an already existing area. Make sure this area is covered
- // by a single placeholder.
- static void create_callback(const ZMemory* area) {
- assert(is_aligned(area->size(), ZGranuleSize), "Must be granule aligned");
+ // Callback implementations
- coalesce_into_one_placeholder(area->start(), area->size());
+ // Called when a memory area is going to be handed out to be used.
+ //
+ // Splits the memory area into granule-sized placeholders.
+ static void prepare_for_hand_out_callback(const ZMemory& area) {
+ assert(is_aligned(area.size(), ZGranuleSize), "Must be granule aligned");
+
+ split_into_granule_sized_placeholders(area.start(), area.size());
}
- // Called when a complete memory area in the memory manager is allocated.
- // Create granule sized placeholders for the entire area.
- static void destroy_callback(const ZMemory* area) {
- assert(is_aligned(area->size(), ZGranuleSize), "Must be granule aligned");
+ // Called when a memory area is handed back to the memory manager.
+ //
+ // Combines the granule-sized placeholders into one placeholder.
+ static void prepare_for_hand_back_callback(const ZMemory& area) {
+ assert(is_aligned(area.size(), ZGranuleSize), "Must be granule aligned");
- split_into_granule_sized_placeholders(area->start(), area->size());
+ coalesce_into_one_placeholder(area.start(), area.size());
}
- // Called when a memory area is allocated at the front of an exising memory area.
- // Turn the first part of the memory area into granule sized placeholders.
- static void shrink_from_front_callback(const ZMemory* area, size_t size) {
- assert(area->size() > size, "Must be larger than what we try to split out");
- assert(is_aligned(size, ZGranuleSize), "Must be granule aligned");
+ // Called when inserting a memory area and it can be merged with an
+ // existing, adjacent memory area.
+ //
+ // Coalesces the underlying placeholders into one.
+ static void grow_callback(const ZMemory& from, const ZMemory& to) {
+ assert(is_aligned(from.size(), ZGranuleSize), "Must be granule aligned");
+ assert(is_aligned(to.size(), ZGranuleSize), "Must be granule aligned");
+ assert(from != to, "Must have grown");
+ assert(to.contains(from), "Must be within");
+
+ coalesce_into_one_placeholder(to.start(), to.size());
+ }
+
+ // Called when a memory area is removed from the front or back of an existing
+ // memory area.
+ //
+ // Splits the memory into two placeholders.
+ static void shrink_callback(const ZMemory& from, const ZMemory& to) {
+ assert(is_aligned(from.size(), ZGranuleSize), "Must be granule aligned");
+ assert(is_aligned(to.size(), ZGranuleSize), "Must be granule aligned");
+ assert(from != to, "Must have shrunk");
+ assert(from.contains(to), "Must be larger than what we try to split out");
+ assert(from.start() == to.start() || from.end() == to.end(),
+ "Only verified to work if we split a placeholder into two placeholders");
// Split the area into two placeholders
- split_placeholder(area->start(), size);
-
- // Split the first part into granule sized placeholders
- split_into_granule_sized_placeholders(area->start(), size);
+ split_placeholder(to.start(), to.size());
}
- // Called when a memory area is allocated at the end of an existing memory area.
- // Turn the second part of the memory area into granule sized placeholders.
- static void shrink_from_back_callback(const ZMemory* area, size_t size) {
- assert(area->size() > size, "Must be larger than what we try to split out");
- assert(is_aligned(size, ZGranuleSize), "Must be granule aligned");
-
- // Split the area into two placeholders
- const zoffset start = to_zoffset(area->end() - size);
- split_placeholder(start, size);
-
- // Split the second part into granule sized placeholders
- split_into_granule_sized_placeholders(start, size);
- }
-
- // Called when freeing a memory area and it can be merged at the start of an
- // existing area. Coalesce the underlying placeholders into one.
- static void grow_from_front_callback(const ZMemory* area, size_t size) {
- assert(is_aligned(area->size(), ZGranuleSize), "Must be granule aligned");
-
- const zoffset start = area->start() - size;
- coalesce_into_one_placeholder(start, area->size() + size);
- }
-
- // Called when freeing a memory area and it can be merged at the end of an
- // existing area. Coalesce the underlying placeholders into one.
- static void grow_from_back_callback(const ZMemory* area, size_t size) {
- assert(is_aligned(area->size(), ZGranuleSize), "Must be granule aligned");
-
- coalesce_into_one_placeholder(area->start(), area->size() + size);
- }
-
- static void register_with(ZMemoryManager* manager) {
+ public:
+ static ZMemoryManager::Callbacks callbacks() {
// Each reserved virtual memory address area registered in _manager is
// exactly covered by a single placeholder. Callbacks are installed so
// that whenever a memory area changes, the corresponding placeholder
// is adjusted.
//
- // The create and grow callbacks are called when virtual memory is
- // returned to the memory manager. The new memory area is then covered
- // by a new single placeholder.
+ // The prepare_for_hand_out callback is called when virtual memory is
+ // handed out to callers. The memory area is split into granule-sized
+ // placeholders.
//
- // The destroy and shrink callbacks are called when virtual memory is
- // allocated from the memory manager. The memory area is then is split
- // into granule-sized placeholders.
+ // The prepare_for_hand_back callback is called when previously handed
+ // out virtual memory is handed back to the memory manager. The
+ // returned memory area is then covered by a new single placeholder.
+ //
+ // The grow callback is called when a virtual memory area grows. The
+ // resulting memory area is then covered by a single placeholder.
+ //
+ // The shrink callback is called when a virtual memory area is split into
+ // two parts. The two resulting memory areas are then covered by two
+ // separate placeholders.
//
// See comment in zMapper_windows.cpp explaining why placeholders are
// split into ZGranuleSize sized placeholders.
ZMemoryManager::Callbacks callbacks;
- callbacks._create = &create_callback;
- callbacks._destroy = &destroy_callback;
- callbacks._shrink_from_front = &shrink_from_front_callback;
- callbacks._shrink_from_back = &shrink_from_back_callback;
- callbacks._grow_from_front = &grow_from_front_callback;
- callbacks._grow_from_back = &grow_from_back_callback;
+ callbacks._prepare_for_hand_out = &prepare_for_hand_out_callback;
+ callbacks._prepare_for_hand_back = &prepare_for_hand_back_callback;
+ callbacks._grow = &grow_callback;
+ callbacks._shrink = &shrink_callback;
- manager->register_callbacks(callbacks);
+ return callbacks;
}
};
- virtual void initialize_after_reserve(ZMemoryManager* manager) {
- PlaceholderCallbacks::register_with(manager);
+ virtual void register_callbacks(ZMemoryManager* manager) {
+ manager->register_callbacks(PlaceholderCallbacks::callbacks());
}
virtual bool reserve(zaddress_unsafe addr, size_t size) {
@@ -220,8 +214,8 @@ void ZVirtualMemoryManager::pd_initialize_before_reserve() {
_impl->initialize_before_reserve();
}
-void ZVirtualMemoryManager::pd_initialize_after_reserve() {
- _impl->initialize_after_reserve(&_manager);
+void ZVirtualMemoryManager::pd_register_callbacks(ZMemoryManager* manager) {
+ _impl->register_callbacks(manager);
}
bool ZVirtualMemoryManager::pd_reserve(zaddress_unsafe addr, size_t size) {
diff --git a/src/hotspot/os/windows/os_windows.cpp b/src/hotspot/os/windows/os_windows.cpp
index 7ad9f80141c..ec03a45594c 100644
--- a/src/hotspot/os/windows/os_windows.cpp
+++ b/src/hotspot/os/windows/os_windows.cpp
@@ -2600,6 +2600,7 @@ static inline void report_error(Thread* t, DWORD exception_code,
//-----------------------------------------------------------------------------
JNIEXPORT
LONG WINAPI topLevelExceptionFilter(struct _EXCEPTION_POINTERS* exceptionInfo) {
+ PreserveLastError ple;
if (InterceptOSException) return EXCEPTION_CONTINUE_SEARCH;
PEXCEPTION_RECORD exception_record = exceptionInfo->ExceptionRecord;
DWORD exception_code = exception_record->ExceptionCode;
@@ -3409,8 +3410,8 @@ static char* find_aligned_address(size_t size, size_t alignment) {
}
static char* reserve_large_pages_aligned(size_t size, size_t alignment, bool exec) {
- log_debug(pagesize)("Reserving large pages at an aligned address, alignment=%zu%s",
- byte_size_in_exact_unit(alignment), exact_unit_for_byte_size(alignment));
+ log_debug(pagesize)("Reserving large pages at an aligned address, alignment=" EXACTFMT,
+ EXACTFMTARGS(alignment));
// Will try to find a suitable address at most 20 times. The reason we need to try
// multiple times is that between finding the aligned address and trying to commit
diff --git a/src/hotspot/os/windows/systemMemoryBarrier_windows.cpp b/src/hotspot/os/windows/systemMemoryBarrier_windows.cpp
index 03811fcd64b..6156469ff24 100644
--- a/src/hotspot/os/windows/systemMemoryBarrier_windows.cpp
+++ b/src/hotspot/os/windows/systemMemoryBarrier_windows.cpp
@@ -24,7 +24,7 @@
#include "systemMemoryBarrier_windows.hpp"
-#include
+#include // do not reorder
#include
bool WindowsSystemMemoryBarrier::initialize() {
diff --git a/src/hotspot/os/windows/vmStructs_windows.hpp b/src/hotspot/os/windows/vmStructs_windows.hpp
index 26ad17bb166..7d457c82fe8 100644
--- a/src/hotspot/os/windows/vmStructs_windows.hpp
+++ b/src/hotspot/os/windows/vmStructs_windows.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2015, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,7 @@
// constants required by the Serviceability Agent. This file is
// referenced by vmStructs.cpp.
-#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field) \
+#define VM_STRUCTS_OS(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field) \
\
/******************************/ \
/* Threads (NOTE: incomplete) */ \
@@ -38,13 +38,13 @@
nonstatic_field(OSThread, _thread_id, OSThread::thread_id_t) \
unchecked_nonstatic_field(OSThread, _thread_handle, sizeof(HANDLE)) /* NOTE: no type */
-#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type) \
+#define VM_TYPES_OS(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type) \
\
declare_unsigned_integer_type(OSThread::thread_id_t)
-#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_INT_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
-#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
+#define VM_LONG_CONSTANTS_OS(declare_constant, declare_preprocessor_constant)
#define VM_ADDRESSES_OS(declare_address, declare_preprocessor_address, declare_function)
diff --git a/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp b/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp
index 7cd57b65d32..59bbd5db3a0 100644
--- a/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp
+++ b/src/hotspot/os_cpu/aix_ppc/javaThread_aix_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* Copyright (c) 2022, IBM Corp.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
@@ -27,6 +27,7 @@
#include "memory/metaspace.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/javaThread.hpp"
+#include "runtime/os.inline.hpp"
frame JavaThread::pd_last_frame() {
assert(has_last_Java_frame(), "must have last_Java_sp() when suspended");
@@ -47,9 +48,17 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
if (has_last_Java_frame() && frame_anchor()->walkable()) {
intptr_t* sp = last_Java_sp();
address pc = _anchor.last_Java_pc();
- // pc can be seen as null because not all writers use store pc + release store sp.
- // Simply discard the sample in this very rare case.
- if (pc == nullptr) return false;
+ if (pc == nullptr) {
+ // This is not uncommon. Many c1/c2 runtime stubs do not set the pc in the anchor.
+ intptr_t* top_sp = os::Aix::ucontext_get_sp((const ucontext_t*)ucontext);
+ if ((uint64_t)sp <= ((frame::common_abi*)top_sp)->callers_sp) {
+ // The interrupt occurred either in the last java frame or in its direct callee.
+ // We cannot be sure that the link register LR was already saved to the
+ // java frame. Therefore we discard this sample.
+ return false;
+ }
+ // The last java pc will be found in the abi part of the last java frame.
+ }
*fr_addr = frame(sp, pc, frame::kind::code_blob);
return true;
}
diff --git a/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp b/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp
deleted file mode 100644
index d09b0125f25..00000000000
--- a/src/hotspot/os_cpu/aix_ppc/vmStructs_aix_ppc.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_AIX_PPC_VMSTRUCTS_AIX_PPC_HPP
-#define OS_CPU_AIX_PPC_VMSTRUCTS_AIX_PPC_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_AIX_PPC_VMSTRUCTS_AIX_PPC_HPP
diff --git a/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp b/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp
deleted file mode 100644
index 24d5c0f4dc6..00000000000
--- a/src/hotspot/os_cpu/bsd_aarch64/vmStructs_bsd_aarch64.hpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_BSD_AARCH64_VMSTRUCTS_BSD_AARCH64_HPP
-#define OS_CPU_BSD_AARCH64_VMSTRUCTS_BSD_AARCH64_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_BSD_AARCH64_VMSTRUCTS_BSD_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S b/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
deleted file mode 100644
index 7d8892bcd87..00000000000
--- a/src/hotspot/os_cpu/bsd_x86/bsd_x86_32.S
+++ /dev/null
@@ -1,525 +0,0 @@
-#
-# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-#include "defs.S.inc"
-
- # NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
-
- .text
-
-# Set fpu to 53 bit precision. This happens too early to use a stub.
- .p2align 4,,15
-DECLARE_FUNC(fixcw):
- pushl $0x27f
- fldcw 0(%esp)
- popl %eax
- ret
-
- .p2align 4,,15
-DECLARE_FUNC(SpinPause):
- rep
- nop
- movl $1, %eax
- ret
-
- # Support for void Copy::arrayof_conjoint_bytes(void* from,
- # void* to,
- # size_t count)
- #
- .p2align 4,,15
-DECLARE_FUNC(_Copy_arrayof_conjoint_bytes):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -1(%esi,%ecx),%eax # from + count - 1
- jbe acb_CopyRight
- cmpl %eax,%edi
- jbe acb_CopyLeft
- # copy from low to high
-acb_CopyRight:
- cmpl $3,%ecx
- jbe 5f
-1: movl %ecx,%eax
- shrl $2,%ecx
- jz 4f
- cmpl $32,%ecx
- ja 3f
- # copy aligned dwords
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
- # copy aligned dwords
-3: rep; smovl
-4: movl %eax,%ecx
-5: andl $3,%ecx
- jz 7f
- # copy suffix
- xorl %eax,%eax
-6: movb (%esi,%eax,1),%dl
- movb %dl,(%edi,%eax,1)
- addl $1,%eax
- subl $1,%ecx
- jnz 6b
-7: popl %edi
- popl %esi
- ret
-acb_CopyLeft:
- std
- leal -4(%edi,%ecx),%edi # to + count - 4
- movl %eax,%esi # from + count - 1
- movl %ecx,%eax
- subl $3,%esi # from + count - 4
- cmpl $3,%ecx
- jbe 5f
-1: shrl $2,%ecx
- jz 4f
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- rep; smovl
- jmp 4f
- .space 8
-2: subl %esi,%edi
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 3b
- addl %esi,%edi
-4: movl %eax,%ecx
-5: andl $3,%ecx
- jz 7f
- subl %esi,%edi
- addl $3,%esi
-6: movb (%esi),%dl
- movb %dl,(%edi,%esi,1)
- subl $1,%esi
- subl $1,%ecx
- jnz 6b
-7: cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::conjoint_jshorts_atomic(void* from,
- # void* to,
- # size_t count)
- .p2align 4,,15
-DECLARE_FUNC(_Copy_conjoint_jshorts_atomic):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -2(%esi,%ecx,2),%eax # from + count*2 - 2
- jbe cs_CopyRight
- cmpl %eax,%edi
- jbe cs_CopyLeft
- # copy from low to high
-cs_CopyRight:
- # align source address at dword address boundary
- movl %esi,%eax # original from
- andl $3,%eax # either 0 or 2
- jz 1f # no prefix
- # copy prefix
- subl $1,%ecx
- jl 5f # zero count
- movw (%esi),%dx
- movw %dx,(%edi)
- addl %eax,%esi # %eax == 2
- addl %eax,%edi
-1: movl %ecx,%eax # word count less prefix
- sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- # copy aligned dwords
- rep; smovl
- jmp 4f
- # copy aligned dwords
-2: subl %esi,%edi
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 3b
- addl %esi,%edi
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- movw (%esi),%dx
- movw %dx,(%edi)
-5: popl %edi
- popl %esi
- ret
- # copy from high to low
-cs_CopyLeft:
- std
- leal -4(%edi,%ecx,2),%edi # to + count*2 - 4
- movl %eax,%esi # from + count*2 - 2
- movl %ecx,%eax
- subl $2,%esi # from + count*2 - 4
-1: sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- ja 3f # > 32 dwords
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
-3: rep; smovl
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- addl $2,%esi
- addl $2,%edi
- movw (%esi),%dx
- movw %dx,(%edi)
-5: cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::arrayof_conjoint_jshorts(void* from,
- # void* to,
- # size_t count)
- .p2align 4,,15
-DECLARE_FUNC(_Copy_arrayof_conjoint_jshorts):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -2(%esi,%ecx,2),%eax # from + count*2 - 2
- jbe acs_CopyRight
- cmpl %eax,%edi
- jbe acs_CopyLeft
-acs_CopyRight:
- movl %ecx,%eax # word count
- sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- # copy aligned dwords
- rep; smovl
- jmp 4f
- # copy aligned dwords
- .space 5
-2: subl %esi,%edi
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 3b
- addl %esi,%edi
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- movw (%esi),%dx
- movw %dx,(%edi)
-5: popl %edi
- popl %esi
- ret
-acs_CopyLeft:
- std
- leal -4(%edi,%ecx,2),%edi # to + count*2 - 4
- movl %eax,%esi # from + count*2 - 2
- movl %ecx,%eax
- subl $2,%esi # from + count*2 - 4
- sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- ja 3f # > 32 dwords
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
-3: rep; smovl
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- addl $2,%esi
- addl $2,%edi
- movw (%esi),%dx
- movw %dx,(%edi)
-5: cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::conjoint_jints_atomic(void* from,
- # void* to,
- # size_t count)
- # Equivalent to
- # arrayof_conjoint_jints
- .p2align 4,,15
-DECLARE_FUNC(_Copy_conjoint_jints_atomic):
-DECLARE_FUNC(_Copy_arrayof_conjoint_jints):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -4(%esi,%ecx,4),%eax # from + count*4 - 4
- jbe ci_CopyRight
- cmpl %eax,%edi
- jbe ci_CopyLeft
-ci_CopyRight:
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- rep; smovl
- popl %edi
- popl %esi
- ret
- .space 10
-2: subl %esi,%edi
- jmp 4f
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
-4: subl $1,%ecx
- jge 3b
- popl %edi
- popl %esi
- ret
-ci_CopyLeft:
- std
- leal -4(%edi,%ecx,4),%edi # to + count*4 - 4
- cmpl $32,%ecx
- ja 4f # > 32 dwords
- subl %eax,%edi # eax == from + count*4 - 4
- jmp 3f
- .p2align 4,,15
-2: movl (%eax),%edx
- movl %edx,(%edi,%eax,1)
- subl $4,%eax
-3: subl $1,%ecx
- jge 2b
- cld
- popl %edi
- popl %esi
- ret
-4: movl %eax,%esi # from + count*4 - 4
- rep; smovl
- cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::conjoint_jlongs_atomic(jlong* from,
- # jlong* to,
- # size_t count)
- #
- # 32-bit
- #
- # count treated as signed
- #
- # // if (from > to) {
- # while (--count >= 0) {
- # *to++ = *from++;
- # }
- # } else {
- # while (--count >= 0) {
- # to[count] = from[count];
- # }
- # }
- .p2align 4,,15
-DECLARE_FUNC(_Copy_conjoint_jlongs_atomic):
- movl 4+8(%esp),%ecx # count
- movl 4+0(%esp),%eax # from
- movl 4+4(%esp),%edx # to
- cmpl %eax,%edx
- jae cla_CopyLeft
-cla_CopyRight:
- subl %eax,%edx
- jmp 2f
- .p2align 4,,15
-1: fildll (%eax)
- fistpll (%edx,%eax,1)
- addl $8,%eax
-2: subl $1,%ecx
- jge 1b
- ret
- .p2align 4,,15
-3: fildll (%eax,%ecx,8)
- fistpll (%edx,%ecx,8)
-cla_CopyLeft:
- subl $1,%ecx
- jge 3b
- ret
-
- # Support for void Copy::arrayof_conjoint_jshorts(void* from,
- # void* to,
- # size_t count)
- .p2align 4,,15
-DECLARE_FUNC(_mmx_Copy_arrayof_conjoint_jshorts):
- pushl %esi
- movl 4+12(%esp),%ecx
- pushl %edi
- movl 8+ 4(%esp),%esi
- movl 8+ 8(%esp),%edi
- cmpl %esi,%edi
- leal -2(%esi,%ecx,2),%eax
- jbe mmx_acs_CopyRight
- cmpl %eax,%edi
- jbe mmx_acs_CopyLeft
-mmx_acs_CopyRight:
- movl %ecx,%eax
- sarl %ecx
- je 5f
- cmpl $33,%ecx
- jae 3f
-1: subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 5f
-3: smovl # align to 8 bytes, we know we are 4 byte aligned to start
- subl $1,%ecx
-4: .p2align 4,,15
- movq 0(%esi),%mm0
- addl $64,%edi
- movq 8(%esi),%mm1
- subl $16,%ecx
- movq 16(%esi),%mm2
- movq %mm0,-64(%edi)
- movq 24(%esi),%mm0
- movq %mm1,-56(%edi)
- movq 32(%esi),%mm1
- movq %mm2,-48(%edi)
- movq 40(%esi),%mm2
- movq %mm0,-40(%edi)
- movq 48(%esi),%mm0
- movq %mm1,-32(%edi)
- movq 56(%esi),%mm1
- movq %mm2,-24(%edi)
- movq %mm0,-16(%edi)
- addl $64,%esi
- movq %mm1,-8(%edi)
- cmpl $16,%ecx
- jge 4b
- emms
- testl %ecx,%ecx
- ja 1b
-5: andl $1,%eax
- je 7f
-6: movw (%esi),%dx
- movw %dx,(%edi)
-7: popl %edi
- popl %esi
- ret
-mmx_acs_CopyLeft:
- std
- leal -4(%edi,%ecx,2),%edi
- movl %eax,%esi
- movl %ecx,%eax
- subl $2,%esi
- sarl %ecx
- je 4f
- cmpl $32,%ecx
- ja 3f
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
-3: rep; smovl
-4: andl $1,%eax
- je 6f
- addl $2,%esi
- addl $2,%edi
-5: movw (%esi),%dx
- movw %dx,(%edi)
-6: cld
- popl %edi
- popl %esi
- ret
-
-
- # Support for int64_t Atomic::cmpxchg(int64_t compare_value,
- # volatile int64_t* dest,
- # int64_t exchange_value)
- #
- .p2align 4,,15
-DECLARE_FUNC(_Atomic_cmpxchg_long):
- # 8(%esp) : return PC
- pushl %ebx # 4(%esp) : old %ebx
- pushl %edi # 0(%esp) : old %edi
- movl 12(%esp), %ebx # 12(%esp) : exchange_value (low)
- movl 16(%esp), %ecx # 16(%esp) : exchange_value (high)
- movl 24(%esp), %eax # 24(%esp) : compare_value (low)
- movl 28(%esp), %edx # 28(%esp) : compare_value (high)
- movl 20(%esp), %edi # 20(%esp) : dest
- lock
- cmpxchg8b (%edi)
- popl %edi
- popl %ebx
- ret
-
-
- # Support for int64_t Atomic::load and Atomic::store.
- # void _Atomic_move_long(const volatile int64_t* src, volatile int64_t* dst)
- .p2align 4,,15
-DECLARE_FUNC(_Atomic_move_long):
- movl 4(%esp), %eax # src
- fildll (%eax)
- movl 8(%esp), %eax # dest
- fistpll (%eax)
- ret
diff --git a/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp b/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp
deleted file mode 100644
index 4f6c4414f5f..00000000000
--- a/src/hotspot/os_cpu/bsd_x86/vmStructs_bsd_x86.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_BSD_X86_VMSTRUCTS_BSD_X86_HPP
-#define OS_CPU_BSD_X86_VMSTRUCTS_BSD_X86_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_BSD_X86_VMSTRUCTS_BSD_X86_HPP
diff --git a/src/hotspot/os_cpu/bsd_zero/vmStructs_bsd_zero.hpp b/src/hotspot/os_cpu/bsd_zero/vmStructs_bsd_zero.hpp
deleted file mode 100644
index ea3771b7ce6..00000000000
--- a/src/hotspot/os_cpu/bsd_zero/vmStructs_bsd_zero.hpp
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2007 Red Hat, Inc.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_BSD_ZERO_VMSTRUCTS_BSD_ZERO_HPP
-#define OS_CPU_BSD_ZERO_VMSTRUCTS_BSD_ZERO_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_BSD_ZERO_VMSTRUCTS_BSD_ZERO_HPP
diff --git a/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp
deleted file mode 100644
index 6cc6d73be77..00000000000
--- a/src/hotspot/os_cpu/linux_aarch64/vmStructs_linux_aarch64.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_AARCH64_VMSTRUCTS_LINUX_AARCH64_HPP
-#define OS_CPU_LINUX_AARCH64_VMSTRUCTS_LINUX_AARCH64_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_AARCH64_VMSTRUCTS_LINUX_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp b/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp
deleted file mode 100644
index a1b73bdee8f..00000000000
--- a/src/hotspot/os_cpu/linux_arm/vmStructs_linux_arm.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2008, 2024, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_ARM_VMSTRUCTS_LINUX_ARM_HPP
-#define OS_CPU_LINUX_ARM_VMSTRUCTS_LINUX_ARM_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_ARM_VMSTRUCTS_LINUX_ARM_HPP
diff --git a/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp b/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp
index a1c3d616eea..967b575bb52 100644
--- a/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp
+++ b/src/hotspot/os_cpu/linux_ppc/javaThread_linux_ppc.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2024 SAP SE. All rights reserved.
+ * Copyright (c) 2012, 2025 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
*/
#include "memory/metaspace.hpp"
+#include "os_linux.hpp"
#include "runtime/frame.inline.hpp"
#include "runtime/javaThread.hpp"
@@ -46,9 +47,17 @@ bool JavaThread::pd_get_top_frame_for_profiling(frame* fr_addr, void* ucontext,
if (has_last_Java_frame() && frame_anchor()->walkable()) {
intptr_t* sp = last_Java_sp();
address pc = _anchor.last_Java_pc();
- // pc can be seen as null because not all writers use store pc + release store sp.
- // Simply discard the sample in this very rare case.
- if (pc == nullptr) return false;
+ if (pc == nullptr) {
+ // This is not uncommon. Many c1/c2 runtime stubs do not set the pc in the anchor.
+ intptr_t* top_sp = os::Linux::ucontext_get_sp((const ucontext_t*)ucontext);
+ if ((uint64_t)sp <= ((frame::common_abi*)top_sp)->callers_sp) {
+ // The interrupt occurred either in the last java frame or in its direct callee.
+ // We cannot be sure that the link register LR was already saved to the
+ // java frame. Therefore we discard this sample.
+ return false;
+ }
+ // The last java pc will be found in the abi part of the last java frame.
+ }
*fr_addr = frame(sp, pc, frame::kind::code_blob);
return true;
}
diff --git a/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp b/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp
deleted file mode 100644
index 46288cdeaab..00000000000
--- a/src/hotspot/os_cpu/linux_ppc/vmStructs_linux_ppc.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2012, 2013 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_PPC_VMSTRUCTS_LINUX_PPC_HPP
-#define OS_CPU_LINUX_PPC_VMSTRUCTS_LINUX_PPC_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_PPC_VMSTRUCTS_LINUX_PPC_HPP
diff --git a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
index 5b427693a8d..06d6aaf109f 100644
--- a/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
+++ b/src/hotspot/os_cpu/linux_riscv/riscv_hwprobe.cpp
@@ -187,18 +187,53 @@ void RiscvHwprobe::add_features_from_query_result() {
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBS)) {
VM_Version::ext_Zbs.enable_feature();
}
+#ifndef PRODUCT
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZBKB)) {
+ VM_Version::ext_Zbkb.enable_feature();
+ }
+#endif
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFH)) {
VM_Version::ext_Zfh.enable_feature();
}
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFHMIN)) {
VM_Version::ext_Zfhmin.enable_feature();
}
+#ifndef PRODUCT
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVBB)) {
+ VM_Version::ext_Zvbb.enable_feature();
+ }
+#endif
+#ifndef PRODUCT
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVBC)) {
VM_Version::ext_Zvbc.enable_feature();
}
+#endif
+#ifndef PRODUCT
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKNED) &&
+ is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKNHB) &&
+ is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKB) &&
+ is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVKT)) {
+ VM_Version::ext_Zvkn.enable_feature();
+ }
+#endif
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZVFH)) {
VM_Version::ext_Zvfh.enable_feature();
}
+#ifndef PRODUCT
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZFA)) {
+ VM_Version::ext_Zfa.enable_feature();
+ }
+#endif
+#ifndef PRODUCT
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZTSO)) {
+ VM_Version::ext_Ztso.enable_feature();
+ }
+#endif
+#ifndef PRODUCT
+ if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZACAS)) {
+ VM_Version::ext_Zacas.enable_feature();
+ }
+#endif
if (is_set(RISCV_HWPROBE_KEY_IMA_EXT_0, RISCV_HWPROBE_EXT_ZICOND)) {
VM_Version::ext_Zicond.enable_feature();
}
diff --git a/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp
deleted file mode 100644
index b39a329335a..00000000000
--- a/src/hotspot/os_cpu/linux_riscv/vmStructs_linux_riscv.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_RISCV_VM_VMSTRUCTS_LINUX_RISCV_HPP
-#define OS_CPU_LINUX_RISCV_VM_VMSTRUCTS_LINUX_RISCV_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_RISCV_VM_VMSTRUCTS_LINUX_RISCV_HPP
diff --git a/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp b/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp
deleted file mode 100644
index a52bc722579..00000000000
--- a/src/hotspot/os_cpu/linux_s390/vmStructs_linux_s390.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2016 SAP SE. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_S390_VMSTRUCTS_LINUX_S390_HPP
-#define OS_CPU_LINUX_S390_VMSTRUCTS_LINUX_S390_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_S390_VMSTRUCTS_LINUX_S390_HPP
diff --git a/src/hotspot/os_cpu/linux_x86/linux_x86_32.S b/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
deleted file mode 100644
index 43a9a38e57f..00000000000
--- a/src/hotspot/os_cpu/linux_x86/linux_x86_32.S
+++ /dev/null
@@ -1,518 +0,0 @@
-#
-# Copyright (c) 2004, 2024, Oracle and/or its affiliates. All rights reserved.
-# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
-#
-# This code is free software; you can redistribute it and/or modify it
-# under the terms of the GNU General Public License version 2 only, as
-# published by the Free Software Foundation.
-#
-# This code is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
-# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
-# version 2 for more details (a copy is included in the LICENSE file that
-# accompanied this code).
-#
-# You should have received a copy of the GNU General Public License version
-# 2 along with this work; if not, write to the Free Software Foundation,
-# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
-# or visit www.oracle.com if you need additional information or have any
-# questions.
-#
-
-#include "defs.S.inc"
-
- # NOTE WELL! The _Copy functions are called directly
- # from server-compiler-generated code via CallLeafNoFP,
- # which means that they *must* either not use floating
- # point or use it in the same manner as does the server
- # compiler.
-
- .text
-
- .p2align 4,,15
-DECLARE_FUNC(SpinPause):
- rep
- nop
- movl $1, %eax
- ret
-
- # Support for void Copy::arrayof_conjoint_bytes(void* from,
- # void* to,
- # size_t count)
- #
- .p2align 4,,15
-DECLARE_FUNC(_Copy_arrayof_conjoint_bytes):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -1(%esi,%ecx),%eax # from + count - 1
- jbe acb_CopyRight
- cmpl %eax,%edi
- jbe acb_CopyLeft
- # copy from low to high
-acb_CopyRight:
- cmpl $3,%ecx
- jbe 5f
-1: movl %ecx,%eax
- shrl $2,%ecx
- jz 4f
- cmpl $32,%ecx
- ja 3f
- # copy aligned dwords
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
- # copy aligned dwords
-3: rep; smovl
-4: movl %eax,%ecx
-5: andl $3,%ecx
- jz 7f
- # copy suffix
- xorl %eax,%eax
-6: movb (%esi,%eax,1),%dl
- movb %dl,(%edi,%eax,1)
- addl $1,%eax
- subl $1,%ecx
- jnz 6b
-7: popl %edi
- popl %esi
- ret
-acb_CopyLeft:
- std
- leal -4(%edi,%ecx),%edi # to + count - 4
- movl %eax,%esi # from + count - 1
- movl %ecx,%eax
- subl $3,%esi # from + count - 4
- cmpl $3,%ecx
- jbe 5f
-1: shrl $2,%ecx
- jz 4f
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- rep; smovl
- jmp 4f
- .space 8
-2: subl %esi,%edi
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 3b
- addl %esi,%edi
-4: movl %eax,%ecx
-5: andl $3,%ecx
- jz 7f
- subl %esi,%edi
- addl $3,%esi
-6: movb (%esi),%dl
- movb %dl,(%edi,%esi,1)
- subl $1,%esi
- subl $1,%ecx
- jnz 6b
-7: cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::conjoint_jshorts_atomic(void* from,
- # void* to,
- # size_t count)
- .p2align 4,,15
-DECLARE_FUNC(_Copy_conjoint_jshorts_atomic):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -2(%esi,%ecx,2),%eax # from + count*2 - 2
- jbe cs_CopyRight
- cmpl %eax,%edi
- jbe cs_CopyLeft
- # copy from low to high
-cs_CopyRight:
- # align source address at dword address boundary
- movl %esi,%eax # original from
- andl $3,%eax # either 0 or 2
- jz 1f # no prefix
- # copy prefix
- subl $1,%ecx
- jl 5f # zero count
- movw (%esi),%dx
- movw %dx,(%edi)
- addl %eax,%esi # %eax == 2
- addl %eax,%edi
-1: movl %ecx,%eax # word count less prefix
- sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- # copy aligned dwords
- rep; smovl
- jmp 4f
- # copy aligned dwords
-2: subl %esi,%edi
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 3b
- addl %esi,%edi
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- movw (%esi),%dx
- movw %dx,(%edi)
-5: popl %edi
- popl %esi
- ret
- # copy from high to low
-cs_CopyLeft:
- std
- leal -4(%edi,%ecx,2),%edi # to + count*2 - 4
- movl %eax,%esi # from + count*2 - 2
- movl %ecx,%eax
- subl $2,%esi # from + count*2 - 4
-1: sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- ja 3f # > 32 dwords
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
-3: rep; smovl
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- addl $2,%esi
- addl $2,%edi
- movw (%esi),%dx
- movw %dx,(%edi)
-5: cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::arrayof_conjoint_jshorts(void* from,
- # void* to,
- # size_t count)
- .p2align 4,,15
-DECLARE_FUNC(_Copy_arrayof_conjoint_jshorts):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -2(%esi,%ecx,2),%eax # from + count*2 - 2
- jbe acs_CopyRight
- cmpl %eax,%edi
- jbe acs_CopyLeft
-acs_CopyRight:
- movl %ecx,%eax # word count
- sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- # copy aligned dwords
- rep; smovl
- jmp 4f
- # copy aligned dwords
- .space 5
-2: subl %esi,%edi
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 3b
- addl %esi,%edi
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- movw (%esi),%dx
- movw %dx,(%edi)
-5: popl %edi
- popl %esi
- ret
-acs_CopyLeft:
- std
- leal -4(%edi,%ecx,2),%edi # to + count*2 - 4
- movl %eax,%esi # from + count*2 - 2
- movl %ecx,%eax
- subl $2,%esi # from + count*2 - 4
- sarl %ecx # dword count
- jz 4f # no dwords to move
- cmpl $32,%ecx
- ja 3f # > 32 dwords
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
-3: rep; smovl
-4: andl $1,%eax # suffix count
- jz 5f # no suffix
- # copy suffix
- addl $2,%esi
- addl $2,%edi
- movw (%esi),%dx
- movw %dx,(%edi)
-5: cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::conjoint_jints_atomic(void* from,
- # void* to,
- # size_t count)
- # Equivalent to
- # arrayof_conjoint_jints
- .p2align 4,,15
-DECLARE_FUNC(_Copy_conjoint_jints_atomic):
-DECLARE_FUNC(_Copy_arrayof_conjoint_jints):
- pushl %esi
- movl 4+12(%esp),%ecx # count
- pushl %edi
- movl 8+ 4(%esp),%esi # from
- movl 8+ 8(%esp),%edi # to
- cmpl %esi,%edi
- leal -4(%esi,%ecx,4),%eax # from + count*4 - 4
- jbe ci_CopyRight
- cmpl %eax,%edi
- jbe ci_CopyLeft
-ci_CopyRight:
- cmpl $32,%ecx
- jbe 2f # <= 32 dwords
- rep; smovl
- popl %edi
- popl %esi
- ret
- .space 10
-2: subl %esi,%edi
- jmp 4f
- .p2align 4,,15
-3: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
-4: subl $1,%ecx
- jge 3b
- popl %edi
- popl %esi
- ret
-ci_CopyLeft:
- std
- leal -4(%edi,%ecx,4),%edi # to + count*4 - 4
- cmpl $32,%ecx
- ja 4f # > 32 dwords
- subl %eax,%edi # eax == from + count*4 - 4
- jmp 3f
- .p2align 4,,15
-2: movl (%eax),%edx
- movl %edx,(%edi,%eax,1)
- subl $4,%eax
-3: subl $1,%ecx
- jge 2b
- cld
- popl %edi
- popl %esi
- ret
-4: movl %eax,%esi # from + count*4 - 4
- rep; smovl
- cld
- popl %edi
- popl %esi
- ret
-
- # Support for void Copy::conjoint_jlongs_atomic(jlong* from,
- # jlong* to,
- # size_t count)
- #
- # 32-bit
- #
- # count treated as signed
- /*
- #
- # if (from > to) {
- # while (--count >= 0) {
- # *to++ = *from++;
- # }
- # } else {
- # while (--count >= 0) {
- # to[count] = from[count];
- # }
- # }
- */
- .p2align 4,,15
-DECLARE_FUNC(_Copy_conjoint_jlongs_atomic):
- movl 4+8(%esp),%ecx # count
- movl 4+0(%esp),%eax # from
- movl 4+4(%esp),%edx # to
- cmpl %eax,%edx
- jae cla_CopyLeft
-cla_CopyRight:
- subl %eax,%edx
- jmp 2f
- .p2align 4,,15
-1: fildll (%eax)
- fistpll (%edx,%eax,1)
- addl $8,%eax
-2: subl $1,%ecx
- jge 1b
- ret
- .p2align 4,,15
-3: fildll (%eax,%ecx,8)
- fistpll (%edx,%ecx,8)
-cla_CopyLeft:
- subl $1,%ecx
- jge 3b
- ret
-
- # Support for void Copy::arrayof_conjoint_jshorts(void* from,
- # void* to,
- # size_t count)
- .p2align 4,,15
-DECLARE_FUNC(_mmx_Copy_arrayof_conjoint_jshorts):
- pushl %esi
- movl 4+12(%esp),%ecx
- pushl %edi
- movl 8+ 4(%esp),%esi
- movl 8+ 8(%esp),%edi
- cmpl %esi,%edi
- leal -2(%esi,%ecx,2),%eax
- jbe mmx_acs_CopyRight
- cmpl %eax,%edi
- jbe mmx_acs_CopyLeft
-mmx_acs_CopyRight:
- movl %ecx,%eax
- sarl %ecx
- je 5f
- cmpl $33,%ecx
- jae 3f
-1: subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- addl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 5f
-3: smovl # align to 8 bytes, we know we are 4 byte aligned to start
- subl $1,%ecx
-4: .p2align 4,,15
- movq 0(%esi),%mm0
- addl $64,%edi
- movq 8(%esi),%mm1
- subl $16,%ecx
- movq 16(%esi),%mm2
- movq %mm0,-64(%edi)
- movq 24(%esi),%mm0
- movq %mm1,-56(%edi)
- movq 32(%esi),%mm1
- movq %mm2,-48(%edi)
- movq 40(%esi),%mm2
- movq %mm0,-40(%edi)
- movq 48(%esi),%mm0
- movq %mm1,-32(%edi)
- movq 56(%esi),%mm1
- movq %mm2,-24(%edi)
- movq %mm0,-16(%edi)
- addl $64,%esi
- movq %mm1,-8(%edi)
- cmpl $16,%ecx
- jge 4b
- emms
- testl %ecx,%ecx
- ja 1b
-5: andl $1,%eax
- je 7f
-6: movw (%esi),%dx
- movw %dx,(%edi)
-7: popl %edi
- popl %esi
- ret
-mmx_acs_CopyLeft:
- std
- leal -4(%edi,%ecx,2),%edi
- movl %eax,%esi
- movl %ecx,%eax
- subl $2,%esi
- sarl %ecx
- je 4f
- cmpl $32,%ecx
- ja 3f
- subl %esi,%edi
- .p2align 4,,15
-2: movl (%esi),%edx
- movl %edx,(%edi,%esi,1)
- subl $4,%esi
- subl $1,%ecx
- jnz 2b
- addl %esi,%edi
- jmp 4f
-3: rep; smovl
-4: andl $1,%eax
- je 6f
- addl $2,%esi
- addl $2,%edi
-5: movw (%esi),%dx
- movw %dx,(%edi)
-6: cld
- popl %edi
- popl %esi
- ret
-
-
- # Support for jlong Atomic::cmpxchg(volatile jlong* dest,
- # jlong compare_value,
- # jlong exchange_value)
- #
- .p2align 4,,15
-DECLARE_FUNC(_Atomic_cmpxchg_long):
- # 8(%esp) : return PC
- pushl %ebx # 4(%esp) : old %ebx
- pushl %edi # 0(%esp) : old %edi
- movl 12(%esp), %ebx # 12(%esp) : exchange_value (low)
- movl 16(%esp), %ecx # 16(%esp) : exchange_value (high)
- movl 24(%esp), %eax # 24(%esp) : compare_value (low)
- movl 28(%esp), %edx # 28(%esp) : compare_value (high)
- movl 20(%esp), %edi # 20(%esp) : dest
- lock cmpxchg8b (%edi)
- popl %edi
- popl %ebx
- ret
-
-
- # Support for jlong Atomic::load and Atomic::store.
- # void _Atomic_move_long(const volatile jlong* src, volatile jlong* dst)
- .p2align 4,,15
-DECLARE_FUNC(_Atomic_move_long):
- movl 4(%esp), %eax # src
- fildll (%eax)
- movl 8(%esp), %eax # dest
- fistpll (%eax)
- ret
diff --git a/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp b/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp
deleted file mode 100644
index ddba9daf131..00000000000
--- a/src/hotspot/os_cpu/linux_x86/vmStructs_linux_x86.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_X86_VMSTRUCTS_LINUX_X86_HPP
-#define OS_CPU_LINUX_X86_VMSTRUCTS_LINUX_X86_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_X86_VMSTRUCTS_LINUX_X86_HPP
diff --git a/src/hotspot/os_cpu/linux_zero/vmStructs_linux_zero.hpp b/src/hotspot/os_cpu/linux_zero/vmStructs_linux_zero.hpp
deleted file mode 100644
index 271193cd705..00000000000
--- a/src/hotspot/os_cpu/linux_zero/vmStructs_linux_zero.hpp
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2007 Red Hat, Inc.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_LINUX_ZERO_VMSTRUCTS_LINUX_ZERO_HPP
-#define OS_CPU_LINUX_ZERO_VMSTRUCTS_LINUX_ZERO_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_LINUX_ZERO_VMSTRUCTS_LINUX_ZERO_HPP
diff --git a/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp b/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp
deleted file mode 100644
index 18a5588b743..00000000000
--- a/src/hotspot/os_cpu/windows_aarch64/vmStructs_windows_aarch64.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2020, Microsoft Corporation. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_WINDOWS_AARCH64_VMSTRUCTS_WINDOWS_AARCH64_HPP
-#define OS_CPU_WINDOWS_AARCH64_VMSTRUCTS_WINDOWS_AARCH64_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_WINDOWS_AARCH64_VMSTRUCTS_WINDOWS_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp b/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp
deleted file mode 100644
index 4ed62839d51..00000000000
--- a/src/hotspot/os_cpu/windows_x86/vmStructs_windows_x86.hpp
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-#ifndef OS_CPU_WINDOWS_X86_VMSTRUCTS_WINDOWS_X86_HPP
-#define OS_CPU_WINDOWS_X86_VMSTRUCTS_WINDOWS_X86_HPP
-
-// These are the OS and CPU-specific fields, types and integer
-// constants required by the Serviceability Agent. This file is
-// referenced by vmStructs.cpp.
-
-#define VM_STRUCTS_OS_CPU(nonstatic_field, static_field, unchecked_nonstatic_field, volatile_nonstatic_field, nonproduct_nonstatic_field, c2_nonstatic_field, unchecked_c1_static_field, unchecked_c2_static_field)
-
-#define VM_TYPES_OS_CPU(declare_type, declare_toplevel_type, declare_oop_type, declare_integer_type, declare_unsigned_integer_type, declare_c1_toplevel_type, declare_c2_type, declare_c2_toplevel_type)
-
-#define VM_INT_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#define VM_LONG_CONSTANTS_OS_CPU(declare_constant, declare_preprocessor_constant, declare_c1_constant, declare_c2_constant, declare_c2_preprocessor_constant)
-
-#endif // OS_CPU_WINDOWS_X86_VMSTRUCTS_WINDOWS_X86_HPP
diff --git a/src/hotspot/share/adlc/archDesc.cpp b/src/hotspot/share/adlc/archDesc.cpp
index edb07d2d22c..263752c521d 100644
--- a/src/hotspot/share/adlc/archDesc.cpp
+++ b/src/hotspot/share/adlc/archDesc.cpp
@@ -24,7 +24,7 @@
// archDesc.cpp - Internal format for architecture definition
-#include
+#include // do not reorder
#include "adlc.hpp"
static FILE *errfile = stderr;
@@ -751,12 +751,11 @@ bool ArchDesc::check_usage() {
callback.do_form_by_name("sRegL");
// special generic vector operands only used in Matcher::pd_specialize_generic_vector_operand
- // x86_32 combine x86.ad and x86_32.ad, the vec*/legVec* can not be cleaned from IA32
#if defined(AARCH64)
callback.do_form_by_name("vecA");
callback.do_form_by_name("vecD");
callback.do_form_by_name("vecX");
-#elif defined(IA32) || defined(AMD64)
+#elif defined(AMD64)
callback.do_form_by_name("vecS");
callback.do_form_by_name("vecD");
callback.do_form_by_name("vecX");
diff --git a/src/hotspot/share/adlc/formssel.cpp b/src/hotspot/share/adlc/formssel.cpp
index f18e9eddba5..7239031c898 100644
--- a/src/hotspot/share/adlc/formssel.cpp
+++ b/src/hotspot/share/adlc/formssel.cpp
@@ -1854,14 +1854,14 @@ void InsEncode::output(FILE *fp) {
fprintf(fp,"InsEncode: ");
_encoding.reset();
- while ( (encoding = (NameAndList*)_encoding.iter()) != 0 ) {
+ while ( (encoding = (NameAndList*)_encoding.iter()) != nullptr ) {
// Output the encoding being used
fprintf(fp,"%s(", encoding->name() );
// Output its parameter list, if any
bool first_param = true;
encoding->reset();
- while ( (parameter = encoding->iter()) != 0 ) {
+ while ( (parameter = encoding->iter()) != nullptr ) {
// Output the ',' between parameters
if ( ! first_param ) fprintf(fp,", ");
first_param = false;
@@ -3305,7 +3305,7 @@ void ComponentList::output(FILE *fp) {
MatchNode::MatchNode(ArchDesc &ad, const char *result, const char *mexpr,
const char *opType, MatchNode *lChild, MatchNode *rChild)
: _AD(ad), _result(result), _name(mexpr), _opType(opType),
- _lChild(lChild), _rChild(rChild), _internalop(0), _numleaves(0),
+ _lChild(lChild), _rChild(rChild), _internalop(nullptr), _numleaves(0),
_commutative_id(0) {
_numleaves = (lChild ? lChild->_numleaves : 0)
+ (rChild ? rChild->_numleaves : 0);
@@ -3314,14 +3314,14 @@ MatchNode::MatchNode(ArchDesc &ad, const char *result, const char *mexpr,
MatchNode::MatchNode(ArchDesc &ad, MatchNode& mnode)
: _AD(ad), _result(mnode._result), _name(mnode._name),
_opType(mnode._opType), _lChild(mnode._lChild), _rChild(mnode._rChild),
- _internalop(0), _numleaves(mnode._numleaves),
+ _internalop(nullptr), _numleaves(mnode._numleaves),
_commutative_id(mnode._commutative_id) {
}
MatchNode::MatchNode(ArchDesc &ad, MatchNode& mnode, int clone)
: _AD(ad), _result(mnode._result), _name(mnode._name),
_opType(mnode._opType),
- _internalop(0), _numleaves(mnode._numleaves),
+ _internalop(nullptr), _numleaves(mnode._numleaves),
_commutative_id(mnode._commutative_id) {
if (mnode._lChild) {
_lChild = new MatchNode(ad, *mnode._lChild, clone);
@@ -3624,7 +3624,7 @@ void MatchNode::dump() {
}
void MatchNode::output(FILE *fp) {
- if (_lChild==0 && _rChild==0) {
+ if (_lChild==nullptr && _rChild==nullptr) {
fprintf(fp," %s",_name); // operand
}
else {
@@ -4225,9 +4225,7 @@ int MatchRule::is_expensive() const {
strcmp(opType,"FmaD") == 0 ||
strcmp(opType,"FmaF") == 0 ||
strcmp(opType,"FmaHF") == 0 ||
- strcmp(opType,"RoundDouble")==0 ||
strcmp(opType,"RoundDoubleMode")==0 ||
- strcmp(opType,"RoundFloat")==0 ||
strcmp(opType,"ReverseBytesI")==0 ||
strcmp(opType,"ReverseBytesL")==0 ||
strcmp(opType,"ReverseBytesUS")==0 ||
diff --git a/src/hotspot/share/adlc/formssel.hpp b/src/hotspot/share/adlc/formssel.hpp
index 61d0fb40f18..dca1a50991b 100644
--- a/src/hotspot/share/adlc/formssel.hpp
+++ b/src/hotspot/share/adlc/formssel.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1998, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -973,8 +973,8 @@ public:
int _commutative_id; // id of commutative operation
// Public Methods
- MatchNode(ArchDesc &ad, const char *result = 0, const char *expr = 0,
- const char *opType=0, MatchNode *lChild=nullptr,
+ MatchNode(ArchDesc &ad, const char *result = nullptr, const char *expr = nullptr,
+ const char *opType=nullptr, MatchNode *lChild=nullptr,
MatchNode *rChild=nullptr);
MatchNode(ArchDesc &ad, MatchNode& mNode); // Shallow copy constructor;
MatchNode(ArchDesc &ad, MatchNode& mNode, int clone); // Construct clone
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index 2aa77abc5f2..917569e2be6 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -980,6 +980,7 @@ void CodeBuffer::take_over_code_from(CodeBuffer* cb) {
void CodeBuffer::verify_section_allocation() {
address tstart = _total_start;
+ if (tstart == nullptr) return; // ignore not fully initialized buffer
if (tstart == badAddress) return; // smashed by set_blob(nullptr)
address tend = tstart + _total_size;
if (_blob != nullptr) {
diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp
index 025aa641d2c..f855d41b181 100644
--- a/src/hotspot/share/asm/codeBuffer.hpp
+++ b/src/hotspot/share/asm/codeBuffer.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -455,6 +455,8 @@ class CodeBuffer: public StackObj DEBUG_ONLY(COMMA private Scrubber) {
_name = name;
_before_expand = nullptr;
_blob = nullptr;
+ _total_start = nullptr;
+ _total_size = 0;
_oop_recorder = nullptr;
_overflow_arena = nullptr;
_last_insn = nullptr;
diff --git a/src/hotspot/share/c1/c1_CFGPrinter.cpp b/src/hotspot/share/c1/c1_CFGPrinter.cpp
index d178b699238..a846507c3f0 100644
--- a/src/hotspot/share/c1/c1_CFGPrinter.cpp
+++ b/src/hotspot/share/c1/c1_CFGPrinter.cpp
@@ -23,10 +23,10 @@
*/
#include "c1/c1_CFGPrinter.hpp"
-#include "c1/c1_IR.hpp"
#include "c1/c1_InstructionPrinter.hpp"
-#include "c1/c1_LIR.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_LinearScan.hpp"
+#include "c1/c1_LIR.hpp"
#include "c1/c1_ValueStack.hpp"
#include "jvm.h"
diff --git a/src/hotspot/share/c1/c1_Canonicalizer.cpp b/src/hotspot/share/c1/c1_Canonicalizer.cpp
index f5a1d14e694..573e1ac24d7 100644
--- a/src/hotspot/share/c1/c1_Canonicalizer.cpp
+++ b/src/hotspot/share/c1/c1_Canonicalizer.cpp
@@ -840,7 +840,6 @@ void Canonicalizer::do_Throw (Throw* x) {}
void Canonicalizer::do_Base (Base* x) {}
void Canonicalizer::do_OsrEntry (OsrEntry* x) {}
void Canonicalizer::do_ExceptionObject(ExceptionObject* x) {}
-void Canonicalizer::do_RoundFP (RoundFP* x) {}
void Canonicalizer::do_UnsafeGet (UnsafeGet* x) {}
void Canonicalizer::do_UnsafePut (UnsafePut* x) {}
void Canonicalizer::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {}
diff --git a/src/hotspot/share/c1/c1_Canonicalizer.hpp b/src/hotspot/share/c1/c1_Canonicalizer.hpp
index 8c7651256e9..f1c99d4996c 100644
--- a/src/hotspot/share/c1/c1_Canonicalizer.hpp
+++ b/src/hotspot/share/c1/c1_Canonicalizer.hpp
@@ -88,7 +88,6 @@ class Canonicalizer: InstructionVisitor {
virtual void do_Base (Base* x);
virtual void do_OsrEntry (OsrEntry* x);
virtual void do_ExceptionObject(ExceptionObject* x);
- virtual void do_RoundFP (RoundFP* x);
virtual void do_UnsafeGet (UnsafeGet* x);
virtual void do_UnsafePut (UnsafePut* x);
virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
diff --git a/src/hotspot/share/c1/c1_CodeStubs.hpp b/src/hotspot/share/c1/c1_CodeStubs.hpp
index 9abfa45785b..a885e4c0816 100644
--- a/src/hotspot/share/c1/c1_CodeStubs.hpp
+++ b/src/hotspot/share/c1/c1_CodeStubs.hpp
@@ -26,8 +26,8 @@
#define SHARE_C1_C1_CODESTUBS_HPP
#include "c1/c1_FrameMap.hpp"
-#include "c1/c1_IR.hpp"
#include "c1/c1_Instruction.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_LIR.hpp"
#include "c1/c1_Runtime1.hpp"
#include "code/nativeInst.hpp"
diff --git a/src/hotspot/share/c1/c1_Compilation.cpp b/src/hotspot/share/c1/c1_Compilation.cpp
index 9b80c8a20a8..53a97ce1042 100644
--- a/src/hotspot/share/c1/c1_Compilation.cpp
+++ b/src/hotspot/share/c1/c1_Compilation.cpp
@@ -25,8 +25,8 @@
#include "c1/c1_CFGPrinter.hpp"
#include "c1/c1_Compilation.hpp"
#include "c1/c1_IR.hpp"
-#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_LinearScan.hpp"
+#include "c1/c1_LIRAssembler.hpp"
#include "c1/c1_MacroAssembler.hpp"
#include "c1/c1_RangeCheckElimination.hpp"
#include "c1/c1_ValueMap.hpp"
@@ -34,11 +34,10 @@
#include "code/debugInfoRec.hpp"
#include "compiler/compilationFailureInfo.hpp"
#include "compiler/compilationMemoryStatistic.hpp"
-#include "compiler/compilerDirectives.hpp"
#include "compiler/compileLog.hpp"
-#include "compiler/compileTask.hpp"
#include "compiler/compiler_globals.hpp"
#include "compiler/compilerDirectives.hpp"
+#include "compiler/compileTask.hpp"
#include "memory/resourceArea.hpp"
#include "runtime/sharedRuntime.hpp"
#include "runtime/timerTrace.hpp"
@@ -410,6 +409,8 @@ int Compilation::compile_java_method() {
env()->dump_replay_data(env()->compile_id());
}
+ DEBUG_ONLY(CompilationMemoryStatistic::do_test_allocations();)
+
{
PhaseTraceTime timeit(_t_codeemit);
return emit_code_body();
diff --git a/src/hotspot/share/c1/c1_Compiler.cpp b/src/hotspot/share/c1/c1_Compiler.cpp
index eb7c42e4576..dd13b84edf5 100644
--- a/src/hotspot/share/c1/c1_Compiler.cpp
+++ b/src/hotspot/share/c1/c1_Compiler.cpp
@@ -47,16 +47,19 @@
Compiler::Compiler() : AbstractCompiler(compiler_c1) {
}
-void Compiler::init_c1_runtime() {
+bool Compiler::init_c1_runtime() {
BufferBlob* buffer_blob = CompilerThread::current()->get_buffer_blob();
FrameMap::initialize();
- Runtime1::initialize(buffer_blob);
+ if (!Runtime1::initialize(buffer_blob)) {
+ return false;
+ }
// initialize data structures
ValueType::initialize();
GraphBuilder::initialize();
// note: to use more than one instance of LinearScan at a time this function call has to
// be moved somewhere outside of this constructor:
Interval::initialize();
+ return true;
}
@@ -65,12 +68,11 @@ void Compiler::initialize() {
BufferBlob* buffer_blob = init_buffer_blob();
if (should_perform_init()) {
- if (buffer_blob == nullptr) {
+ if (buffer_blob == nullptr || !init_c1_runtime()) {
// When we come here we are in state 'initializing'; entire C1 compilation
// can be shut down.
set_state(failed);
} else {
- init_c1_runtime();
set_state(initialized);
}
}
@@ -222,7 +224,7 @@ bool Compiler::is_intrinsic_supported(vmIntrinsics::ID id) {
case vmIntrinsics::_updateCRC32:
case vmIntrinsics::_updateBytesCRC32:
case vmIntrinsics::_updateByteBufferCRC32:
-#if defined(S390) || defined(PPC64) || defined(AARCH64)
+#if defined(S390) || defined(PPC64) || defined(AARCH64) || defined(AMD64)
case vmIntrinsics::_updateBytesCRC32C:
case vmIntrinsics::_updateDirectByteBufferCRC32C:
#endif
diff --git a/src/hotspot/share/c1/c1_Compiler.hpp b/src/hotspot/share/c1/c1_Compiler.hpp
index 0160ee63574..fde7c8e7932 100644
--- a/src/hotspot/share/c1/c1_Compiler.hpp
+++ b/src/hotspot/share/c1/c1_Compiler.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,7 @@ class DirectiveSet;
class Compiler: public AbstractCompiler {
private:
- static void init_c1_runtime();
+ static bool init_c1_runtime();
BufferBlob* init_buffer_blob();
public:
diff --git a/src/hotspot/share/c1/c1_Defs.hpp b/src/hotspot/share/c1/c1_Defs.hpp
index 0e7b120ef8d..5803e1ce686 100644
--- a/src/hotspot/share/c1/c1_Defs.hpp
+++ b/src/hotspot/share/c1/c1_Defs.hpp
@@ -44,13 +44,6 @@ enum {
hi_word_offset_in_bytes = pd_hi_word_offset_in_bytes
};
-
-// the processor may require explicit rounding operations to implement the strictFP mode
-enum {
- strict_fp_requires_explicit_rounding = pd_strict_fp_requires_explicit_rounding
-};
-
-
// for debug info: a float value in a register may be saved in double precision by runtime stubs
enum {
float_saved_as_double = pd_float_saved_as_double
diff --git a/src/hotspot/share/c1/c1_FrameMap.hpp b/src/hotspot/share/c1/c1_FrameMap.hpp
index dab3aa6e734..4e4fde0cb4a 100644
--- a/src/hotspot/share/c1/c1_FrameMap.hpp
+++ b/src/hotspot/share/c1/c1_FrameMap.hpp
@@ -29,10 +29,10 @@
#include "c1/c1_LIR.hpp"
#include "code/vmreg.hpp"
#include "memory/allocation.hpp"
+#include "oops/compressedOops.hpp"
#include "runtime/frame.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/macros.hpp"
-#include "oops/compressedOops.hpp"
class ciMethod;
class CallingConvention;
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.cpp b/src/hotspot/share/c1/c1_GraphBuilder.cpp
index e918aa7d19a..201ee695f69 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.cpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.cpp
@@ -22,8 +22,8 @@
*
*/
-#include "c1/c1_CFGPrinter.hpp"
#include "c1/c1_Canonicalizer.hpp"
+#include "c1/c1_CFGPrinter.hpp"
#include "c1/c1_Compilation.hpp"
#include "c1/c1_GraphBuilder.hpp"
#include "c1/c1_InstructionPrinter.hpp"
@@ -673,17 +673,6 @@ class MemoryBuffer: public CompilationResourceObj {
return load;
}
- if (strict_fp_requires_explicit_rounding && load->type()->is_float_kind()) {
-#ifdef IA32
- if (UseSSE < 2) {
- // can't skip load since value might get rounded as a side effect
- return load;
- }
-#else
- Unimplemented();
-#endif // IA32
- }
-
ciField* field = load->field();
Value object = load->obj();
if (field->holder()->is_loaded() && !field->is_volatile()) {
@@ -1052,7 +1041,7 @@ void GraphBuilder::store_local(ValueStack* state, Value x, int index) {
}
}
- state->store_local(index, round_fp(x));
+ state->store_local(index, x);
}
@@ -1204,10 +1193,7 @@ void GraphBuilder::arithmetic_op(ValueType* type, Bytecodes::Code code, ValueSta
Value y = pop(type);
Value x = pop(type);
Value res = new ArithmeticOp(code, x, y, state_before);
- // Note: currently single-precision floating-point rounding on Intel is handled at the LIRGenerator level
- res = append(res);
- res = round_fp(res);
- push(type, res);
+ push(type, append(res));
}
@@ -2228,7 +2214,7 @@ void GraphBuilder::invoke(Bytecodes::Code code) {
append_split(result);
if (result_type != voidType) {
- push(result_type, round_fp(result));
+ push(result_type, result);
}
if (profile_return() && result_type->is_object_kind()) {
profile_return_type(result, target);
@@ -2356,29 +2342,6 @@ void GraphBuilder::throw_op(int bci) {
}
-Value GraphBuilder::round_fp(Value fp_value) {
- if (strict_fp_requires_explicit_rounding) {
-#ifdef IA32
- // no rounding needed if SSE2 is used
- if (UseSSE < 2) {
- // Must currently insert rounding node for doubleword values that
- // are results of expressions (i.e., not loads from memory or
- // constants)
- if (fp_value->type()->tag() == doubleTag &&
- fp_value->as_Constant() == nullptr &&
- fp_value->as_Local() == nullptr && // method parameters need no rounding
- fp_value->as_RoundFP() == nullptr) {
- return append(new RoundFP(fp_value));
- }
- }
-#else
- Unimplemented();
-#endif // IA32
- }
- return fp_value;
-}
-
-
Instruction* GraphBuilder::append_with_bci(Instruction* instr, int bci) {
Canonicalizer canon(compilation(), instr, bci);
Instruction* i1 = canon.canonical();
diff --git a/src/hotspot/share/c1/c1_GraphBuilder.hpp b/src/hotspot/share/c1/c1_GraphBuilder.hpp
index 270c344833e..6eaeda5adcf 100644
--- a/src/hotspot/share/c1/c1_GraphBuilder.hpp
+++ b/src/hotspot/share/c1/c1_GraphBuilder.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_C1_C1_GRAPHBUILDER_HPP
#define SHARE_C1_C1_GRAPHBUILDER_HPP
-#include "c1/c1_IR.hpp"
#include "c1/c1_Instruction.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_ValueMap.hpp"
#include "c1/c1_ValueStack.hpp"
#include "ci/ciMethodData.hpp"
@@ -266,7 +266,6 @@ class GraphBuilder {
void monitorexit(Value x, int bci);
void new_multi_array(int dimensions);
void throw_op(int bci);
- Value round_fp(Value fp_value);
// stack/code manipulation helpers
Instruction* append_with_bci(Instruction* instr, int bci);
diff --git a/src/hotspot/share/c1/c1_IR.cpp b/src/hotspot/share/c1/c1_IR.cpp
index 7f006c0b3ff..ae8332116b3 100644
--- a/src/hotspot/share/c1/c1_IR.cpp
+++ b/src/hotspot/share/c1/c1_IR.cpp
@@ -25,8 +25,8 @@
#include "c1/c1_Compilation.hpp"
#include "c1/c1_FrameMap.hpp"
#include "c1/c1_GraphBuilder.hpp"
-#include "c1/c1_IR.hpp"
#include "c1/c1_InstructionPrinter.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_Optimizer.hpp"
#include "compiler/oopMap.hpp"
#include "memory/resourceArea.hpp"
diff --git a/src/hotspot/share/c1/c1_Instruction.cpp b/src/hotspot/share/c1/c1_Instruction.cpp
index 92affba99b2..3a7edef0088 100644
--- a/src/hotspot/share/c1/c1_Instruction.cpp
+++ b/src/hotspot/share/c1/c1_Instruction.cpp
@@ -22,9 +22,9 @@
*
*/
-#include "c1/c1_IR.hpp"
#include "c1/c1_Instruction.hpp"
#include "c1/c1_InstructionPrinter.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_ValueStack.hpp"
#include "ci/ciObjArrayKlass.hpp"
#include "ci/ciTypeArrayKlass.hpp"
diff --git a/src/hotspot/share/c1/c1_Instruction.hpp b/src/hotspot/share/c1/c1_Instruction.hpp
index e950afc981d..6b5838f6062 100644
--- a/src/hotspot/share/c1/c1_Instruction.hpp
+++ b/src/hotspot/share/c1/c1_Instruction.hpp
@@ -91,7 +91,6 @@ class LookupSwitch;
class Return;
class Throw;
class Base;
-class RoundFP;
class UnsafeOp;
class UnsafeGet;
class UnsafePut;
@@ -187,7 +186,6 @@ class InstructionVisitor: public StackObj {
virtual void do_Base (Base* x) = 0;
virtual void do_OsrEntry (OsrEntry* x) = 0;
virtual void do_ExceptionObject(ExceptionObject* x) = 0;
- virtual void do_RoundFP (RoundFP* x) = 0;
virtual void do_UnsafeGet (UnsafeGet* x) = 0;
virtual void do_UnsafePut (UnsafePut* x) = 0;
virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x) = 0;
@@ -556,7 +554,6 @@ class Instruction: public CompilationResourceObj {
virtual Return* as_Return() { return nullptr; }
virtual Throw* as_Throw() { return nullptr; }
virtual Base* as_Base() { return nullptr; }
- virtual RoundFP* as_RoundFP() { return nullptr; }
virtual ExceptionObject* as_ExceptionObject() { return nullptr; }
virtual UnsafeOp* as_UnsafeOp() { return nullptr; }
virtual ProfileInvoke* as_ProfileInvoke() { return nullptr; }
@@ -1610,7 +1607,6 @@ LEAF(BlockBegin, StateSplit)
ResourceBitMap _live_kill; // set of registers defined in this block
ResourceBitMap _fpu_register_usage;
- intArray* _fpu_stack_state; // For x86 FPU code generation with UseLinearScan
int _first_lir_instruction_id; // ID of first LIR instruction in this block
int _last_lir_instruction_id; // ID of last LIR instruction in this block
@@ -1657,7 +1653,6 @@ LEAF(BlockBegin, StateSplit)
, _live_gen()
, _live_kill()
, _fpu_register_usage()
- , _fpu_stack_state(nullptr)
, _first_lir_instruction_id(-1)
, _last_lir_instruction_id(-1)
{
@@ -1685,7 +1680,6 @@ LEAF(BlockBegin, StateSplit)
ResourceBitMap& live_gen() { return _live_gen; }
ResourceBitMap& live_kill() { return _live_kill; }
ResourceBitMap& fpu_register_usage() { return _fpu_register_usage; }
- intArray* fpu_stack_state() const { return _fpu_stack_state; }
int first_lir_instruction_id() const { return _first_lir_instruction_id; }
int last_lir_instruction_id() const { return _last_lir_instruction_id; }
int total_preds() const { return _total_preds; }
@@ -1708,7 +1702,6 @@ LEAF(BlockBegin, StateSplit)
void set_live_gen (const ResourceBitMap& map) { _live_gen = map; }
void set_live_kill(const ResourceBitMap& map) { _live_kill = map; }
void set_fpu_register_usage(const ResourceBitMap& map) { _fpu_register_usage = map; }
- void set_fpu_stack_state(intArray* state) { _fpu_stack_state = state; }
void set_first_lir_instruction_id(int id) { _first_lir_instruction_id = id; }
void set_last_lir_instruction_id(int id) { _last_lir_instruction_id = id; }
void increment_total_preds(int n = 1) { _total_preds += n; }
@@ -2142,30 +2135,6 @@ LEAF(ExceptionObject, Instruction)
};
-// Models needed rounding for floating-point values on Intel.
-// Currently only used to represent rounding of double-precision
-// values stored into local variables, but could be used to model
-// intermediate rounding of single-precision values as well.
-LEAF(RoundFP, Instruction)
- private:
- Value _input; // floating-point value to be rounded
-
- public:
- RoundFP(Value input)
- : Instruction(input->type()) // Note: should not be used for constants
- , _input(input)
- {
- ASSERT_VALUES
- }
-
- // accessors
- Value input() const { return _input; }
-
- // generic
- virtual void input_values_do(ValueVisitor* f) { f->visit(&_input); }
-};
-
-
BASE(UnsafeOp, Instruction)
private:
Value _object; // Object to be fetched from or mutated
diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.cpp b/src/hotspot/share/c1/c1_InstructionPrinter.cpp
index 35818188496..59e24c3e6c5 100644
--- a/src/hotspot/share/c1/c1_InstructionPrinter.cpp
+++ b/src/hotspot/share/c1/c1_InstructionPrinter.cpp
@@ -22,12 +22,12 @@
*
*/
-#include "classfile/vmSymbols.hpp"
#include "c1/c1_InstructionPrinter.hpp"
#include "c1/c1_ValueStack.hpp"
#include "ci/ciArray.hpp"
#include "ci/ciInstance.hpp"
#include "ci/ciObject.hpp"
+#include "classfile/vmSymbols.hpp"
#ifndef PRODUCT
@@ -779,12 +779,6 @@ void InstructionPrinter::do_ExceptionObject(ExceptionObject* x) {
output()->print("incoming exception");
}
-
-void InstructionPrinter::do_RoundFP(RoundFP* x) {
- output()->print("round_fp ");
- print_value(x->input());
-}
-
void InstructionPrinter::do_UnsafeGet(UnsafeGet* x) {
print_unsafe_op(x, x->is_raw() ? "UnsafeGet (raw)" : "UnsafeGet");
output()->put(')');
diff --git a/src/hotspot/share/c1/c1_InstructionPrinter.hpp b/src/hotspot/share/c1/c1_InstructionPrinter.hpp
index 0e5ba78bdc7..b7bac561327 100644
--- a/src/hotspot/share/c1/c1_InstructionPrinter.hpp
+++ b/src/hotspot/share/c1/c1_InstructionPrinter.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_C1_C1_INSTRUCTIONPRINTER_HPP
#define SHARE_C1_C1_INSTRUCTIONPRINTER_HPP
-#include "c1/c1_IR.hpp"
#include "c1/c1_Instruction.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_Runtime1.hpp"
#ifndef PRODUCT
@@ -120,7 +120,6 @@ class InstructionPrinter: public InstructionVisitor {
virtual void do_Base (Base* x);
virtual void do_OsrEntry (OsrEntry* x);
virtual void do_ExceptionObject(ExceptionObject* x);
- virtual void do_RoundFP (RoundFP* x);
virtual void do_UnsafeGet (UnsafeGet* x);
virtual void do_UnsafePut (UnsafePut* x);
virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp
index fc90530ec95..e6ba03554cb 100644
--- a/src/hotspot/share/c1/c1_LIR.cpp
+++ b/src/hotspot/share/c1/c1_LIR.cpp
@@ -403,7 +403,6 @@ void LIR_OpVisitState::visit(LIR_Op* op) {
switch (op->code()) {
// LIR_Op0
- case lir_fpop_raw: // result and info always invalid
case lir_breakpoint: // result and info always invalid
case lir_membar: // result and info always invalid
case lir_membar_acquire: // result and info always invalid
@@ -443,8 +442,6 @@ void LIR_OpVisitState::visit(LIR_Op* op) {
// LIR_Op1
- case lir_fxch: // input always valid, result and info always invalid
- case lir_fld: // input always valid, result and info always invalid
case lir_push: // input always valid, result and info always invalid
case lir_pop: // input always valid, result and info always invalid
case lir_leal: // input and result always valid, info always invalid
@@ -550,20 +547,6 @@ void LIR_OpVisitState::visit(LIR_Op* op) {
}
-// LIR_OpRoundFP;
- case lir_roundfp: {
- assert(op->as_OpRoundFP() != nullptr, "must be");
- LIR_OpRoundFP* opRoundFP = (LIR_OpRoundFP*)op;
-
- assert(op->_info == nullptr, "info not used by this instruction");
- assert(opRoundFP->_tmp->is_illegal(), "not used");
- do_input(opRoundFP->_opr);
- do_output(opRoundFP->_result);
-
- break;
- }
-
-
// LIR_Op2
case lir_cmp:
case lir_cmp_l2i:
@@ -1716,12 +1699,9 @@ const char * LIR_Op::name() const {
case lir_on_spin_wait: s = "on_spin_wait"; break;
case lir_std_entry: s = "std_entry"; break;
case lir_osr_entry: s = "osr_entry"; break;
- case lir_fpop_raw: s = "fpop_raw"; break;
case lir_breakpoint: s = "breakpoint"; break;
case lir_get_thread: s = "get_thread"; break;
// LIR_Op1
- case lir_fxch: s = "fxch"; break;
- case lir_fld: s = "fld"; break;
case lir_push: s = "push"; break;
case lir_pop: s = "pop"; break;
case lir_null_check: s = "null_check"; break;
@@ -1731,7 +1711,6 @@ const char * LIR_Op::name() const {
case lir_branch: s = "branch"; break;
case lir_cond_float_branch: s = "flt_cond_br"; break;
case lir_move: s = "move"; break;
- case lir_roundfp: s = "roundfp"; break;
case lir_abs: s = "abs"; break;
case lir_neg: s = "neg"; break;
case lir_sqrt: s = "sqrt"; break;
@@ -1976,12 +1955,6 @@ void LIR_OpAllocObj::print_instr(outputStream* out) const {
out->print("[lbl:" INTPTR_FORMAT "]", p2i(stub()->entry()));
}
-void LIR_OpRoundFP::print_instr(outputStream* out) const {
- _opr->print(out); out->print(" ");
- tmp()->print(out); out->print(" ");
- result_opr()->print(out); out->print(" ");
-}
-
// LIR_Op2
void LIR_Op2::print_instr(outputStream* out) const {
if (code() == lir_cmp || code() == lir_branch || code() == lir_cond_float_branch) {
diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp
index d9005c49c89..0de69e658a3 100644
--- a/src/hotspot/share/c1/c1_LIR.hpp
+++ b/src/hotspot/share/c1/c1_LIR.hpp
@@ -43,7 +43,6 @@ class LIR_Op;
class ciType;
class ValueType;
class LIR_OpVisitState;
-class FpuStackSim;
//---------------------------------------------------------------------
// LIR Operands
@@ -236,9 +235,8 @@ class LIR_Opr {
, virtual_bits = 1
, is_xmm_bits = 1
, last_use_bits = 1
- , is_fpu_stack_offset_bits = 1 // used in assertion checking on x86 for FPU stack slot allocation
, non_data_bits = kind_bits + type_bits + size_bits + destroys_bits + virtual_bits
- + is_xmm_bits + last_use_bits + is_fpu_stack_offset_bits
+ + is_xmm_bits + last_use_bits
, data_bits = BitsPerInt - non_data_bits
, reg_bits = data_bits / 2 // for two registers in one value encoding
};
@@ -249,8 +247,7 @@ class LIR_Opr {
, size_shift = type_shift + type_bits
, destroys_shift = size_shift + size_bits
, last_use_shift = destroys_shift + destroys_bits
- , is_fpu_stack_offset_shift = last_use_shift + last_use_bits
- , virtual_shift = is_fpu_stack_offset_shift + is_fpu_stack_offset_bits
+ , virtual_shift = last_use_shift + last_use_bits
, is_xmm_shift = virtual_shift + virtual_bits
, data_shift = is_xmm_shift + is_xmm_bits
, reg1_shift = data_shift
@@ -268,12 +265,11 @@ class LIR_Opr {
, type_mask = right_n_bits(type_bits) << type_shift
, size_mask = right_n_bits(size_bits) << size_shift
, last_use_mask = right_n_bits(last_use_bits) << last_use_shift
- , is_fpu_stack_offset_mask = right_n_bits(is_fpu_stack_offset_bits) << is_fpu_stack_offset_shift
, virtual_mask = right_n_bits(virtual_bits) << virtual_shift
, is_xmm_mask = right_n_bits(is_xmm_bits) << is_xmm_shift
, pointer_mask = right_n_bits(pointer_bits)
, lower_reg_mask = right_n_bits(reg_bits)
- , no_type_mask = (int)(~(type_mask | last_use_mask | is_fpu_stack_offset_mask))
+ , no_type_mask = (int)(~(type_mask | last_use_mask))
};
uint32_t data() const { return (uint32_t)value() >> data_shift; }
@@ -426,9 +422,7 @@ class LIR_Opr {
BasicType type_register() const { assert(is_register() || is_stack(), "type check"); return as_BasicType(type_field_valid()); }
bool is_last_use() const { assert(is_register(), "only works for registers"); return (value() & last_use_mask) != 0; }
- bool is_fpu_stack_offset() const { assert(is_register(), "only works for registers"); return (value() & is_fpu_stack_offset_mask) != 0; }
LIR_Opr make_last_use() { assert(is_register(), "only works for registers"); return (LIR_Opr)(value() | last_use_mask); }
- LIR_Opr make_fpu_stack_offset() { assert(is_register(), "only works for registers"); return (LIR_Opr)(value() | is_fpu_stack_offset_mask); }
int single_stack_ix() const { assert(is_single_stack() && !is_virtual(), "type check"); return (int)data(); }
@@ -884,7 +878,6 @@ class LIR_OpBranch;
class LIR_OpConvert;
class LIR_OpAllocObj;
class LIR_OpReturn;
-class LIR_OpRoundFP;
class LIR_Op2;
class LIR_OpDelay;
class LIR_Op3;
@@ -913,7 +906,6 @@ enum LIR_Code {
, lir_nop
, lir_std_entry
, lir_osr_entry
- , lir_fpop_raw
, lir_breakpoint
, lir_rtcall
, lir_membar
@@ -927,8 +919,6 @@ enum LIR_Code {
, lir_on_spin_wait
, end_op0
, begin_op1
- , lir_fxch
- , lir_fld
, lir_push
, lir_pop
, lir_null_check
@@ -938,7 +928,6 @@ enum LIR_Code {
, lir_convert
, lir_alloc_object
, lir_monaddr
- , lir_roundfp
, lir_sqrt
, lir_abs
, lir_neg
@@ -1072,7 +1061,6 @@ class LIR_Op: public CompilationResourceObj {
unsigned short _flags;
CodeEmitInfo* _info;
int _id; // value id for register allocation
- int _fpu_pop_count;
Instruction* _source; // for debugging
static void print_condition(outputStream* out, LIR_Condition cond) PRODUCT_RETURN;
@@ -1092,7 +1080,6 @@ class LIR_Op: public CompilationResourceObj {
, _flags(0)
, _info(nullptr)
, _id(-1)
- , _fpu_pop_count(0)
, _source(nullptr) {}
LIR_Op(LIR_Code code, LIR_Opr result, CodeEmitInfo* info)
@@ -1106,7 +1093,6 @@ class LIR_Op: public CompilationResourceObj {
, _flags(0)
, _info(info)
, _id(-1)
- , _fpu_pop_count(0)
, _source(nullptr) {}
CodeEmitInfo* info() const { return _info; }
@@ -1127,11 +1113,6 @@ class LIR_Op: public CompilationResourceObj {
int id() const { return _id; }
void set_id(int id) { _id = id; }
- // FPU stack simulation helpers -- only used on Intel
- void set_fpu_pop_count(int count) { assert(count >= 0 && count <= 1, "currently only 0 and 1 are valid"); _fpu_pop_count = count; }
- int fpu_pop_count() const { return _fpu_pop_count; }
- bool pop_fpu_stack() { return _fpu_pop_count > 0; }
-
Instruction* source() const { return _source; }
void set_source(Instruction* ins) { _source = ins; }
@@ -1147,7 +1128,6 @@ class LIR_Op: public CompilationResourceObj {
virtual LIR_OpLock* as_OpLock() { return nullptr; }
virtual LIR_OpAllocArray* as_OpAllocArray() { return nullptr; }
virtual LIR_OpAllocObj* as_OpAllocObj() { return nullptr; }
- virtual LIR_OpRoundFP* as_OpRoundFP() { return nullptr; }
virtual LIR_OpBranch* as_OpBranch() { return nullptr; }
virtual LIR_OpReturn* as_OpReturn() { return nullptr; }
virtual LIR_OpRTCall* as_OpRTCall() { return nullptr; }
@@ -1527,23 +1507,6 @@ class LIR_OpAllocObj : public LIR_Op1 {
};
-// LIR_OpRoundFP
-class LIR_OpRoundFP : public LIR_Op1 {
- friend class LIR_OpVisitState;
-
- private:
- LIR_Opr _tmp;
-
- public:
- LIR_OpRoundFP(LIR_Opr reg, LIR_Opr stack_loc_temp, LIR_Opr result)
- : LIR_Op1(lir_roundfp, reg, result)
- , _tmp(stack_loc_temp) {}
-
- LIR_Opr tmp() const { return _tmp; }
- virtual LIR_OpRoundFP* as_OpRoundFP() { return this; }
- void print_instr(outputStream* out) const PRODUCT_RETURN;
-};
-
// LIR_OpTypeCheck
class LIR_OpTypeCheck: public LIR_Op {
friend class LIR_OpVisitState;
@@ -2205,7 +2168,6 @@ class LIR_List: public CompilationResourceObj {
// result is a stack location for old backend and vreg for UseLinearScan
// stack_loc_temp is an illegal register for old backend
- void roundfp(LIR_Opr reg, LIR_Opr stack_loc_temp, LIR_Opr result) { append(new LIR_OpRoundFP(reg, stack_loc_temp, result)); }
void move(LIR_Opr src, LIR_Opr dst, CodeEmitInfo* info = nullptr) { append(new LIR_Op1(lir_move, src, dst, dst->type(), lir_patch_none, info)); }
void move(LIR_Address* src, LIR_Opr dst, CodeEmitInfo* info = nullptr) { append(new LIR_Op1(lir_move, LIR_OprFact::address(src), dst, src->type(), lir_patch_none, info)); }
void move(LIR_Opr src, LIR_Address* dst, CodeEmitInfo* info = nullptr) { append(new LIR_Op1(lir_move, src, LIR_OprFact::address(dst), dst->type(), lir_patch_none, info)); }
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.cpp b/src/hotspot/share/c1/c1_LIRAssembler.cpp
index a5930ba54d8..7cf414ae7dc 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.cpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.cpp
@@ -485,19 +485,6 @@ void LIR_Assembler::emit_call(LIR_OpJavaCall* op) {
if (op->is_method_handle_invoke()) {
compilation()->set_has_method_handle_invokes(true);
}
-
-#if defined(IA32) && defined(COMPILER2)
- // C2 leave fpu stack dirty clean it
- if (UseSSE < 2 && !CompilerConfig::is_c1_only_no_jvmci()) {
- int i;
- for ( i = 1; i <= 7 ; i++ ) {
- ffree(i);
- }
- if (!op->result_opr()->is_float_kind()) {
- ffree(0);
- }
- }
-#endif // IA32 && COMPILER2
}
@@ -514,17 +501,11 @@ void LIR_Assembler::emit_op1(LIR_Op1* op) {
volatile_move_op(op->in_opr(), op->result_opr(), op->type(), op->info());
} else {
move_op(op->in_opr(), op->result_opr(), op->type(),
- op->patch_code(), op->info(), op->pop_fpu_stack(),
+ op->patch_code(), op->info(),
op->move_kind() == lir_move_wide);
}
break;
- case lir_roundfp: {
- LIR_OpRoundFP* round_op = op->as_OpRoundFP();
- roundfp_op(round_op->in_opr(), round_op->tmp(), round_op->result_opr(), round_op->pop_fpu_stack());
- break;
- }
-
case lir_abs:
case lir_sqrt:
case lir_f2hf:
@@ -723,14 +704,12 @@ void LIR_Assembler::emit_op2(LIR_Op2* op) {
case lir_mul:
case lir_div:
case lir_rem:
- assert(op->fpu_pop_count() < 2, "");
arith_op(
op->code(),
op->in_opr1(),
op->in_opr2(),
op->result_opr(),
- op->info(),
- op->fpu_pop_count() == 1);
+ op->info());
break;
case lir_logic_and:
@@ -775,26 +754,16 @@ void LIR_Assembler::build_frame() {
}
-void LIR_Assembler::roundfp_op(LIR_Opr src, LIR_Opr tmp, LIR_Opr dest, bool pop_fpu_stack) {
- assert(strict_fp_requires_explicit_rounding, "not required");
- assert((src->is_single_fpu() && dest->is_single_stack()) ||
- (src->is_double_fpu() && dest->is_double_stack()),
- "round_fp: rounds register -> stack location");
-
- reg2stack (src, dest, src->type(), pop_fpu_stack);
-}
-
-
-void LIR_Assembler::move_op(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
+void LIR_Assembler::move_op(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
if (src->is_register()) {
if (dest->is_register()) {
assert(patch_code == lir_patch_none && info == nullptr, "no patching and info allowed here");
reg2reg(src, dest);
} else if (dest->is_stack()) {
assert(patch_code == lir_patch_none && info == nullptr, "no patching and info allowed here");
- reg2stack(src, dest, type, pop_fpu_stack);
+ reg2stack(src, dest, type);
} else if (dest->is_address()) {
- reg2mem(src, dest, type, patch_code, info, pop_fpu_stack, wide);
+ reg2mem(src, dest, type, patch_code, info, wide);
} else {
ShouldNotReachHere();
}
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp
index 34aa679daed..a4c5fd61d4c 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.hpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp
@@ -166,11 +166,11 @@ class LIR_Assembler: public CompilationResourceObj {
void const2reg (LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info);
void const2stack(LIR_Opr src, LIR_Opr dest);
void const2mem (LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide);
- void reg2stack (LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack);
+ void reg2stack (LIR_Opr src, LIR_Opr dest, BasicType type);
void reg2reg (LIR_Opr src, LIR_Opr dest);
void reg2mem (LIR_Opr src, LIR_Opr dest, BasicType type,
LIR_PatchCode patch_code, CodeEmitInfo* info,
- bool pop_fpu_stack, bool wide);
+ bool wide);
void stack2reg (LIR_Opr src, LIR_Opr dest, BasicType type);
void stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type);
void mem2reg (LIR_Opr src, LIR_Opr dest, BasicType type,
@@ -206,7 +206,7 @@ class LIR_Assembler: public CompilationResourceObj {
void emit_profile_type(LIR_OpProfileType* op);
void emit_delay(LIR_OpDelay* op);
- void arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack);
+ void arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info);
void arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info);
void intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr temp, LIR_Opr dest, LIR_Op* op);
#ifdef ASSERT
@@ -215,9 +215,8 @@ class LIR_Assembler: public CompilationResourceObj {
void logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest);
- void roundfp_op(LIR_Opr src, LIR_Opr tmp, LIR_Opr dest, bool pop_fpu_stack);
void move_op(LIR_Opr src, LIR_Opr result, BasicType type,
- LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide);
+ LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide);
void volatile_move_op(LIR_Opr src, LIR_Opr result, BasicType type, CodeEmitInfo* info);
void comp_mem_op(LIR_Opr src, LIR_Opr result, BasicType type, CodeEmitInfo* info); // info set for null exceptions
void comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr result, LIR_Op2* op);
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.cpp b/src/hotspot/share/c1/c1_LIRGenerator.cpp
index 959e49749c5..214da537993 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.cpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.cpp
@@ -881,27 +881,6 @@ void LIRGenerator::arraycopy_helper(Intrinsic* x, int* flagsp, ciArrayKlass** ex
}
-LIR_Opr LIRGenerator::round_item(LIR_Opr opr) {
- assert(opr->is_register(), "why spill if item is not register?");
-
- if (strict_fp_requires_explicit_rounding) {
-#ifdef IA32
- if (UseSSE < 1 && opr->is_single_fpu()) {
- LIR_Opr result = new_register(T_FLOAT);
- set_vreg_flag(result, must_start_in_memory);
- assert(opr->is_register(), "only a register can be spilled");
- assert(opr->value_type()->is_float(), "rounding only for floats available");
- __ roundfp(opr, LIR_OprFact::illegalOpr, result);
- return result;
- }
-#else
- Unimplemented();
-#endif // IA32
- }
- return opr;
-}
-
-
LIR_Opr LIRGenerator::force_to_spill(LIR_Opr value, BasicType t) {
assert(type2size[t] == type2size[value->type()],
"size mismatch: t=%s, value->type()=%s", type2name(t), type2name(value->type()));
@@ -2053,25 +2032,6 @@ void LIRGenerator::do_Throw(Throw* x) {
}
-void LIRGenerator::do_RoundFP(RoundFP* x) {
- assert(strict_fp_requires_explicit_rounding, "not required");
-
- LIRItem input(x->input(), this);
- input.load_item();
- LIR_Opr input_opr = input.result();
- assert(input_opr->is_register(), "why round if value is not in a register?");
- assert(input_opr->is_single_fpu() || input_opr->is_double_fpu(), "input should be floating-point value");
- if (input_opr->is_single_fpu()) {
- set_result(x, round_item(input_opr)); // This code path not currently taken
- } else {
- LIR_Opr result = new_register(T_DOUBLE);
- set_vreg_flag(result, must_start_in_memory);
- __ roundfp(input_opr, LIR_OprFact::illegalOpr, result);
- set_result(x, result);
- }
-}
-
-
void LIRGenerator::do_UnsafeGet(UnsafeGet* x) {
BasicType type = x->basic_type();
LIRItem src(x->object(), this);
diff --git a/src/hotspot/share/c1/c1_LIRGenerator.hpp b/src/hotspot/share/c1/c1_LIRGenerator.hpp
index 73bd883a746..e70bbd96189 100644
--- a/src/hotspot/share/c1/c1_LIRGenerator.hpp
+++ b/src/hotspot/share/c1/c1_LIRGenerator.hpp
@@ -233,7 +233,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
friend class LIRItem;
- LIR_Opr round_item(LIR_Opr opr);
LIR_Opr force_to_spill(LIR_Opr value, BasicType t);
PhiResolverState& resolver_state() { return _resolver_state; }
@@ -579,7 +578,6 @@ class LIRGenerator: public InstructionVisitor, public BlockClosure {
virtual void do_Base (Base* x);
virtual void do_OsrEntry (OsrEntry* x);
virtual void do_ExceptionObject(ExceptionObject* x);
- virtual void do_RoundFP (RoundFP* x);
virtual void do_UnsafeGet (UnsafeGet* x);
virtual void do_UnsafePut (UnsafePut* x);
virtual void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
diff --git a/src/hotspot/share/c1/c1_LinearScan.cpp b/src/hotspot/share/c1/c1_LinearScan.cpp
index c099bb47d97..3736284892f 100644
--- a/src/hotspot/share/c1/c1_LinearScan.cpp
+++ b/src/hotspot/share/c1/c1_LinearScan.cpp
@@ -27,8 +27,8 @@
#include "c1/c1_Compilation.hpp"
#include "c1/c1_FrameMap.hpp"
#include "c1/c1_IR.hpp"
-#include "c1/c1_LIRGenerator.hpp"
#include "c1/c1_LinearScan.hpp"
+#include "c1/c1_LIRGenerator.hpp"
#include "c1/c1_ValueStack.hpp"
#include "code/vmreg.inline.hpp"
#include "runtime/timerTrace.hpp"
@@ -91,9 +91,6 @@ LinearScan::LinearScan(IR* ir, LIRGenerator* gen, FrameMap* frame_map)
, _has_call(0)
, _interval_in_loop(0) // initialized later with correct length
, _scope_value_cache(0) // initialized later with correct length
-#ifdef IA32
- , _fpu_stack_allocator(nullptr)
-#endif
{
assert(this->ir() != nullptr, "check if valid");
assert(this->compilation() != nullptr, "check if valid");
@@ -1868,15 +1865,12 @@ void LinearScan::resolve_exception_entry(BlockBegin* block, int reg_num, MoveRes
int reg = interval->assigned_reg();
int regHi = interval->assigned_regHi();
- if ((reg < nof_regs && interval->always_in_memory()) ||
- (use_fpu_stack_allocation() && reg >= pd_first_fpu_reg && reg <= pd_last_fpu_reg)) {
+ if ((reg < nof_regs && interval->always_in_memory())) {
// the interval is split to get a short range that is located on the stack
- // in the following two cases:
+ // in the following case:
// * the interval started in memory (e.g. method parameter), but is currently in a register
// this is an optimization for exception handling that reduces the number of moves that
// are necessary for resolving the states when an exception uses this exception handler
- // * the interval would be on the fpu stack at the begin of the exception handler
- // this is not allowed because of the complicated fpu stack handling on Intel
// range that will be spilled to memory
int from_op_id = block->first_lir_instruction_id();
@@ -2665,17 +2659,9 @@ int LinearScan::append_scope_value_for_operand(LIR_Opr opr, GrowableArrayis_single_fpu()) {
-#ifdef IA32
- // the exact location of fpu stack values is only known
- // during fpu stack allocation, so the stack allocator object
- // must be present
- assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)");
- assert(_fpu_stack_allocator != nullptr, "must be present");
- opr = _fpu_stack_allocator->to_fpu_stack(opr);
-#elif defined(AMD64)
+#if defined(AMD64)
assert(false, "FPU not used on x86-64");
#endif
-
Location::Type loc_type = float_saved_as_double ? Location::float_in_dbl : Location::normal;
VMReg rname = frame_map()->fpu_regname(opr->fpu_regnr());
#ifndef __SOFTFP__
@@ -2778,16 +2764,6 @@ int LinearScan::append_scope_value_for_operand(LIR_Opr opr, GrowableArrayto_fpu_stack(opr);
-
- assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation (only fpu_regnrLo is used)");
-#endif
#ifdef AMD64
assert(false, "FPU not used on x86-64");
#endif
@@ -3021,15 +2997,9 @@ void LinearScan::assign_reg_num(LIR_OpList* instructions, IntervalWalker* iw) {
compute_oop_map(iw, visitor, op);
// compute debug information
- if (!use_fpu_stack_allocation()) {
- // compute debug information if fpu stack allocation is not needed.
- // when fpu stack allocation is needed, the debug information can not
- // be computed here because the exact location of fpu operands is not known
- // -> debug information is created inside the fpu stack allocator
- int n = visitor.info_count();
- for (int k = 0; k < n; k++) {
- compute_debug_info(visitor.info_at(k), op_id);
- }
+ int n = visitor.info_count();
+ for (int k = 0; k < n; k++) {
+ compute_debug_info(visitor.info_at(k), op_id);
}
}
@@ -3126,14 +3096,6 @@ void LinearScan::do_linear_scan() {
NOT_PRODUCT(print_lir(2, "LIR after assignment of register numbers:"));
NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_after_asign));
- { TIME_LINEAR_SCAN(timer_allocate_fpu_stack);
-
- if (use_fpu_stack_allocation()) {
- allocate_fpu_stack(); // Only has effect on Intel
- NOT_PRODUCT(print_lir(2, "LIR after FPU stack allocation:"));
- }
- }
-
#ifndef RISCV
// Disable these optimizations on riscv temporarily, because it does not
// work when the comparison operands are bound to branches or cmoves.
@@ -5888,7 +5850,6 @@ bool LinearScanWalker::activate_current() {
} else if (allocator()->gen()->is_vreg_flag_set(cur->reg_num(), LIRGenerator::must_start_in_memory)) {
// activating an interval that must start in a stack slot, but may get a register later
- // used for lir_roundfp: rounding is done by store to stack and reload later
TRACE_LINEAR_SCAN(4, tty->print_cr(" interval must start in stack slot -> split it before first use"));
assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned");
@@ -6008,20 +5969,6 @@ bool EdgeMoveOptimizer::operations_different(LIR_Op* op1, LIR_Op* op2) {
// these moves are exactly equal and can be optimized
return false;
}
-
- } else if (op1->code() == lir_fxch && op2->code() == lir_fxch) {
- assert(op1->as_Op1() != nullptr, "fxch must be LIR_Op1");
- assert(op2->as_Op1() != nullptr, "fxch must be LIR_Op1");
- LIR_Op1* fxch1 = (LIR_Op1*)op1;
- LIR_Op1* fxch2 = (LIR_Op1*)op2;
- if (fxch1->in_opr()->as_jint() == fxch2->in_opr()->as_jint()) {
- // equal FPU stack operations can be optimized
- return false;
- }
-
- } else if (op1->code() == lir_fpop_raw && op2->code() == lir_fpop_raw) {
- // equal FPU stack operations can be optimized
- return false;
}
// no optimization possible
@@ -6541,7 +6488,6 @@ const char* LinearScanStatistic::counter_name(int counter_idx) {
case counter_throw: return "throw";
case counter_unwind: return "unwind";
case counter_typecheck: return "type+null-checks";
- case counter_fpu_stack: return "fpu-stack";
case counter_misc_inst: return "other instructions";
case counter_other_inst: return "misc. instructions";
@@ -6762,15 +6708,10 @@ void LinearScanStatistic::collect(LinearScan* allocator) {
case lir_checkcast:
case lir_store_check: inc_counter(counter_typecheck); break;
- case lir_fpop_raw:
- case lir_fxch:
- case lir_fld: inc_counter(counter_fpu_stack); break;
-
case lir_nop:
case lir_push:
case lir_pop:
case lir_convert:
- case lir_roundfp:
case lir_cmove: inc_counter(counter_misc_inst); break;
default: inc_counter(counter_other_inst); break;
@@ -6819,7 +6760,6 @@ const char* LinearScanTimers::timer_name(int idx) {
case timer_sort_intervals_after: return "Sort Intervals After";
case timer_eliminate_spill_moves: return "Spill optimization";
case timer_assign_reg_num: return "Assign Reg Num";
- case timer_allocate_fpu_stack: return "Allocate FPU Stack";
case timer_optimize_lir: return "Optimize LIR";
default: ShouldNotReachHere(); return "";
}
diff --git a/src/hotspot/share/c1/c1_LinearScan.hpp b/src/hotspot/share/c1/c1_LinearScan.hpp
index 4e8adef3c7d..36d84a4fad9 100644
--- a/src/hotspot/share/c1/c1_LinearScan.hpp
+++ b/src/hotspot/share/c1/c1_LinearScan.hpp
@@ -25,10 +25,9 @@
#ifndef SHARE_C1_C1_LINEARSCAN_HPP
#define SHARE_C1_C1_LINEARSCAN_HPP
-#include "c1/c1_FpuStackSim.hpp"
#include "c1/c1_FrameMap.hpp"
-#include "c1/c1_IR.hpp"
#include "c1/c1_Instruction.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_LIR.hpp"
#include "c1/c1_LIRGenerator.hpp"
#include "compiler/oopMap.hpp"
@@ -177,15 +176,6 @@ class LinearScan : public CompilationResourceObj {
int num_loops() const { return ir()->num_loops(); }
bool is_interval_in_loop(int interval, int loop) const { return _interval_in_loop.at(interval, loop); }
- // handling of fpu stack allocation (platform dependent, needed for debug information generation)
-#ifdef IA32
- FpuStackAllocator* _fpu_stack_allocator;
- bool use_fpu_stack_allocation() const { return UseSSE < 2 && has_fpu_registers(); }
-#else
- bool use_fpu_stack_allocation() const { return false; }
-#endif
-
-
// access to interval list
int interval_count() const { return _intervals.length(); }
Interval* interval_at(int reg_num) const { return _intervals.at(reg_num); }
@@ -357,12 +347,6 @@ class LinearScan : public CompilationResourceObj {
void assign_reg_num(LIR_OpList* instructions, IntervalWalker* iw);
void assign_reg_num();
-
- // Phase 8: fpu stack allocation
- // (Used only on x86 when fpu operands are present)
- void allocate_fpu_stack();
-
-
// helper functions for printing state
#ifndef PRODUCT
static void print_bitmap(BitMap& bitmap);
@@ -953,7 +937,6 @@ class LinearScanTimers : public StackObj {
timer_sort_intervals_after,
timer_eliminate_spill_moves,
timer_assign_reg_num,
- timer_allocate_fpu_stack,
timer_optimize_lir,
number_of_timers
diff --git a/src/hotspot/share/c1/c1_Optimizer.cpp b/src/hotspot/share/c1/c1_Optimizer.cpp
index f8339b00049..0c18694df78 100644
--- a/src/hotspot/share/c1/c1_Optimizer.cpp
+++ b/src/hotspot/share/c1/c1_Optimizer.cpp
@@ -25,9 +25,9 @@
#include "c1/c1_Optimizer.hpp"
#include "c1/c1_ValueSet.hpp"
#include "c1/c1_ValueStack.hpp"
+#include "compiler/compileLog.hpp"
#include "memory/resourceArea.hpp"
#include "utilities/bitMap.inline.hpp"
-#include "compiler/compileLog.hpp"
typedef GrowableArray ValueSetList;
@@ -577,7 +577,6 @@ public:
void do_Base (Base* x);
void do_OsrEntry (OsrEntry* x);
void do_ExceptionObject(ExceptionObject* x);
- void do_RoundFP (RoundFP* x);
void do_UnsafeGet (UnsafeGet* x);
void do_UnsafePut (UnsafePut* x);
void do_UnsafeGetAndSet(UnsafeGetAndSet* x);
@@ -762,7 +761,6 @@ void NullCheckVisitor::do_Throw (Throw* x) { nce()->clear_las
void NullCheckVisitor::do_Base (Base* x) {}
void NullCheckVisitor::do_OsrEntry (OsrEntry* x) {}
void NullCheckVisitor::do_ExceptionObject(ExceptionObject* x) { nce()->handle_ExceptionObject(x); }
-void NullCheckVisitor::do_RoundFP (RoundFP* x) {}
void NullCheckVisitor::do_UnsafeGet (UnsafeGet* x) {}
void NullCheckVisitor::do_UnsafePut (UnsafePut* x) {}
void NullCheckVisitor::do_UnsafeGetAndSet(UnsafeGetAndSet* x) {}
diff --git a/src/hotspot/share/c1/c1_Optimizer.hpp b/src/hotspot/share/c1/c1_Optimizer.hpp
index 542ee8acbd0..c26b7534bca 100644
--- a/src/hotspot/share/c1/c1_Optimizer.hpp
+++ b/src/hotspot/share/c1/c1_Optimizer.hpp
@@ -25,8 +25,8 @@
#ifndef SHARE_C1_C1_OPTIMIZER_HPP
#define SHARE_C1_C1_OPTIMIZER_HPP
-#include "c1/c1_IR.hpp"
#include "c1/c1_Instruction.hpp"
+#include "c1/c1_IR.hpp"
class Optimizer {
private:
diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
index 6320fc15efe..1adf99cce61 100644
--- a/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
+++ b/src/hotspot/share/c1/c1_RangeCheckElimination.cpp
@@ -22,9 +22,9 @@
*
*/
-#include "c1/c1_ValueStack.hpp"
-#include "c1/c1_RangeCheckElimination.hpp"
#include "c1/c1_IR.hpp"
+#include "c1/c1_RangeCheckElimination.hpp"
+#include "c1/c1_ValueStack.hpp"
#include "ci/ciMethodData.hpp"
#include "runtime/deoptimization.hpp"
#include "utilities/bitMap.inline.hpp"
diff --git a/src/hotspot/share/c1/c1_RangeCheckElimination.hpp b/src/hotspot/share/c1/c1_RangeCheckElimination.hpp
index 371dc59714b..833f5dd1e99 100644
--- a/src/hotspot/share/c1/c1_RangeCheckElimination.hpp
+++ b/src/hotspot/share/c1/c1_RangeCheckElimination.hpp
@@ -154,7 +154,6 @@ public:
void do_Base (Base* x) { /* nothing to do */ };
void do_OsrEntry (OsrEntry* x) { /* nothing to do */ };
void do_ExceptionObject(ExceptionObject* x) { /* nothing to do */ };
- void do_RoundFP (RoundFP* x) { /* nothing to do */ };
void do_UnsafePut (UnsafePut* x) { /* nothing to do */ };
void do_UnsafeGet (UnsafeGet* x) { /* nothing to do */ };
void do_UnsafeGetAndSet(UnsafeGetAndSet* x) { /* nothing to do */ };
diff --git a/src/hotspot/share/c1/c1_Runtime1.cpp b/src/hotspot/share/c1/c1_Runtime1.cpp
index 3a30c9846aa..d7a9c8d56d4 100644
--- a/src/hotspot/share/c1/c1_Runtime1.cpp
+++ b/src/hotspot/share/c1/c1_Runtime1.cpp
@@ -49,8 +49,8 @@
#include "memory/resourceArea.hpp"
#include "memory/universe.hpp"
#include "oops/access.inline.hpp"
-#include "oops/objArrayOop.inline.hpp"
#include "oops/objArrayKlass.hpp"
+#include "oops/objArrayOop.inline.hpp"
#include "oops/oop.inline.hpp"
#include "prims/jvmtiExport.hpp"
#include "runtime/atomic.hpp"
@@ -229,12 +229,12 @@ CodeBlob* Runtime1::generate_blob(BufferBlob* buffer_blob, C1StubId id, const ch
CodeOffsets::frame_never_safe,
frame_size,
oop_maps,
- must_gc_arguments);
- assert(blob != nullptr, "blob must exist");
+ must_gc_arguments,
+ false /* alloc_fail_is_fatal */ );
return blob;
}
-void Runtime1::generate_blob_for(BufferBlob* buffer_blob, C1StubId id) {
+bool Runtime1::generate_blob_for(BufferBlob* buffer_blob, C1StubId id) {
assert(C1StubId::NO_STUBID < id && id < C1StubId::NUM_STUBIDS, "illegal stub id");
bool expect_oop_map = true;
#ifdef ASSERT
@@ -257,14 +257,19 @@ void Runtime1::generate_blob_for(BufferBlob* buffer_blob, C1StubId id) {
CodeBlob* blob = generate_blob(buffer_blob, id, name_for(id), expect_oop_map, &cl);
// install blob
_blobs[(int)id] = blob;
+ return blob != nullptr;
}
-void Runtime1::initialize(BufferBlob* blob) {
+bool Runtime1::initialize(BufferBlob* blob) {
// platform-dependent initialization
initialize_pd();
// generate stubs
int limit = (int)C1StubId::NUM_STUBIDS;
- for (int id = 0; id < limit; id++) generate_blob_for(blob, (C1StubId)id);
+ for (int id = 0; id < limit; id++) {
+ if (!generate_blob_for(blob, (C1StubId) id)) {
+ return false;
+ }
+ }
// printing
#ifndef PRODUCT
if (PrintSimpleStubs) {
@@ -278,7 +283,7 @@ void Runtime1::initialize(BufferBlob* blob) {
}
#endif
BarrierSetC1* bs = BarrierSet::barrier_set()->barrier_set_c1();
- bs->generate_c1_runtime_stubs(blob);
+ return bs->generate_c1_runtime_stubs(blob);
}
CodeBlob* Runtime1::blob_for(C1StubId id) {
@@ -795,7 +800,7 @@ JRT_ENTRY(void, Runtime1::deoptimize(JavaThread* current, jint trap_request))
Deoptimization::DeoptReason reason = Deoptimization::trap_request_reason(trap_request);
if (action == Deoptimization::Action_make_not_entrant) {
- if (nm->make_not_entrant()) {
+ if (nm->make_not_entrant("C1 deoptimize")) {
if (reason == Deoptimization::Reason_tenured) {
MethodData* trap_mdo = Deoptimization::get_method_data(current, method, true /*create_if_missing*/);
if (trap_mdo != nullptr) {
@@ -1087,7 +1092,7 @@ JRT_ENTRY(void, Runtime1::patch_code(JavaThread* current, C1StubId stub_id ))
// safepoint, but if it's still alive then make it not_entrant.
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
if (nm != nullptr) {
- nm->make_not_entrant();
+ nm->make_not_entrant("C1 code patch");
}
Deoptimization::deoptimize_frame(current, caller_frame.id());
@@ -1335,7 +1340,7 @@ void Runtime1::patch_code(JavaThread* current, C1StubId stub_id) {
// Make sure the nmethod is invalidated, i.e. made not entrant.
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
if (nm != nullptr) {
- nm->make_not_entrant();
+ nm->make_not_entrant("C1 deoptimize for patching");
}
}
@@ -1463,7 +1468,7 @@ JRT_ENTRY(void, Runtime1::predicate_failed_trap(JavaThread* current))
nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
assert (nm != nullptr, "no more nmethod?");
- nm->make_not_entrant();
+ nm->make_not_entrant("C1 predicate failed trap");
methodHandle m(current, nm->method());
MethodData* mdo = m->method_data();
diff --git a/src/hotspot/share/c1/c1_Runtime1.hpp b/src/hotspot/share/c1/c1_Runtime1.hpp
index 5f1ae4333bc..c09de00ce55 100644
--- a/src/hotspot/share/c1/c1_Runtime1.hpp
+++ b/src/hotspot/share/c1/c1_Runtime1.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -53,7 +53,6 @@ enum class C1StubId :int {
#undef C1_STUB_ID_ENUM_DECLARE
class Runtime1: public AllStatic {
- friend class VMStructs;
friend class ArrayCopyStub;
public:
@@ -86,7 +85,7 @@ public:
// stub generation
public:
static CodeBlob* generate_blob(BufferBlob* buffer_blob, C1StubId id, const char* name, bool expect_oop_map, StubAssemblerCodeGenClosure *cl);
- static void generate_blob_for(BufferBlob* blob, C1StubId id);
+ static bool generate_blob_for(BufferBlob* blob, C1StubId id);
static OopMapSet* generate_code_for(C1StubId id, StubAssembler* sasm);
private:
static OopMapSet* generate_exception_throw(StubAssembler* sasm, address target, bool has_argument);
@@ -131,7 +130,7 @@ public:
public:
// initialization
- static void initialize(BufferBlob* blob);
+ static bool initialize(BufferBlob* blob);
static void initialize_pd();
// return offset in words
diff --git a/src/hotspot/share/c1/c1_ValueMap.hpp b/src/hotspot/share/c1/c1_ValueMap.hpp
index c36bb5559ba..12c372f27c8 100644
--- a/src/hotspot/share/c1/c1_ValueMap.hpp
+++ b/src/hotspot/share/c1/c1_ValueMap.hpp
@@ -203,7 +203,6 @@ class ValueNumberingVisitor: public InstructionVisitor {
void do_Base (Base* x) { /* nothing to do */ }
void do_OsrEntry (OsrEntry* x) { /* nothing to do */ }
void do_ExceptionObject(ExceptionObject* x) { /* nothing to do */ }
- void do_RoundFP (RoundFP* x) { /* nothing to do */ }
void do_ProfileCall (ProfileCall* x) { /* nothing to do */ }
void do_ProfileReturnType (ProfileReturnType* x) { /* nothing to do */ }
void do_ProfileInvoke (ProfileInvoke* x) { /* nothing to do */ };
diff --git a/src/hotspot/share/c1/c1_ValueStack.cpp b/src/hotspot/share/c1/c1_ValueStack.cpp
index f1ac940bf16..9a09c186541 100644
--- a/src/hotspot/share/c1/c1_ValueStack.cpp
+++ b/src/hotspot/share/c1/c1_ValueStack.cpp
@@ -22,8 +22,8 @@
*
*/
-#include "c1/c1_IR.hpp"
#include "c1/c1_InstructionPrinter.hpp"
+#include "c1/c1_IR.hpp"
#include "c1/c1_ValueStack.hpp"
diff --git a/src/hotspot/share/c1/c1_globals.hpp b/src/hotspot/share/c1/c1_globals.hpp
index 72306585355..ec01bc7a790 100644
--- a/src/hotspot/share/c1/c1_globals.hpp
+++ b/src/hotspot/share/c1/c1_globals.hpp
@@ -280,12 +280,6 @@
"Maximum size of a compiled method.") \
range(0, 1*M) \
\
- develop(bool, TraceFPUStack, false, \
- "Trace emulation of the FPU stack (intel only)") \
- \
- develop(bool, TraceFPURegisterUsage, false, \
- "Trace usage of FPU registers at start of blocks (intel only)") \
- \
develop(intx, InstructionCountCutoff, 37000, \
"If GraphBuilder adds this many instructions, bails out") \
range(0, max_jint) \
diff --git a/src/hotspot/share/cds/aotArtifactFinder.cpp b/src/hotspot/share/cds/aotArtifactFinder.cpp
index 48f31635c63..6a7faee5133 100644
--- a/src/hotspot/share/cds/aotArtifactFinder.cpp
+++ b/src/hotspot/share/cds/aotArtifactFinder.cpp
@@ -27,6 +27,7 @@
#include "cds/aotClassInitializer.hpp"
#include "cds/dumpTimeClassInfo.inline.hpp"
#include "cds/heapShared.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
#include "classfile/systemDictionaryShared.hpp"
#include "logging/log.hpp"
#include "memory/metaspaceClosure.hpp"
@@ -108,19 +109,22 @@ void AOTArtifactFinder::find_artifacts() {
// Add all the InstanceKlasses (and their array classes) that are always included.
SystemDictionaryShared::dumptime_table()->iterate_all_live_classes([&] (InstanceKlass* ik, DumpTimeClassInfo& info) {
- if (!info.is_excluded()) {
+ // Skip "AOT tooling classes" in this block. They will be included in the AOT cache only if
+ // - One of their subtypes is included
+ // - One of their instances is found by HeapShared.
+ if (!info.is_excluded() && !info.is_aot_tooling_class()) {
bool add = false;
if (!ik->is_hidden()) {
// All non-hidden classes are always included into the AOT cache
add = true;
} else {
- if (!CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
// Legacy support of lambda proxies -- these are always included into the AOT cache
- if (SystemDictionaryShared::is_registered_lambda_proxy_class(ik)) {
+ if (LambdaProxyClassDictionary::is_registered_lambda_proxy_class(ik)) {
add = true;
}
} else {
- assert(!SystemDictionaryShared::is_registered_lambda_proxy_class(ik),
+ assert(!LambdaProxyClassDictionary::is_registered_lambda_proxy_class(ik),
"registered lambda proxies are only for legacy lambda proxy support");
}
}
@@ -148,10 +152,10 @@ void AOTArtifactFinder::find_artifacts() {
SystemDictionaryShared::dumptime_table()->iterate_all_live_classes([&] (InstanceKlass* k, DumpTimeClassInfo& info) {
if (!info.is_excluded() && _seen_classes->get(k) == nullptr) {
info.set_excluded();
- assert(k->is_hidden(), "must be");
if (log_is_enabled(Info, cds)) {
ResourceMark rm;
- log_info(cds)("Skipping %s: Hidden class", k->name()->as_C_string());
+ log_info(cds)("Skipping %s: %s class", k->name()->as_C_string(),
+ k->is_hidden() ? "Hidden" : "AOT tooling");
}
}
});
diff --git a/src/hotspot/share/cds/aotClassFilter.cpp b/src/hotspot/share/cds/aotClassFilter.cpp
new file mode 100644
index 00000000000..959bea6a623
--- /dev/null
+++ b/src/hotspot/share/cds/aotClassFilter.cpp
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "cds/aotClassFilter.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/mutexLocker.hpp"
+
+AOTClassFilter::FilterMark* AOTClassFilter::_current_mark = nullptr;
+Thread* AOTClassFilter::_filtering_thread = nullptr;
+
+AOTClassFilter::FilterMark::FilterMark() {
+ MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+ assert(_current_mark == nullptr &&_filtering_thread == nullptr,
+ "impl note: we support only a single AOTClassFilter used by a single thread");
+ _current_mark = this;
+ _filtering_thread = Thread::current();
+}
+
+AOTClassFilter::FilterMark::~FilterMark() {
+ MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+ assert(_current_mark == this && _filtering_thread == Thread::current(), "sanity");
+ _current_mark = nullptr;
+ _filtering_thread = nullptr;
+}
+
+// Is called only from SystemDictionaryShared::init_dumptime_info(), which holds DumpTimeTable_lock
+bool AOTClassFilter::is_aot_tooling_class(InstanceKlass* ik) {
+ assert_lock_strong(DumpTimeTable_lock);
+ if (_current_mark == nullptr || _filtering_thread != Thread::current()) {
+ return false;
+ } else {
+ return _current_mark->is_aot_tooling_class(ik);
+ }
+}
diff --git a/src/hotspot/share/cds/aotClassFilter.hpp b/src/hotspot/share/cds/aotClassFilter.hpp
new file mode 100644
index 00000000000..b85b83c3a32
--- /dev/null
+++ b/src/hotspot/share/cds/aotClassFilter.hpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_CDS_AOTCLASSFILTER_HPP
+#define SHARE_CDS_AOTCLASSFILTER_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/debug.hpp"
+
+class InstanceKlass;
+class Thread;
+
+// Used by SystemDictionaryShared/AOTArtifactFinder to filter out classes that
+// shouldn't be included into the AOT cache -- for example, classes that are used only
+// in the training/assembly phases for building contents in the AOT cache.
+//
+// The only use case today is in lambdaFormInvokers.cpp.
+class AOTClassFilter : AllStatic {
+public:
+
+ // Filters should be defined using RAII pattern
+ class FilterMark {
+ public:
+ FilterMark();
+ ~FilterMark();
+ virtual bool is_aot_tooling_class(InstanceKlass* ik) = 0;
+ };
+
+ // Called when ik is being loaded. Return true iff this class is loaded
+ // only because it's used by the AOT tooling code.
+ static bool is_aot_tooling_class(InstanceKlass* ik);
+
+private:
+ // For the time being, we allow at most one filter.
+ static FilterMark* _current_mark;
+ static Thread* _filtering_thread;
+};
+
+#endif // SHARE_CDS_AOTCLASSFILTER_HPP
diff --git a/src/hotspot/share/cds/aotClassInitializer.cpp b/src/hotspot/share/cds/aotClassInitializer.cpp
index 5b022cae244..297f8109eb4 100644
--- a/src/hotspot/share/cds/aotClassInitializer.cpp
+++ b/src/hotspot/share/cds/aotClassInitializer.cpp
@@ -26,11 +26,16 @@
#include "cds/archiveBuilder.hpp"
#include "cds/cdsConfig.hpp"
#include "cds/heapShared.hpp"
+#include "classfile/symbolTable.hpp"
+#include "classfile/systemDictionaryShared.hpp"
#include "classfile/vmSymbols.hpp"
#include "oops/instanceKlass.inline.hpp"
#include "oops/symbol.hpp"
+#include "runtime/java.hpp"
#include "runtime/javaCalls.hpp"
+DEBUG_ONLY(InstanceKlass* _aot_init_class = nullptr;)
+
// Detector for class names we wish to handle specially.
// It is either an exact string match or a string prefix match.
class AOTClassInitializer::AllowedSpec {
@@ -93,12 +98,12 @@ bool AOTClassInitializer::is_allowed(AllowedSpec* specs, InstanceKlass* ik) {
bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
- assert(!ArchiveBuilder::current()->is_in_buffer_space(ik), "must be source klass");
+ assert(!ArchiveBuilder::is_active() || !ArchiveBuilder::current()->is_in_buffer_space(ik), "must be source klass");
if (!CDSConfig::is_initing_classes_at_dump_time()) {
return false;
}
- if (!ik->is_initialized()) {
+ if (!ik->is_initialized() && !ik->is_being_initialized()) {
return false;
}
@@ -107,7 +112,7 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
// Automatic selection for aot-inited classes
// ==========================================
//
- // When CDSConfig::is_initing_classes_at_dump_time() is enabled,
+ // When CDSConfig::is_initing_classes_at_dump_time is enabled,
// AOTArtifactFinder::find_artifacts() finds the classes of all
// heap objects that are reachable from HeapShared::_run_time_special_subgraph,
// and mark these classes as aot-inited. This preserves the initialized
@@ -266,7 +271,7 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
}
}
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
// This table was created with the help of CDSHeapVerifier.
// Also, some $Holder classes are needed. E.g., Invokers. explicitly
// initializes Invokers$Holder. Since Invokers. won't be executed
@@ -293,9 +298,12 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
{"java/lang/invoke/LambdaForm"},
{"java/lang/invoke/LambdaForm$Holder"}, // UNSAFE.ensureClassInitialized()
{"java/lang/invoke/LambdaForm$NamedFunction"},
+ {"java/lang/invoke/LambdaMetafactory"},
{"java/lang/invoke/MethodHandle"},
{"java/lang/invoke/MethodHandles"},
{"java/lang/invoke/SimpleMethodHandle"},
+ {"java/lang/invoke/StringConcatFactory"},
+ {"java/lang/invoke/VarHandleGuards"},
{"java/util/Collections"},
{"java/util/stream/Collectors"},
{"jdk/internal/constant/ConstantUtils"},
@@ -315,6 +323,12 @@ bool AOTClassInitializer::can_archive_initialized_mirror(InstanceKlass* ik) {
}
}
+#ifdef ASSERT
+ if (ik == _aot_init_class) {
+ return true;
+ }
+#endif
+
return false;
}
@@ -345,3 +359,33 @@ void AOTClassInitializer::call_runtime_setup(JavaThread* current, InstanceKlass*
}
}
}
+
+#ifdef ASSERT
+void AOTClassInitializer::init_test_class(TRAPS) {
+ // -XX:AOTInitTestClass is used in regression tests for adding additional AOT-initialized classes
+ // and heap objects into the AOT cache. The tests must be carefully written to avoid including
+ // any classes that cannot be AOT-initialized.
+ //
+ // -XX:AOTInitTestClass is NOT a general mechanism for including user-defined objects into
+ // the AOT cache. Therefore, this option is NOT available in product JVM.
+ if (AOTInitTestClass != nullptr && CDSConfig::is_initing_classes_at_dump_time()) {
+ log_info(cds)("Debug build only: force initialization of AOTInitTestClass %s", AOTInitTestClass);
+ TempNewSymbol class_name = SymbolTable::new_symbol(AOTInitTestClass);
+ Handle app_loader(THREAD, SystemDictionary::java_system_loader());
+ Klass* k = SystemDictionary::resolve_or_null(class_name, app_loader, CHECK);
+ if (k == nullptr) {
+ vm_exit_during_initialization("AOTInitTestClass not found", AOTInitTestClass);
+ }
+ if (!k->is_instance_klass()) {
+ vm_exit_during_initialization("Invalid name for AOTInitTestClass", AOTInitTestClass);
+ }
+
+ _aot_init_class = InstanceKlass::cast(k);
+ _aot_init_class->initialize(CHECK);
+ }
+}
+
+bool AOTClassInitializer::has_test_class() {
+ return _aot_init_class != nullptr;
+}
+#endif
diff --git a/src/hotspot/share/cds/aotClassInitializer.hpp b/src/hotspot/share/cds/aotClassInitializer.hpp
index c8693a0add3..d2f6d18ef3c 100644
--- a/src/hotspot/share/cds/aotClassInitializer.hpp
+++ b/src/hotspot/share/cds/aotClassInitializer.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,6 +41,10 @@ public:
static bool is_runtime_setup_required(InstanceKlass* ik);
static void call_runtime_setup(JavaThread* current, InstanceKlass* ik);
+
+ // Support for regression testing. Available in debug builds only.
+ static void init_test_class(TRAPS) NOT_DEBUG_RETURN;
+ static bool has_test_class() NOT_DEBUG({ return false; });
};
#endif // SHARE_CDS_AOTCLASSINITIALIZER_HPP
diff --git a/src/hotspot/share/cds/aotClassLinker.cpp b/src/hotspot/share/cds/aotClassLinker.cpp
index a1cacd735dd..dc539eb3d55 100644
--- a/src/hotspot/share/cds/aotClassLinker.cpp
+++ b/src/hotspot/share/cds/aotClassLinker.cpp
@@ -142,7 +142,7 @@ bool AOTClassLinker::try_add_candidate(InstanceKlass* ik) {
if (ik->is_hidden()) {
assert(ik->shared_class_loader_type() != ClassLoader::OTHER, "must have been set");
- if (!CDSConfig::is_dumping_invokedynamic()) {
+ if (!CDSConfig::is_dumping_method_handles()) {
return false;
}
if (HeapShared::is_lambda_proxy_klass(ik)) {
diff --git a/src/hotspot/share/cds/aotClassLocation.cpp b/src/hotspot/share/cds/aotClassLocation.cpp
index 8d453fe1773..656d7551b03 100644
--- a/src/hotspot/share/cds/aotClassLocation.cpp
+++ b/src/hotspot/share/cds/aotClassLocation.cpp
@@ -270,7 +270,7 @@ AOTClassLocation* AOTClassLocation::allocate(JavaThread* current, const char* pa
cs->_manifest_length = manifest_length;
cs->_check_time = check_time;
cs->_from_cpattr = from_cpattr;
- cs->_timestamp = timestamp;
+ cs->_timestamp = check_time ? timestamp : 0;
cs->_filesize = filesize;
cs->_file_type = type;
cs->_group = group;
@@ -989,8 +989,14 @@ bool AOTClassLocationConfig::validate(bool has_aot_linked_classes, bool* has_ext
const char* hint_msg = log_is_enabled(Info, class, path) ?
"" : " (hint: enable -Xlog:class+path=info to diagnose the failure)";
if (RequireSharedSpaces && !PrintSharedArchiveAndExit) {
- log_error(cds)("%s%s", mismatch_msg, hint_msg);
- MetaspaceShared::unrecoverable_loading_error();
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ log_error(cds)("class path and/or module path are not compatible with the "
+ "ones specified when the AOTConfiguration file was recorded%s", hint_msg);
+ vm_exit_during_initialization("Unable to use create AOT cache.", nullptr);
+ } else {
+ log_error(cds)("%s%s", mismatch_msg, hint_msg);
+ MetaspaceShared::unrecoverable_loading_error();
+ }
} else {
log_warning(cds)("%s%s", mismatch_msg, hint_msg);
}
diff --git a/src/hotspot/share/cds/archiveBuilder.cpp b/src/hotspot/share/cds/archiveBuilder.cpp
index afd2d909595..c309de17b4c 100644
--- a/src/hotspot/share/cds/archiveBuilder.cpp
+++ b/src/hotspot/share/cds/archiveBuilder.cpp
@@ -153,7 +153,6 @@ void ArchiveBuilder::SourceObjList::relocate(int i, ArchiveBuilder* builder) {
ArchiveBuilder::ArchiveBuilder() :
_current_dump_region(nullptr),
_buffer_bottom(nullptr),
- _num_dump_regions_used(0),
_requested_static_archive_bottom(nullptr),
_requested_static_archive_top(nullptr),
_requested_dynamic_archive_bottom(nullptr),
@@ -161,6 +160,7 @@ ArchiveBuilder::ArchiveBuilder() :
_mapped_static_archive_bottom(nullptr),
_mapped_static_archive_top(nullptr),
_buffer_to_requested_delta(0),
+ _pz_region("pz", MAX_SHARED_DELTA), // protection zone -- used only during dumping; does NOT exist in cds archive.
_rw_region("rw", MAX_SHARED_DELTA),
_ro_region("ro", MAX_SHARED_DELTA),
_ptrmap(mtClassShared),
@@ -323,8 +323,12 @@ address ArchiveBuilder::reserve_buffer() {
_shared_rs = rs;
_buffer_bottom = buffer_bottom;
- _current_dump_region = &_rw_region;
- _num_dump_regions_used = 1;
+
+ if (CDSConfig::is_dumping_static_archive()) {
+ _current_dump_region = &_pz_region;
+ } else {
+ _current_dump_region = &_rw_region;
+ }
_current_dump_region->init(&_shared_rs, &_shared_vs);
ArchivePtrMarker::initialize(&_ptrmap, &_shared_vs);
@@ -366,7 +370,8 @@ address ArchiveBuilder::reserve_buffer() {
if (CDSConfig::is_dumping_static_archive()) {
// We don't want any valid object to be at the very bottom of the archive.
// See ArchivePtrMarker::mark_pointer().
- rw_region()->allocate(16);
+ _pz_region.allocate(MetaspaceShared::protection_zone_size());
+ start_dump_region(&_rw_region);
}
return buffer_bottom;
@@ -507,9 +512,8 @@ bool ArchiveBuilder::is_excluded(Klass* klass) {
return SystemDictionaryShared::is_excluded_class(ik);
} else if (klass->is_objArray_klass()) {
Klass* bottom = ObjArrayKlass::cast(klass)->bottom_klass();
- if (MetaspaceShared::is_shared_static(bottom)) {
+ if (CDSConfig::is_dumping_dynamic_archive() && MetaspaceShared::is_shared_static(bottom)) {
// The bottom class is in the static archive so it's clearly not excluded.
- assert(CDSConfig::is_dumping_dynamic_archive(), "sanity");
return false;
} else if (bottom->is_instance_klass()) {
return SystemDictionaryShared::is_excluded_class(InstanceKlass::cast(bottom));
@@ -521,7 +525,7 @@ bool ArchiveBuilder::is_excluded(Klass* klass) {
ArchiveBuilder::FollowMode ArchiveBuilder::get_follow_mode(MetaspaceClosure::Ref *ref) {
address obj = ref->obj();
- if (MetaspaceShared::is_in_shared_metaspace(obj)) {
+ if (CDSConfig::is_dumping_dynamic_archive() && MetaspaceShared::is_in_shared_metaspace(obj)) {
// Don't dump existing shared metadata again.
return point_to_it;
} else if (ref->msotype() == MetaspaceObj::MethodDataType ||
@@ -545,7 +549,6 @@ ArchiveBuilder::FollowMode ArchiveBuilder::get_follow_mode(MetaspaceClosure::Ref
void ArchiveBuilder::start_dump_region(DumpRegion* next) {
current_dump_region()->pack(next);
_current_dump_region = next;
- _num_dump_regions_used ++;
}
char* ArchiveBuilder::ro_strdup(const char* s) {
@@ -787,6 +790,7 @@ void ArchiveBuilder::make_klasses_shareable() {
const char* aotlinked_msg = "";
const char* inited_msg = "";
Klass* k = get_buffered_addr(klasses()->at(i));
+ bool inited = false;
k->remove_java_mirror();
#ifdef _LP64
if (UseCompactObjectHeaders) {
@@ -811,7 +815,7 @@ void ArchiveBuilder::make_klasses_shareable() {
InstanceKlass* ik = InstanceKlass::cast(k);
InstanceKlass* src_ik = get_source_addr(ik);
bool aotlinked = AOTClassLinker::is_candidate(src_ik);
- bool inited = ik->has_aot_initialized_mirror();
+ inited = ik->has_aot_initialized_mirror();
ADD_COUNT(num_instance_klasses);
if (CDSConfig::is_dumping_dynamic_archive()) {
// For static dump, class loader type are already set.
@@ -834,7 +838,7 @@ void ArchiveBuilder::make_klasses_shareable() {
type = "bad";
assert(0, "shouldn't happen");
}
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
assert(HeapShared::is_archivable_hidden_klass(ik), "sanity");
} else {
// Legacy CDS support for lambda proxies
@@ -892,7 +896,11 @@ void ArchiveBuilder::make_klasses_shareable() {
aotlinked_msg = " aot-linked";
}
if (inited) {
- inited_msg = " inited";
+ if (InstanceKlass::cast(k)->static_field_size() == 0) {
+ inited_msg = " inited (no static fields)";
+ } else {
+ inited_msg = " inited";
+ }
}
MetaspaceShared::rewrite_nofast_bytecodes_and_calculate_fingerprints(Thread::current(), ik);
@@ -1536,6 +1544,7 @@ void ArchiveBuilder::write_archive(FileMapInfo* mapinfo, ArchiveHeapInfo* heap_i
mapinfo->close();
if (log_is_enabled(Info, cds)) {
+ log_info(cds)("Full module graph = %s", CDSConfig::is_dumping_full_module_graph() ? "enabled" : "disabled");
print_stats();
}
diff --git a/src/hotspot/share/cds/archiveBuilder.hpp b/src/hotspot/share/cds/archiveBuilder.hpp
index e3efedd46f1..5913ae29c78 100644
--- a/src/hotspot/share/cds/archiveBuilder.hpp
+++ b/src/hotspot/share/cds/archiveBuilder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -96,7 +96,6 @@ class ArchiveBuilder : public StackObj {
protected:
DumpRegion* _current_dump_region;
address _buffer_bottom; // for writing the contents of rw/ro regions
- int _num_dump_regions_used;
// These are the addresses where we will request the static and dynamic archives to be
// mapped at run time. If the request fails (due to ASLR), we will map the archives at
@@ -210,6 +209,12 @@ private:
ReservedSpace _shared_rs;
VirtualSpace _shared_vs;
+ // The "pz" region is used only during static dumps to reserve an unused space between SharedBaseAddress and
+ // the bottom of the rw region. During runtime, this space will be filled with a reserved area that disallows
+ // read/write/exec, so we can track for bad CompressedKlassPointers encoding.
+ // Note: this region does NOT exist in the cds archive.
+ DumpRegion _pz_region;
+
DumpRegion _rw_region;
DumpRegion _ro_region;
@@ -270,9 +275,6 @@ private:
protected:
virtual void iterate_roots(MetaspaceClosure* it) = 0;
-
- static const int _total_dump_regions = 2;
-
void start_dump_region(DumpRegion* next);
public:
@@ -367,6 +369,7 @@ public:
void remember_embedded_pointer_in_enclosing_obj(MetaspaceClosure::Ref* ref);
static void serialize_dynamic_archivable_items(SerializeClosure* soc);
+ DumpRegion* pz_region() { return &_pz_region; }
DumpRegion* rw_region() { return &_rw_region; }
DumpRegion* ro_region() { return &_ro_region; }
diff --git a/src/hotspot/share/cds/archiveUtils.cpp b/src/hotspot/share/cds/archiveUtils.cpp
index 90eefd13d46..217b30e401d 100644
--- a/src/hotspot/share/cds/archiveUtils.cpp
+++ b/src/hotspot/share/cds/archiveUtils.cpp
@@ -31,6 +31,7 @@
#include "cds/dynamicArchive.hpp"
#include "cds/filemap.hpp"
#include "cds/heapShared.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/metaspaceShared.hpp"
#include "classfile/systemDictionaryShared.hpp"
#include "classfile/vmClasses.hpp"
@@ -73,34 +74,39 @@ void ArchivePtrMarker::initialize(CHeapBitMap* ptrmap, VirtualSpace* vs) {
}
void ArchivePtrMarker::initialize_rw_ro_maps(CHeapBitMap* rw_ptrmap, CHeapBitMap* ro_ptrmap) {
- address* rw_bottom = (address*)ArchiveBuilder::current()->rw_region()->base();
- address* ro_bottom = (address*)ArchiveBuilder::current()->ro_region()->base();
+ address* buff_bottom = (address*)ArchiveBuilder::current()->buffer_bottom();
+ address* rw_bottom = (address*)ArchiveBuilder::current()->rw_region()->base();
+ address* ro_bottom = (address*)ArchiveBuilder::current()->ro_region()->base();
+
+ // The bit in _ptrmap that cover the very first word in the rw/ro regions.
+ size_t rw_start = rw_bottom - buff_bottom;
+ size_t ro_start = ro_bottom - buff_bottom;
+
+ // The number of bits used by the rw/ro ptrmaps. We might have lots of zero
+ // bits at the bottom and top of rw/ro ptrmaps, but these zeros will be
+ // removed by FileMapInfo::write_bitmap_region().
+ size_t rw_size = ArchiveBuilder::current()->rw_region()->used() / sizeof(address);
+ size_t ro_size = ArchiveBuilder::current()->ro_region()->used() / sizeof(address);
+
+ // The last (exclusive) bit in _ptrmap that covers the rw/ro regions.
+ // Note: _ptrmap is dynamically expanded only when an actual pointer is written, so
+ // it may not be as large as we want.
+ size_t rw_end = MIN2(rw_start + rw_size, _ptrmap->size());
+ size_t ro_end = MIN2(ro_start + ro_size, _ptrmap->size());
+
+ rw_ptrmap->initialize(rw_size);
+ ro_ptrmap->initialize(ro_size);
+
+ for (size_t rw_bit = rw_start; rw_bit < rw_end; rw_bit++) {
+ rw_ptrmap->at_put(rw_bit - rw_start, _ptrmap->at(rw_bit));
+ }
+
+ for(size_t ro_bit = ro_start; ro_bit < ro_end; ro_bit++) {
+ ro_ptrmap->at_put(ro_bit - ro_start, _ptrmap->at(ro_bit));
+ }
_rw_ptrmap = rw_ptrmap;
_ro_ptrmap = ro_ptrmap;
-
- size_t rw_size = ArchiveBuilder::current()->rw_region()->used() / sizeof(address);
- size_t ro_size = ArchiveBuilder::current()->ro_region()->used() / sizeof(address);
- // ro_start is the first bit in _ptrmap that covers the pointer that would sit at ro_bottom.
- // E.g., if rw_bottom = (address*)100
- // ro_bottom = (address*)116
- // then for 64-bit platform:
- // ro_start = ro_bottom - rw_bottom = (116 - 100) / sizeof(address) = 2;
- size_t ro_start = ro_bottom - rw_bottom;
-
- // Note: ptrmap is big enough only to cover the last pointer in ro_region.
- // See ArchivePtrMarker::compact()
- _rw_ptrmap->initialize(rw_size);
- _ro_ptrmap->initialize(_ptrmap->size() - ro_start);
-
- for (size_t rw_bit = 0; rw_bit < _rw_ptrmap->size(); rw_bit++) {
- _rw_ptrmap->at_put(rw_bit, _ptrmap->at(rw_bit));
- }
-
- for(size_t ro_bit = ro_start; ro_bit < _ptrmap->size(); ro_bit++) {
- _ro_ptrmap->at_put(ro_bit-ro_start, _ptrmap->at(ro_bit));
- }
- assert(_ptrmap->size() - ro_start == _ro_ptrmap->size(), "must be");
}
void ArchivePtrMarker::mark_pointer(address* ptr_loc) {
@@ -351,7 +357,7 @@ void ReadClosure::do_tag(int tag) {
void ArchiveUtils::log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) {
if (ClassListWriter::is_enabled()) {
- if (SystemDictionaryShared::is_supported_invokedynamic(bootstrap_specifier)) {
+ if (LambdaProxyClassDictionary::is_supported_invokedynamic(bootstrap_specifier)) {
const constantPoolHandle& pool = bootstrap_specifier->pool();
if (SystemDictionaryShared::is_builtin_loader(pool->pool_holder()->class_loader_data())) {
// Currently lambda proxy classes are supported only for the built-in loaders.
diff --git a/src/hotspot/share/cds/archiveUtils.hpp b/src/hotspot/share/cds/archiveUtils.hpp
index a10117e9f9a..59146547aca 100644
--- a/src/hotspot/share/cds/archiveUtils.hpp
+++ b/src/hotspot/share/cds/archiveUtils.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -255,11 +255,23 @@ public:
};
class ArchiveUtils {
+ template static Array* archive_non_ptr_array(GrowableArray* tmp_array);
+ template static Array* archive_ptr_array(GrowableArray* tmp_array);
+
public:
static const uintx MAX_SHARED_DELTA = 0x7FFFFFFF;
static void log_to_classlist(BootstrapInfo* bootstrap_specifier, TRAPS) NOT_CDS_RETURN;
static bool has_aot_initialized_mirror(InstanceKlass* src_ik);
- template static Array* archive_array(GrowableArray* tmp_array);
+
+ template ::value)>
+ static Array* archive_array(GrowableArray* tmp_array) {
+ return archive_non_ptr_array(tmp_array);
+ }
+
+ template ::value)>
+ static Array* archive_array(GrowableArray* tmp_array) {
+ return archive_ptr_array(tmp_array);
+ }
// The following functions translate between a u4 offset and an address in the
// the range of the mapped CDS archive (e.g., Metaspace::is_in_shared_metaspace()).
diff --git a/src/hotspot/share/cds/archiveUtils.inline.hpp b/src/hotspot/share/cds/archiveUtils.inline.hpp
index 537b3d1670c..9388bca18c7 100644
--- a/src/hotspot/share/cds/archiveUtils.inline.hpp
+++ b/src/hotspot/share/cds/archiveUtils.inline.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,8 @@
#include "cds/archiveUtils.hpp"
#include "cds/archiveBuilder.hpp"
+#include "cds/cdsConfig.hpp"
+#include "cds/metaspaceShared.hpp"
#include "oops/array.hpp"
#include "utilities/bitMap.inline.hpp"
#include "utilities/growableArray.hpp"
@@ -52,13 +54,40 @@ inline bool SharedDataRelocator::do_bit(size_t offset) {
// Returns the address of an Array that's allocated in the ArchiveBuilder "buffer" space.
template
-Array* ArchiveUtils::archive_array(GrowableArray* tmp_array) {
+Array* ArchiveUtils::archive_non_ptr_array(GrowableArray* tmp_array) {
+ ArchiveBuilder* builder = ArchiveBuilder::current();
+
Array* archived_array = ArchiveBuilder::new_ro_array(tmp_array->length());
for (int i = 0; i < tmp_array->length(); i++) {
archived_array->at_put(i, tmp_array->at(i));
- if (std::is_pointer::value) {
+ }
+
+ return archived_array;
+}
+
+// Returns the address of an Array that's allocated in the ArchiveBuilder "buffer" space.
+// All pointers in tmp_array must point to:
+// - a buffered object; or
+// - a source object that has been archived; or
+// - (only when dumping dynamic archive) an object in the static archive.
+template
+Array* ArchiveUtils::archive_ptr_array(GrowableArray* tmp_array) {
+ ArchiveBuilder* builder = ArchiveBuilder::current();
+ const bool is_dynamic_dump = CDSConfig::is_dumping_dynamic_archive();
+
+ Array* archived_array = ArchiveBuilder::new_ro_array(tmp_array->length());
+ for (int i = 0; i < tmp_array->length(); i++) {
+ T ptr = tmp_array->at(i);
+ if (!builder->is_in_buffer_space(ptr)) {
+ if (is_dynamic_dump && MetaspaceShared::is_in_shared_metaspace(ptr)) {
+ // We have a pointer that lives in the dynamic archive but points into
+ // the static archive.
+ } else {
+ ptr = builder->get_buffered_addr(ptr);
+ }
+ }
+ archived_array->at_put(i, ptr);
ArchivePtrMarker::mark_pointer(archived_array->adr_at(i));
- }
}
return archived_array;
diff --git a/src/hotspot/share/cds/cdsConfig.cpp b/src/hotspot/share/cds/cdsConfig.cpp
index 564298fa5c8..47613e09008 100644
--- a/src/hotspot/share/cds/cdsConfig.cpp
+++ b/src/hotspot/share/cds/cdsConfig.cpp
@@ -40,13 +40,15 @@
#include "utilities/formatBuffer.hpp"
bool CDSConfig::_is_dumping_static_archive = false;
+bool CDSConfig::_is_dumping_preimage_static_archive = false;
+bool CDSConfig::_is_dumping_final_static_archive = false;
bool CDSConfig::_is_dumping_dynamic_archive = false;
bool CDSConfig::_is_using_optimized_module_handling = true;
bool CDSConfig::_is_dumping_full_module_graph = true;
bool CDSConfig::_is_using_full_module_graph = true;
bool CDSConfig::_has_aot_linked_classes = false;
-bool CDSConfig::_has_archived_invokedynamic = false;
bool CDSConfig::_old_cds_flags_used = false;
+bool CDSConfig::_new_aot_flags_used = false;
bool CDSConfig::_disable_heap_dumping = false;
char* CDSConfig::_default_archive_path = nullptr;
@@ -58,16 +60,24 @@ JavaThread* CDSConfig::_dumper_thread = nullptr;
int CDSConfig::get_status() {
assert(Universe::is_fully_initialized(), "status is finalized only after Universe is initialized");
return (is_dumping_archive() ? IS_DUMPING_ARCHIVE : 0) |
+ (is_dumping_method_handles() ? IS_DUMPING_METHOD_HANDLES : 0) |
(is_dumping_static_archive() ? IS_DUMPING_STATIC_ARCHIVE : 0) |
(is_logging_lambda_form_invokers() ? IS_LOGGING_LAMBDA_FORM_INVOKERS : 0) |
(is_using_archive() ? IS_USING_ARCHIVE : 0);
}
void CDSConfig::initialize() {
- if (is_dumping_static_archive()) {
- if (RequireSharedSpaces) {
- warning("Cannot dump shared archive while using shared archive");
- }
+ if (is_dumping_static_archive() && !is_dumping_final_static_archive()) {
+ // Note: -Xshare and -XX:AOTMode flags are mutually exclusive.
+ // - Classic workflow: -Xshare:on and -Xshare:dump cannot take effect at the same time.
+ // - JEP 483 workflow: -XX:AOTMode:record and -XX:AOTMode=on cannot take effect at the same time.
+ // So we can never come to here with RequireSharedSpaces==true.
+ assert(!RequireSharedSpaces, "sanity");
+
+ // If dumping the classic archive, or making an AOT training run (dumping a preimage archive),
+ // for sanity, parse all classes from classfiles.
+ // TODO: in the future, if we want to support re-training on top of an existing AOT cache, this
+ // needs to be changed.
UseSharedSpaces = false;
}
@@ -86,12 +96,10 @@ void CDSConfig::initialize() {
char* CDSConfig::default_archive_path() {
if (_default_archive_path == nullptr) {
- char jvm_path[JVM_MAXPATHLEN];
- os::jvm_path(jvm_path, sizeof(jvm_path));
- char *end = strrchr(jvm_path, *os::file_separator());
- if (end != nullptr) *end = '\0';
stringStream tmp;
- tmp.print("%s%sclasses", jvm_path, os::file_separator());
+ const char* subdir = WINDOWS_ONLY("bin") NOT_WINDOWS("lib");
+ tmp.print("%s%s%s%s%s%sclasses", Arguments::get_java_home(), os::file_separator(), subdir,
+ os::file_separator(), Abstract_VM_Version::vm_variant(), os::file_separator());
#ifdef _LP64
if (!UseCompressedOops) {
tmp.print_raw("_nocoops");
@@ -210,6 +218,7 @@ void CDSConfig::init_shared_archive_paths() {
warning("-XX:+AutoCreateSharedArchive is unsupported when base CDS archive is not loaded. Run with -Xlog:cds for more info.");
AutoCreateSharedArchive = false;
}
+ log_error(cds)("Not a valid %s (%s)", new_aot_flags_used() ? "AOT cache" : "archive", SharedArchiveFile);
Arguments::no_shared_spaces("invalid archive");
}
} else if (base_archive_path == nullptr) {
@@ -333,7 +342,11 @@ bool CDSConfig::has_unsupported_runtime_module_options() {
if (RequireSharedSpaces) {
warning("CDS is disabled when the %s option is specified.", option);
} else {
- log_info(cds)("CDS is disabled when the %s option is specified.", option);
+ if (new_aot_flags_used()) {
+ log_warning(cds)("AOT cache is disabled when the %s option is specified.", option);
+ } else {
+ log_info(cds)("CDS is disabled when the %s option is specified.", option);
+ }
}
return true;
}
@@ -343,7 +356,7 @@ bool CDSConfig::has_unsupported_runtime_module_options() {
#define CHECK_ALIAS(f) check_flag_alias(FLAG_IS_DEFAULT(f), #f)
void CDSConfig::check_flag_alias(bool alias_is_default, const char* alias_name) {
- if (_old_cds_flags_used && !alias_is_default) {
+ if (old_cds_flags_used() && !alias_is_default) {
vm_exit_during_initialization(err_msg("Option %s cannot be used at the same time with "
"-Xshare:on, -Xshare:auto, -Xshare:off, -Xshare:dump, "
"DumpLoadedClassList, SharedClassListFile, or SharedArchiveFile",
@@ -351,7 +364,7 @@ void CDSConfig::check_flag_alias(bool alias_is_default, const char* alias_name)
}
}
-void CDSConfig::check_flag_aliases() {
+void CDSConfig::check_aot_flags() {
if (!FLAG_IS_DEFAULT(DumpLoadedClassList) ||
!FLAG_IS_DEFAULT(SharedClassListFile) ||
!FLAG_IS_DEFAULT(SharedArchiveFile)) {
@@ -363,30 +376,16 @@ void CDSConfig::check_flag_aliases() {
CHECK_ALIAS(AOTMode);
if (FLAG_IS_DEFAULT(AOTCache) && FLAG_IS_DEFAULT(AOTConfiguration) && FLAG_IS_DEFAULT(AOTMode)) {
- // Aliases not used.
+ // AOTCache/AOTConfiguration/AOTMode not used.
return;
+ } else {
+ _new_aot_flags_used = true;
}
if (FLAG_IS_DEFAULT(AOTMode) || strcmp(AOTMode, "auto") == 0 || strcmp(AOTMode, "on") == 0) {
- if (!FLAG_IS_DEFAULT(AOTConfiguration)) {
- vm_exit_during_initialization("AOTConfiguration can only be used with -XX:AOTMode=record or -XX:AOTMode=create");
- }
-
- if (!FLAG_IS_DEFAULT(AOTCache)) {
- assert(FLAG_IS_DEFAULT(SharedArchiveFile), "already checked");
- FLAG_SET_ERGO(SharedArchiveFile, AOTCache);
- }
-
- UseSharedSpaces = true;
- if (FLAG_IS_DEFAULT(AOTMode) || (strcmp(AOTMode, "auto") == 0)) {
- RequireSharedSpaces = false;
- } else {
- assert(strcmp(AOTMode, "on") == 0, "already checked");
- RequireSharedSpaces = true;
- }
+ check_aotmode_auto_or_on();
} else if (strcmp(AOTMode, "off") == 0) {
- UseSharedSpaces = false;
- RequireSharedSpaces = false;
+ check_aotmode_off();
} else {
// AOTMode is record or create
if (FLAG_IS_DEFAULT(AOTConfiguration)) {
@@ -394,32 +393,78 @@ void CDSConfig::check_flag_aliases() {
}
if (strcmp(AOTMode, "record") == 0) {
- if (!FLAG_IS_DEFAULT(AOTCache)) {
- vm_exit_during_initialization("AOTCache must not be specified when using -XX:AOTMode=record");
- }
-
- assert(FLAG_IS_DEFAULT(DumpLoadedClassList), "already checked");
- FLAG_SET_ERGO(DumpLoadedClassList, AOTConfiguration);
- UseSharedSpaces = false;
- RequireSharedSpaces = false;
+ check_aotmode_record();
} else {
assert(strcmp(AOTMode, "create") == 0, "checked by AOTModeConstraintFunc");
- if (FLAG_IS_DEFAULT(AOTCache)) {
- vm_exit_during_initialization("AOTCache must be specified when using -XX:AOTMode=create");
- }
-
- assert(FLAG_IS_DEFAULT(SharedClassListFile), "already checked");
- FLAG_SET_ERGO(SharedClassListFile, AOTConfiguration);
- assert(FLAG_IS_DEFAULT(SharedArchiveFile), "already checked");
- FLAG_SET_ERGO(SharedArchiveFile, AOTCache);
-
- CDSConfig::enable_dumping_static_archive();
+ check_aotmode_create();
}
}
}
+void CDSConfig::check_aotmode_off() {
+ UseSharedSpaces = false;
+ RequireSharedSpaces = false;
+}
+
+void CDSConfig::check_aotmode_auto_or_on() {
+ if (!FLAG_IS_DEFAULT(AOTConfiguration)) {
+ vm_exit_during_initialization("AOTConfiguration can only be used with -XX:AOTMode=record or -XX:AOTMode=create");
+ }
+
+ if (!FLAG_IS_DEFAULT(AOTCache)) {
+ assert(FLAG_IS_DEFAULT(SharedArchiveFile), "already checked");
+ FLAG_SET_ERGO(SharedArchiveFile, AOTCache);
+ }
+
+ UseSharedSpaces = true;
+ if (FLAG_IS_DEFAULT(AOTMode) || (strcmp(AOTMode, "auto") == 0)) {
+ RequireSharedSpaces = false;
+ } else {
+ assert(strcmp(AOTMode, "on") == 0, "already checked");
+ RequireSharedSpaces = true;
+ }
+}
+
+void CDSConfig::check_aotmode_record() {
+ if (!FLAG_IS_DEFAULT(AOTCache)) {
+ vm_exit_during_initialization("AOTCache must not be specified when using -XX:AOTMode=record");
+ }
+
+ assert(FLAG_IS_DEFAULT(DumpLoadedClassList), "already checked");
+ assert(FLAG_IS_DEFAULT(SharedArchiveFile), "already checked");
+ FLAG_SET_ERGO(SharedArchiveFile, AOTConfiguration);
+ FLAG_SET_ERGO(DumpLoadedClassList, nullptr);
+ UseSharedSpaces = false;
+ RequireSharedSpaces = false;
+ _is_dumping_static_archive = true;
+ _is_dumping_preimage_static_archive = true;
+
+ // At VM exit, the module graph may be contaminated with program states.
+ // We will rebuild the module graph when dumping the CDS final image.
+ disable_heap_dumping();
+}
+
+void CDSConfig::check_aotmode_create() {
+ if (FLAG_IS_DEFAULT(AOTCache)) {
+ vm_exit_during_initialization("AOTCache must be specified when using -XX:AOTMode=create");
+ }
+
+ assert(FLAG_IS_DEFAULT(SharedArchiveFile), "already checked");
+
+ _is_dumping_final_static_archive = true;
+ FLAG_SET_ERGO(SharedArchiveFile, AOTConfiguration);
+ UseSharedSpaces = true;
+ RequireSharedSpaces = true;
+
+ if (!FileMapInfo::is_preimage_static_archive(AOTConfiguration)) {
+ vm_exit_during_initialization("Must be a valid AOT configuration generated by the current JVM", AOTConfiguration);
+ }
+
+ CDSConfig::enable_dumping_static_archive();
+}
+
bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_flag_cmd_line) {
- check_flag_aliases();
+ check_aot_flags();
if (!FLAG_IS_DEFAULT(AOTMode)) {
// Using any form of the new AOTMode switch enables enhanced optimizations.
@@ -435,7 +480,9 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla
}
if (is_dumping_static_archive()) {
- if (!mode_flag_cmd_line) {
+ if (is_dumping_preimage_static_archive()) {
+ // Don't tweak execution mode
+ } else if (!mode_flag_cmd_line) {
// By default, -Xshare:dump runs in interpreter-only mode, which is required for deterministic archive.
//
// If your classlist is large and you don't care about deterministic dumping, you can use
@@ -499,9 +546,23 @@ bool CDSConfig::check_vm_args_consistency(bool patch_mod_javabase, bool mode_fla
return true;
}
+bool CDSConfig::is_dumping_classic_static_archive() {
+ return _is_dumping_static_archive &&
+ !is_dumping_preimage_static_archive() &&
+ !is_dumping_final_static_archive();
+}
+
+bool CDSConfig::is_dumping_preimage_static_archive() {
+ return _is_dumping_preimage_static_archive;
+}
+
+bool CDSConfig::is_dumping_final_static_archive() {
+ return _is_dumping_final_static_archive;
+}
+
bool CDSConfig::allow_only_single_java_thread() {
// See comments in JVM_StartThread()
- return is_dumping_static_archive();
+ return is_dumping_classic_static_archive() || is_dumping_final_static_archive();
}
bool CDSConfig::is_using_archive() {
@@ -512,6 +573,24 @@ bool CDSConfig::is_logging_lambda_form_invokers() {
return ClassListWriter::is_enabled() || is_dumping_dynamic_archive();
}
+bool CDSConfig::is_dumping_regenerated_lambdaform_invokers() {
+ if (is_dumping_final_static_archive()) {
+ // No need to regenerate -- the lambda form invokers should have been regenerated
+ // in the preimage archive (if allowed)
+ return false;
+ } else if (is_dumping_dynamic_archive() && is_using_aot_linked_classes()) {
+ // The base archive has aot-linked classes that may have AOT-resolved CP references
+ // that point to the lambda form invokers in the base archive. Such pointers will
+ // be invalid if lambda form invokers are regenerated in the dynamic archive.
+ return false;
+ } else if (CDSConfig::is_dumping_method_handles()) {
+ // Work around JDK-8310831, as some methods in lambda form holder classes may not get generated.
+ return false;
+ } else {
+ return is_dumping_archive();
+ }
+}
+
void CDSConfig::stop_using_optimized_module_handling() {
_is_using_optimized_module_handling = false;
_is_dumping_full_module_graph = false; // This requires is_using_optimized_module_handling()
@@ -534,6 +613,26 @@ bool CDSConfig::current_thread_is_vm_or_dumper() {
return t != nullptr && (t->is_VM_thread() || t == _dumper_thread);
}
+const char* CDSConfig::type_of_archive_being_loaded() {
+ if (is_dumping_final_static_archive()) {
+ return "AOT configuration file";
+ } else if (new_aot_flags_used()) {
+ return "AOT cache";
+ } else {
+ return "shared archive file";
+ }
+}
+
+const char* CDSConfig::type_of_archive_being_written() {
+ if (is_dumping_preimage_static_archive()) {
+ return "AOT configuration file";
+ } else if (new_aot_flags_used()) {
+ return "AOT cache";
+ } else {
+ return "shared archive file";
+ }
+}
+
// If an incompatible VM options is found, return a text message that explains why
static const char* check_options_incompatible_with_dumping_heap() {
#if INCLUDE_CDS_JAVA_HEAP
@@ -567,19 +666,22 @@ void CDSConfig::log_reasons_for_not_dumping_heap() {
log_info(cds)("Archived java heap is not supported: %s", reason);
}
+// This is *Legacy* optimization for lambdas before JEP 483. May be removed in the future.
+bool CDSConfig::is_dumping_lambdas_in_legacy_mode() {
+ return !is_dumping_method_handles();
+}
+
#if INCLUDE_CDS_JAVA_HEAP
bool CDSConfig::are_vm_options_incompatible_with_dumping_heap() {
return check_options_incompatible_with_dumping_heap() != nullptr;
}
-
bool CDSConfig::is_dumping_heap() {
- if (!is_dumping_static_archive() // heap dump is not supported in dynamic dump
+ if (!(is_dumping_classic_static_archive() || is_dumping_final_static_archive())
|| are_vm_options_incompatible_with_dumping_heap()
|| _disable_heap_dumping) {
return false;
}
-
return true;
}
@@ -627,7 +729,9 @@ void CDSConfig::stop_using_full_module_graph(const char* reason) {
}
bool CDSConfig::is_dumping_aot_linked_classes() {
- if (is_dumping_dynamic_archive()) {
+ if (is_dumping_preimage_static_archive()) {
+ return false;
+ } else if (is_dumping_dynamic_archive()) {
return is_using_full_module_graph() && AOTClassLinking;
} else if (is_dumping_static_archive()) {
return is_dumping_full_module_graph() && AOTClassLinking;
@@ -656,8 +760,13 @@ bool CDSConfig::is_dumping_invokedynamic() {
return AOTInvokeDynamicLinking && is_dumping_aot_linked_classes() && is_dumping_heap();
}
-bool CDSConfig::is_loading_invokedynamic() {
- return UseSharedSpaces && is_using_full_module_graph() && _has_archived_invokedynamic;
+// When we are dumping aot-linked classes and we are able to write archived heap objects, we automatically
+// enable the archiving of MethodHandles. This will in turn enable the archiving of MethodTypes and hidden
+// classes that are used in the implementation of MethodHandles.
+// Archived MethodHandles are required for higher-level optimizations such as AOT resolution of invokedynamic
+// and dynamic proxies.
+bool CDSConfig::is_dumping_method_handles() {
+ return is_initing_classes_at_dump_time();
}
#endif // INCLUDE_CDS_JAVA_HEAP
diff --git a/src/hotspot/share/cds/cdsConfig.hpp b/src/hotspot/share/cds/cdsConfig.hpp
index c2dc2b41a93..f02258eb6fe 100644
--- a/src/hotspot/share/cds/cdsConfig.hpp
+++ b/src/hotspot/share/cds/cdsConfig.hpp
@@ -34,18 +34,20 @@ class JavaThread;
class CDSConfig : public AllStatic {
#if INCLUDE_CDS
static bool _is_dumping_static_archive;
+ static bool _is_dumping_preimage_static_archive;
+ static bool _is_dumping_final_static_archive;
static bool _is_dumping_dynamic_archive;
static bool _is_using_optimized_module_handling;
static bool _is_dumping_full_module_graph;
static bool _is_using_full_module_graph;
static bool _has_aot_linked_classes;
- static bool _has_archived_invokedynamic;
static char* _default_archive_path;
static char* _static_archive_path;
static char* _dynamic_archive_path;
static bool _old_cds_flags_used;
+ static bool _new_aot_flags_used;
static bool _disable_heap_dumping;
static JavaThread* _dumper_thread;
@@ -57,25 +59,34 @@ class CDSConfig : public AllStatic {
static void init_shared_archive_paths();
static void check_flag_alias(bool alias_is_default, const char* alias_name);
- static void check_flag_aliases();
+ static void check_aot_flags();
+ static void check_aotmode_off();
+ static void check_aotmode_auto_or_on();
+ static void check_aotmode_record();
+ static void check_aotmode_create();
public:
// Used by jdk.internal.misc.CDS.getCDSConfigStatus();
static const int IS_DUMPING_ARCHIVE = 1 << 0;
- static const int IS_DUMPING_STATIC_ARCHIVE = 1 << 1;
- static const int IS_LOGGING_LAMBDA_FORM_INVOKERS = 1 << 2;
- static const int IS_USING_ARCHIVE = 1 << 3;
+ static const int IS_DUMPING_METHOD_HANDLES = 1 << 1;
+ static const int IS_DUMPING_STATIC_ARCHIVE = 1 << 2;
+ static const int IS_LOGGING_LAMBDA_FORM_INVOKERS = 1 << 3;
+ static const int IS_USING_ARCHIVE = 1 << 4;
+
static int get_status() NOT_CDS_RETURN_(0);
// Initialization and command-line checking
static void initialize() NOT_CDS_RETURN;
static void set_old_cds_flags_used() { CDS_ONLY(_old_cds_flags_used = true); }
static bool old_cds_flags_used() { return CDS_ONLY(_old_cds_flags_used) NOT_CDS(false); }
+ static bool new_aot_flags_used() { return CDS_ONLY(_new_aot_flags_used) NOT_CDS(false); }
static void check_internal_module_property(const char* key, const char* value) NOT_CDS_RETURN;
static void check_incompatible_property(const char* key, const char* value) NOT_CDS_RETURN;
static void check_unsupported_dumping_module_options() NOT_CDS_RETURN;
static bool has_unsupported_runtime_module_options() NOT_CDS_RETURN_(false);
static bool check_vm_args_consistency(bool patch_mod_javabase, bool mode_flag_cmd_line) NOT_CDS_RETURN_(true);
+ static const char* type_of_archive_being_loaded();
+ static const char* type_of_archive_being_written();
// --- Basic CDS features
@@ -88,6 +99,30 @@ public:
static bool is_dumping_static_archive() { return CDS_ONLY(_is_dumping_static_archive) NOT_CDS(false); }
static void enable_dumping_static_archive() { CDS_ONLY(_is_dumping_static_archive = true); }
+ // A static CDS archive can be dumped in three modes:
+ //
+ // "classic" - This is the traditional CDS workflow of
+ // "java -Xshare:dump -XX:SharedClassListFile=file.txt".
+ //
+ // "preimage" - This happens when we execute the JEP 483 training run, e.g:
+ // "java -XX:AOTMode=record -XX:AOTConfiguration=app.aotconfig -cp app.jar App"
+ // The above command writes app.aotconfig as a "CDS preimage". This
+ // is a binary file that contains all the classes loaded during the
+ // training run, plus profiling data (e.g., the resolved constant pool entries).
+ //
+ // "final" - This happens when we execute the JEP 483 assembly phase, e.g:
+ // "java -XX:AOTMode=create -XX:AOTConfiguration=app.aotconfig -XX:AOTCache=app.aot -cp app.jar"
+ // The above command loads all classes from app.aotconfig, perform additional linking,
+ // and writes app.aot as a "CDS final image" file.
+ //
+ // The main structural difference between "preimage" and "final" is that the preimage
+ // - has a different magic number (0xcafea07c)
+ // - does not have any archived Java heap objects
+ // - does not have aot-linked classes
+ static bool is_dumping_classic_static_archive() NOT_CDS_RETURN_(false);
+ static bool is_dumping_preimage_static_archive() NOT_CDS_RETURN_(false);
+ static bool is_dumping_final_static_archive() NOT_CDS_RETURN_(false);
+
// dynamic_archive
static bool is_dumping_dynamic_archive() { return CDS_ONLY(_is_dumping_dynamic_archive) NOT_CDS(false); }
static void enable_dumping_dynamic_archive() { CDS_ONLY(_is_dumping_dynamic_archive = true); }
@@ -96,11 +131,15 @@ public:
// Misc CDS features
static bool allow_only_single_java_thread() NOT_CDS_RETURN_(false);
+ // This is *Legacy* optimization for lambdas before JEP 483. May be removed in the future.
+ static bool is_dumping_lambdas_in_legacy_mode() NOT_CDS_RETURN_(false);
+
// optimized_module_handling -- can we skip some expensive operations related to modules?
static bool is_using_optimized_module_handling() { return CDS_ONLY(_is_using_optimized_module_handling) NOT_CDS(false); }
static void stop_using_optimized_module_handling() NOT_CDS_RETURN;
static bool is_logging_lambda_form_invokers() NOT_CDS_RETURN_(false);
+ static bool is_dumping_regenerated_lambdaform_invokers() NOT_CDS_RETURN_(false);
static bool is_dumping_aot_linked_classes() NOT_CDS_JAVA_HEAP_RETURN_(false);
static bool is_using_aot_linked_classes() NOT_CDS_JAVA_HEAP_RETURN_(false);
@@ -126,8 +165,7 @@ public:
static bool is_initing_classes_at_dump_time() NOT_CDS_JAVA_HEAP_RETURN_(false);
static bool is_dumping_invokedynamic() NOT_CDS_JAVA_HEAP_RETURN_(false);
- static bool is_loading_invokedynamic() NOT_CDS_JAVA_HEAP_RETURN_(false);
- static void set_has_archived_invokedynamic() { CDS_JAVA_HEAP_ONLY(_has_archived_invokedynamic = true); }
+ static bool is_dumping_method_handles() NOT_CDS_JAVA_HEAP_RETURN_(false);
// full_module_graph (requires optimized_module_handling)
static bool is_dumping_full_module_graph() { return CDS_ONLY(_is_dumping_full_module_graph) NOT_CDS(false); }
@@ -135,7 +173,6 @@ public:
static void stop_dumping_full_module_graph(const char* reason = nullptr) NOT_CDS_JAVA_HEAP_RETURN;
static void stop_using_full_module_graph(const char* reason = nullptr) NOT_CDS_JAVA_HEAP_RETURN;
-
// Some CDS functions assume that they are called only within a single-threaded context. I.e.,
// they are called from:
// - The VM thread (e.g., inside VM_PopulateDumpSharedSpace)
diff --git a/src/hotspot/share/cds/cdsHeapVerifier.cpp b/src/hotspot/share/cds/cdsHeapVerifier.cpp
index f9e613a74ce..9bab62dabe6 100644
--- a/src/hotspot/share/cds/cdsHeapVerifier.cpp
+++ b/src/hotspot/share/cds/cdsHeapVerifier.cpp
@@ -41,7 +41,11 @@
#if INCLUDE_CDS_JAVA_HEAP
// CDSHeapVerifier is used to check for problems where an archived object references a
-// static field that may be get a different value at runtime. In the following example,
+// static field that may be get a different value at runtime.
+//
+// *Please see comments in aotClassInitializer.cpp for how to avoid such problems*,
+//
+// In the following example,
// Foo.get.test()
// correctly returns true when CDS disabled, but incorrectly returns false when CDS is enabled,
// because the archived archivedFoo.bar value is different than Bar.bar.
@@ -106,6 +110,8 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
ADD_EXCL("java/lang/System", "bootLayer"); // A
+ ADD_EXCL("java/util/Collections", "EMPTY_LIST"); // E
+
// A dummy object used by HashSet. The value doesn't matter and it's never
// tested for equality.
ADD_EXCL("java/util/HashSet", "PRESENT"); // E
@@ -127,10 +133,16 @@ CDSHeapVerifier::CDSHeapVerifier() : _archived_objs(0), _problems(0)
ADD_EXCL("sun/invoke/util/ValueConversions", "ONE_INT", // E
"ZERO_INT"); // E
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
ADD_EXCL("java/lang/invoke/InvokerBytecodeGenerator", "MEMBERNAME_FACTORY", // D
"CD_Object_array", // E same as <...>ConstantUtils.CD_Object_array::CD_Object
"INVOKER_SUPER_DESC"); // E same as java.lang.constant.ConstantDescs::CD_Object
+
+ ADD_EXCL("java/lang/invoke/MethodHandleImpl$ArrayAccessor",
+ "OBJECT_ARRAY_GETTER", // D
+ "OBJECT_ARRAY_SETTER", // D
+ "OBJECT_ARRAY_LENGTH"); // D
+
}
# undef ADD_EXCL
@@ -143,6 +155,7 @@ CDSHeapVerifier::~CDSHeapVerifier() {
log_error(cds, heap)("Scanned %d objects. Found %d case(s) where "
"an object points to a static field that "
"may hold a different value at runtime.", _archived_objs, _problems);
+ log_error(cds, heap)("Please see cdsHeapVerifier.cpp and aotClassInitializer.cpp for details");
MetaspaceShared::unrecoverable_writing_error();
}
}
diff --git a/src/hotspot/share/cds/cds_globals.hpp b/src/hotspot/share/cds/cds_globals.hpp
index 811740cfbcb..2dae9b45221 100644
--- a/src/hotspot/share/cds/cds_globals.hpp
+++ b/src/hotspot/share/cds/cds_globals.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2022, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -71,6 +71,10 @@
"\"archivedObjects\" of the specified class is stored in the " \
"CDS archive heap") \
\
+ develop(ccstr, AOTInitTestClass, nullptr, \
+ "For JVM internal testing only. The specified class is stored " \
+ "in the initialized state in the AOT cache ") \
+ \
product(ccstr, DumpLoadedClassList, nullptr, \
"Dump the names all loaded classes, that could be stored into " \
"the CDS archive, in the specified file") \
@@ -105,7 +109,9 @@
constraint(AOTModeConstraintFunc, AtParse) \
\
product(ccstr, AOTConfiguration, nullptr, \
- "Configuration information used by CreateAOTCache") \
+ "The configuration file written by -XX:AOTMode=record, and " \
+ "loaded by -XX:AOTMode=create. This file contains profiling data "\
+ "for deciding what contents should be added to AOTCache. ") \
\
product(ccstr, AOTCache, nullptr, \
"Cache for improving start up and warm up") \
diff --git a/src/hotspot/share/cds/classListParser.cpp b/src/hotspot/share/cds/classListParser.cpp
index e0c008678ca..10154265bec 100644
--- a/src/hotspot/share/cds/classListParser.cpp
+++ b/src/hotspot/share/cds/classListParser.cpp
@@ -26,6 +26,7 @@
#include "cds/archiveUtils.hpp"
#include "cds/classListParser.hpp"
#include "cds/lambdaFormInvokers.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/metaspaceShared.hpp"
#include "cds/unregisteredClasses.hpp"
#include "classfile/classLoaderExt.hpp"
@@ -627,7 +628,7 @@ void ClassListParser::resolve_indy(JavaThread* current, Symbol* class_name_symbo
}
void ClassListParser::resolve_indy_impl(Symbol* class_name_symbol, TRAPS) {
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
// The CP entry for the invokedynamic instruction will be resolved.
// No need to do the following.
return;
@@ -657,7 +658,7 @@ void ClassListParser::resolve_indy_impl(Symbol* class_name_symbol, TRAPS) {
constantPoolHandle pool(THREAD, cp);
BootstrapInfo bootstrap_specifier(pool, pool_index, indy_index);
Handle bsm = bootstrap_specifier.resolve_bsm(CHECK);
- if (!SystemDictionaryShared::is_supported_invokedynamic(&bootstrap_specifier)) {
+ if (!LambdaProxyClassDictionary::is_supported_invokedynamic(&bootstrap_specifier)) {
log_debug(cds, lambda)("is_supported_invokedynamic check failed for cp_index %d", pool_index);
continue;
}
diff --git a/src/hotspot/share/cds/cppVtables.cpp b/src/hotspot/share/cds/cppVtables.cpp
index 39849571015..b8243cedf6d 100644
--- a/src/hotspot/share/cds/cppVtables.cpp
+++ b/src/hotspot/share/cds/cppVtables.cpp
@@ -189,12 +189,22 @@ enum ClonedVtableKind {
_num_cloned_vtable_kinds
};
-// This is a map of all the original vtptrs. E.g., for
+// _orig_cpp_vtptrs and _archived_cpp_vtptrs are used for type checking in
+// CppVtables::get_archived_vtable().
+//
+// _orig_cpp_vtptrs is a map of all the original vtptrs. E.g., for
// ConstantPool *cp = new (...) ConstantPool(...) ; // a dynamically allocated constant pool
// the following holds true:
-// _orig_cpp_vtptrs[ConstantPool_Kind] == ((intptr_t**)cp)[0]
-static intptr_t* _orig_cpp_vtptrs[_num_cloned_vtable_kinds];
+// _orig_cpp_vtptrs[ConstantPool_Kind] == ((intptr_t**)cp)[0]
+//
+// _archived_cpp_vtptrs is a map of all the vptprs used by classes in a preimage. E.g., for
+// InstanceKlass* k = a class loaded from the preimage;
+// ConstantPool* cp = k->constants();
+// the following holds true:
+// _archived_cpp_vtptrs[ConstantPool_Kind] == ((intptr_t**)cp)[0]
static bool _orig_cpp_vtptrs_inited = false;
+static intptr_t* _orig_cpp_vtptrs[_num_cloned_vtable_kinds];
+static intptr_t* _archived_cpp_vtptrs[_num_cloned_vtable_kinds];
template
void CppVtableCloner::init_orig_cpp_vtptr(int kind) {
@@ -212,15 +222,27 @@ void CppVtableCloner::init_orig_cpp_vtptr(int kind) {
// _index[InstanceKlass_Kind]->cloned_vtable() == ((intptr_t**)ik)[0]
static CppVtableInfo* _index[_num_cloned_vtable_kinds];
-// Vtables are all fixed offsets from ArchiveBuilder::current()->mapped_base()
-// E.g. ConstantPool is at offset 0x58. We can archive these offsets in the
-// RO region and use them to alculate their location at runtime without storing
-// the pointers in the RW region
+// This marks the location in the archive where _index[0] is stored. This location
+// will be stored as FileMapHeader::_cloned_vtables_offset into the archive header.
+// Serviceability Agent uses this information to determine the vtables of
+// archived Metadata objects.
char* CppVtables::_vtables_serialized_base = nullptr;
void CppVtables::dumptime_init(ArchiveBuilder* builder) {
assert(CDSConfig::is_dumping_static_archive(), "cpp tables are only dumped into static archive");
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ // When dumping final archive, _index[kind] at this point is in the preimage.
+ // Remember these vtable pointers in _archived_cpp_vtptrs, as _index[kind] will now be rewritten
+ // to point to the runtime vtable data.
+ for (int i = 0; i < _num_cloned_vtable_kinds; i++) {
+ assert(_index[i] != nullptr, "must have been restored by CppVtables::serialize()");
+ _archived_cpp_vtptrs[i] = _index[i]->cloned_vtable();
+ }
+ } else {
+ memset(_archived_cpp_vtptrs, 0, sizeof(_archived_cpp_vtptrs));
+ }
+
CPP_VTABLE_TYPES_DO(ALLOCATE_AND_INITIALIZE_VTABLE);
size_t cpp_tables_size = builder->rw_region()->top() - builder->rw_region()->base();
@@ -267,7 +289,8 @@ intptr_t* CppVtables::get_archived_vtable(MetaspaceObj::Type msotype, address ob
break;
default:
for (kind = 0; kind < _num_cloned_vtable_kinds; kind ++) {
- if (vtable_of((Metadata*)obj) == _orig_cpp_vtptrs[kind]) {
+ if (vtable_of((Metadata*)obj) == _orig_cpp_vtptrs[kind] ||
+ vtable_of((Metadata*)obj) == _archived_cpp_vtptrs[kind]) {
break;
}
}
@@ -295,5 +318,6 @@ void CppVtables::zero_archived_vtables() {
bool CppVtables::is_valid_shared_method(const Method* m) {
assert(MetaspaceShared::is_in_shared_metaspace(m), "must be");
- return vtable_of(m) == _index[Method_Kind]->cloned_vtable();
+ return vtable_of(m) == _index[Method_Kind]->cloned_vtable() ||
+ vtable_of(m) == _archived_cpp_vtptrs[Method_Kind];
}
diff --git a/src/hotspot/share/cds/dumpTimeClassInfo.cpp b/src/hotspot/share/cds/dumpTimeClassInfo.cpp
index 18136d6eeec..94a0f14ff1f 100644
--- a/src/hotspot/share/cds/dumpTimeClassInfo.cpp
+++ b/src/hotspot/share/cds/dumpTimeClassInfo.cpp
@@ -66,9 +66,9 @@ void DumpTimeClassInfo::add_verification_constraint(InstanceKlass* k, Symbol* na
GrowableArray* vcflags_array = _verifier_constraint_flags;
char c = 0;
- c |= from_field_is_protected ? SystemDictionaryShared::FROM_FIELD_IS_PROTECTED : 0;
- c |= from_is_array ? SystemDictionaryShared::FROM_IS_ARRAY : 0;
- c |= from_is_object ? SystemDictionaryShared::FROM_IS_OBJECT : 0;
+ c |= from_field_is_protected ? RunTimeClassInfo::FROM_FIELD_IS_PROTECTED : 0;
+ c |= from_is_array ? RunTimeClassInfo::FROM_IS_ARRAY : 0;
+ c |= from_is_object ? RunTimeClassInfo::FROM_IS_OBJECT : 0;
vcflags_array->append(c);
if (log_is_enabled(Trace, cds, verification)) {
@@ -142,7 +142,7 @@ bool DumpTimeClassInfo::is_builtin() {
}
DumpTimeClassInfo* DumpTimeSharedClassTable::allocate_info(InstanceKlass* k) {
- assert(!k->is_shared(), "Do not call with shared classes");
+ assert(CDSConfig::is_dumping_final_static_archive() || !k->is_shared(), "Do not call with shared classes");
bool created;
DumpTimeClassInfo* p = put_if_absent(k, &created);
assert(created, "must not exist in table");
@@ -151,7 +151,7 @@ DumpTimeClassInfo* DumpTimeSharedClassTable::allocate_info(InstanceKlass* k) {
}
DumpTimeClassInfo* DumpTimeSharedClassTable::get_info(InstanceKlass* k) {
- assert(!k->is_shared(), "Do not call with shared classes");
+ assert(CDSConfig::is_dumping_final_static_archive() || !k->is_shared(), "Do not call with shared classes");
DumpTimeClassInfo* p = get(k);
assert(p != nullptr, "we must not see any non-shared InstanceKlass* that's "
"not stored with SystemDictionaryShared::init_dumptime_info");
diff --git a/src/hotspot/share/cds/dumpTimeClassInfo.hpp b/src/hotspot/share/cds/dumpTimeClassInfo.hpp
index 79ede224bb6..0bb60dfbb29 100644
--- a/src/hotspot/share/cds/dumpTimeClassInfo.hpp
+++ b/src/hotspot/share/cds/dumpTimeClassInfo.hpp
@@ -39,9 +39,11 @@ class Method;
class Symbol;
class DumpTimeClassInfo: public CHeapObj {
- bool _excluded;
- bool _is_early_klass;
- bool _has_checked_exclusion;
+ bool _excluded;
+ bool _is_aot_tooling_class;
+ bool _is_early_klass;
+ bool _has_checked_exclusion;
+
class DTLoaderConstraint {
Symbol* _name;
char _loader_type1;
@@ -140,6 +142,7 @@ public:
_clsfile_size = -1;
_clsfile_crc32 = -1;
_excluded = false;
+ _is_aot_tooling_class = false;
_is_early_klass = JvmtiExport::is_early_phase();
_verifier_constraints = nullptr;
_verifier_constraint_flags = nullptr;
@@ -199,6 +202,14 @@ public:
return _excluded || _failed_verification;
}
+ bool is_aot_tooling_class() {
+ return _is_aot_tooling_class;
+ }
+
+ void set_is_aot_tooling_class() {
+ _is_aot_tooling_class = true;
+ }
+
// Was this class loaded while JvmtiExport::is_early_phase()==true
bool is_early_klass() {
return _is_early_klass;
diff --git a/src/hotspot/share/cds/dynamicArchive.cpp b/src/hotspot/share/cds/dynamicArchive.cpp
index 4ccf23ff91c..c3a6db9e9b9 100644
--- a/src/hotspot/share/cds/dynamicArchive.cpp
+++ b/src/hotspot/share/cds/dynamicArchive.cpp
@@ -31,6 +31,9 @@
#include "cds/cds_globals.hpp"
#include "cds/cdsConfig.hpp"
#include "cds/dynamicArchive.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
+#include "cds/lambdaFormInvokers.hpp"
+#include "cds/metaspaceShared.hpp"
#include "cds/regeneratedClasses.hpp"
#include "classfile/classLoader.hpp"
#include "classfile/classLoaderData.inline.hpp"
@@ -159,8 +162,10 @@ public:
ArchiveBuilder::serialize_dynamic_archivable_items(&wc);
}
- log_info(cds)("Adjust lambda proxy class dictionary");
- SystemDictionaryShared::adjust_lambda_proxy_class_dictionary();
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ log_info(cds)("Adjust lambda proxy class dictionary");
+ LambdaProxyClassDictionary::adjust_dumptime_table();
+ }
relocate_to_requested();
@@ -170,7 +175,6 @@ public:
post_dump();
- assert(_num_dump_regions_used == _total_dump_regions, "must be");
verify_universe("After CDS dynamic dump");
}
@@ -388,8 +392,9 @@ public:
void doit() {
ResourceMark rm;
if (AllowArchivingWithJavaAgent) {
- log_warning(cds)("This archive was created with AllowArchivingWithJavaAgent. It should be used "
- "for testing purposes only and should not be used in a production environment");
+ log_warning(cds)("This %s was created with AllowArchivingWithJavaAgent. It should be used "
+ "for testing purposes only and should not be used in a production environment",
+ CDSConfig::type_of_archive_being_loaded());
}
AOTClassLocationConfig::dumptime_check_nonempty_dirs();
_builder.doit();
@@ -492,6 +497,16 @@ void DynamicArchive::check_for_dynamic_dump() {
}
}
+void DynamicArchive::dump_impl(bool jcmd_request, const char* archive_name, TRAPS) {
+ MetaspaceShared::link_shared_classes(CHECK);
+ if (!jcmd_request && CDSConfig::is_dumping_regenerated_lambdaform_invokers()) {
+ LambdaFormInvokers::regenerate_holder_classes(CHECK);
+ }
+
+ VM_PopulateDynamicDumpSharedSpace op(archive_name);
+ VMThread::execute(&op);
+}
+
void DynamicArchive::dump_at_exit(JavaThread* current, const char* archive_name) {
ExceptionMark em(current);
ResourceMark rm(current);
@@ -504,20 +519,16 @@ void DynamicArchive::dump_at_exit(JavaThread* current, const char* archive_name)
log_info(cds, dynamic)("Preparing for dynamic dump at exit in thread %s", current->name());
JavaThread* THREAD = current; // For TRAPS processing related to link_shared_classes
- MetaspaceShared::link_shared_classes(false/*not from jcmd*/, THREAD);
- if (!HAS_PENDING_EXCEPTION) {
- VM_PopulateDynamicDumpSharedSpace op(archive_name);
- VMThread::execute(&op);
- return;
+ dump_impl(/*jcmd_request=*/false, archive_name, THREAD);
+ if (HAS_PENDING_EXCEPTION) {
+ // One of the prepatory steps failed
+ oop ex = current->pending_exception();
+ log_error(cds)("Dynamic dump has failed");
+ log_error(cds)("%s: %s", ex->klass()->external_name(),
+ java_lang_String::as_utf8_string(java_lang_Throwable::message(ex)));
+ CLEAR_PENDING_EXCEPTION;
+ CDSConfig::disable_dumping_dynamic_archive(); // Just for good measure
}
-
- // One of the prepatory steps failed
- oop ex = current->pending_exception();
- log_error(cds)("Dynamic dump has failed");
- log_error(cds)("%s: %s", ex->klass()->external_name(),
- java_lang_String::as_utf8_string(java_lang_Throwable::message(ex)));
- CLEAR_PENDING_EXCEPTION;
- CDSConfig::disable_dumping_dynamic_archive(); // Just for good measure
}
// This is called by "jcmd VM.cds dynamic_dump"
@@ -526,10 +537,7 @@ void DynamicArchive::dump_for_jcmd(const char* archive_name, TRAPS) {
assert(CDSConfig::is_using_archive() && RecordDynamicDumpInfo, "already checked in arguments.cpp");
assert(ArchiveClassesAtExit == nullptr, "already checked in arguments.cpp");
assert(CDSConfig::is_dumping_dynamic_archive(), "already checked by check_for_dynamic_dump() during VM startup");
- MetaspaceShared::link_shared_classes(true/*from jcmd*/, CHECK);
- // copy shared path table to saved.
- VM_PopulateDynamicDumpSharedSpace op(archive_name);
- VMThread::execute(&op);
+ dump_impl(/*jcmd_request=*/true, archive_name, CHECK);
}
bool DynamicArchive::validate(FileMapInfo* dynamic_info) {
diff --git a/src/hotspot/share/cds/dynamicArchive.hpp b/src/hotspot/share/cds/dynamicArchive.hpp
index eb5fd5f9aba..905c511c4e0 100644
--- a/src/hotspot/share/cds/dynamicArchive.hpp
+++ b/src/hotspot/share/cds/dynamicArchive.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -66,6 +66,7 @@ public:
static void check_for_dynamic_dump();
static void dump_for_jcmd(const char* archive_name, TRAPS);
static void dump_at_exit(JavaThread* current, const char* archive_name);
+ static void dump_impl(bool jcmd_request, const char* archive_name, TRAPS);
static bool is_mapped() { return FileMapInfo::dynamic_info() != nullptr; }
static bool validate(FileMapInfo* dynamic_info);
static void dump_array_klasses();
diff --git a/src/hotspot/share/cds/filemap.cpp b/src/hotspot/share/cds/filemap.cpp
index 015032b2d38..ebcd33f8bd5 100644
--- a/src/hotspot/share/cds/filemap.cpp
+++ b/src/hotspot/share/cds/filemap.cpp
@@ -151,6 +151,13 @@ FileMapInfo::~FileMapInfo() {
}
}
+void FileMapInfo::free_current_info() {
+ assert(CDSConfig::is_dumping_final_static_archive(), "only supported in this mode");
+ assert(_current_info != nullptr, "sanity");
+ delete _current_info;
+ assert(_current_info == nullptr, "sanity"); // Side effect expected from the above "delete" operator.
+}
+
void FileMapInfo::populate_header(size_t core_region_alignment) {
assert(_header == nullptr, "Sanity check");
size_t c_header_size;
@@ -191,7 +198,13 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
set_header_size((unsigned int)header_size);
set_base_archive_name_offset((unsigned int)base_archive_name_offset);
set_base_archive_name_size((unsigned int)base_archive_name_size);
- set_magic(CDSConfig::is_dumping_dynamic_archive() ? CDS_DYNAMIC_ARCHIVE_MAGIC : CDS_ARCHIVE_MAGIC);
+ if (CDSConfig::is_dumping_dynamic_archive()) {
+ set_magic(CDS_DYNAMIC_ARCHIVE_MAGIC);
+ } else if (CDSConfig::is_dumping_preimage_static_archive()) {
+ set_magic(CDS_PREIMAGE_ARCHIVE_MAGIC);
+ } else {
+ set_magic(CDS_ARCHIVE_MAGIC);
+ }
set_version(CURRENT_CDS_ARCHIVE_VERSION);
if (!info->is_static() && base_archive_name_size != 0) {
@@ -221,7 +234,6 @@ void FileMapHeader::populate(FileMapInfo *info, size_t core_region_alignment,
_use_optimized_module_handling = CDSConfig::is_using_optimized_module_handling();
_has_aot_linked_classes = CDSConfig::is_dumping_aot_linked_classes();
_has_full_module_graph = CDSConfig::is_dumping_full_module_graph();
- _has_archived_invokedynamic = CDSConfig::is_dumping_invokedynamic();
// The following fields are for sanity checks for whether this archive
// will function correctly with this JVM and the bootclasspath it's
@@ -296,14 +308,13 @@ void FileMapHeader::print(outputStream* st) {
st->print_cr("- use_optimized_module_handling: %d", _use_optimized_module_handling);
st->print_cr("- has_full_module_graph %d", _has_full_module_graph);
st->print_cr("- has_aot_linked_classes %d", _has_aot_linked_classes);
- st->print_cr("- has_archived_invokedynamic %d", _has_archived_invokedynamic);
}
bool FileMapInfo::validate_class_location() {
assert(CDSConfig::is_using_archive(), "runtime only");
AOTClassLocationConfig* config = header()->class_location_config();
- bool has_extra_module_paths;
+ bool has_extra_module_paths = false;
if (!config->validate(header()->has_aot_linked_classes(), &has_extra_module_paths)) {
if (PrintSharedArchiveAndExit) {
MetaspaceShared::set_archive_loading_failed();
@@ -386,7 +397,7 @@ public:
assert(_archive_name != nullptr, "Archive name is null");
_fd = os::open(_archive_name, O_RDONLY | O_BINARY, 0);
if (_fd < 0) {
- log_info(cds)("Specified shared archive not found (%s)", _archive_name);
+ log_info(cds)("Specified %s not found (%s)", CDSConfig::type_of_archive_being_loaded(), _archive_name);
return false;
}
return initialize(_fd);
@@ -397,30 +408,32 @@ public:
assert(_archive_name != nullptr, "Archive name is null");
assert(fd != -1, "Archive must be opened already");
// First read the generic header so we know the exact size of the actual header.
+ const char* file_type = CDSConfig::type_of_archive_being_loaded();
GenericCDSFileMapHeader gen_header;
size_t size = sizeof(GenericCDSFileMapHeader);
os::lseek(fd, 0, SEEK_SET);
size_t n = ::read(fd, (void*)&gen_header, (unsigned int)size);
if (n != size) {
- log_warning(cds)("Unable to read generic CDS file map header from shared archive");
+ log_warning(cds)("Unable to read generic CDS file map header from %s", file_type);
return false;
}
if (gen_header._magic != CDS_ARCHIVE_MAGIC &&
- gen_header._magic != CDS_DYNAMIC_ARCHIVE_MAGIC) {
- log_warning(cds)("The shared archive file has a bad magic number: %#x", gen_header._magic);
+ gen_header._magic != CDS_DYNAMIC_ARCHIVE_MAGIC &&
+ gen_header._magic != CDS_PREIMAGE_ARCHIVE_MAGIC) {
+ log_warning(cds)("The %s has a bad magic number: %#x", file_type, gen_header._magic);
return false;
}
if (gen_header._version < CDS_GENERIC_HEADER_SUPPORTED_MIN_VERSION) {
- log_warning(cds)("Cannot handle shared archive file version 0x%x. Must be at least 0x%x.",
- gen_header._version, CDS_GENERIC_HEADER_SUPPORTED_MIN_VERSION);
+ log_warning(cds)("Cannot handle %s version 0x%x. Must be at least 0x%x.",
+ file_type, gen_header._version, CDS_GENERIC_HEADER_SUPPORTED_MIN_VERSION);
return false;
}
if (gen_header._version != CURRENT_CDS_ARCHIVE_VERSION) {
- log_warning(cds)("The shared archive file version 0x%x does not match the required version 0x%x.",
- gen_header._version, CURRENT_CDS_ARCHIVE_VERSION);
+ log_warning(cds)("The %s version 0x%x does not match the required version 0x%x.",
+ file_type, gen_header._version, CURRENT_CDS_ARCHIVE_VERSION);
}
size_t filelen = os::lseek(fd, 0, SEEK_END);
@@ -435,7 +448,7 @@ public:
os::lseek(fd, 0, SEEK_SET);
n = ::read(fd, (void*)_header, (unsigned int)size);
if (n != size) {
- log_warning(cds)("Unable to read actual CDS file map header from shared archive");
+ log_warning(cds)("Unable to read file map header from %s", file_type);
return false;
}
@@ -462,6 +475,18 @@ public:
return _base_archive_name;
}
+ bool is_static_archive() const {
+ return _header->_magic == CDS_ARCHIVE_MAGIC;
+ }
+
+ bool is_dynamic_archive() const {
+ return _header->_magic == CDS_DYNAMIC_ARCHIVE_MAGIC;
+ }
+
+ bool is_preimage_static_archive() const {
+ return _header->_magic == CDS_PREIMAGE_ARCHIVE_MAGIC;
+ }
+
private:
bool check_header_crc() const {
if (VerifySharedSpaces) {
@@ -487,7 +512,8 @@ public:
name_offset, name_size);
return false;
}
- if (_header->_magic == CDS_ARCHIVE_MAGIC) {
+
+ if (is_static_archive() || is_preimage_static_archive()) {
if (name_offset != 0) {
log_warning(cds)("static shared archive must have zero _base_archive_name_offset");
return false;
@@ -497,7 +523,7 @@ public:
return false;
}
} else {
- assert(_header->_magic == CDS_DYNAMIC_ARCHIVE_MAGIC, "must be");
+ assert(is_dynamic_archive(), "must be");
if ((name_size == 0 && name_offset != 0) ||
(name_size != 0 && name_offset == 0)) {
// If either is zero, both must be zero. This indicates that we are using the default base archive.
@@ -545,7 +571,12 @@ bool FileMapInfo::get_base_archive_name_from_header(const char* archive_name,
return false;
}
GenericCDSFileMapHeader* header = file_helper.get_generic_file_header();
- if (header->_magic != CDS_DYNAMIC_ARCHIVE_MAGIC) {
+ switch (header->_magic) {
+ case CDS_PREIMAGE_ARCHIVE_MAGIC:
+ return false; // This is a binary config file, not a proper archive
+ case CDS_DYNAMIC_ARCHIVE_MAGIC:
+ break;
+ default:
assert(header->_magic == CDS_ARCHIVE_MAGIC, "must be");
if (AutoCreateSharedArchive) {
log_warning(cds)("AutoCreateSharedArchive is ignored because %s is a static archive", archive_name);
@@ -563,6 +594,14 @@ bool FileMapInfo::get_base_archive_name_from_header(const char* archive_name,
return true;
}
+bool FileMapInfo::is_preimage_static_archive(const char* file) {
+ FileHeaderHelper file_helper(file, false);
+ if (!file_helper.initialize()) {
+ return false;
+ }
+ return file_helper.is_preimage_static_archive();
+}
+
// Read the FileMapInfo information from the file.
bool FileMapInfo::init_from_file(int fd) {
@@ -573,9 +612,17 @@ bool FileMapInfo::init_from_file(int fd) {
}
GenericCDSFileMapHeader* gen_header = file_helper.get_generic_file_header();
+ const char* file_type = CDSConfig::type_of_archive_being_loaded();
if (_is_static) {
- if (gen_header->_magic != CDS_ARCHIVE_MAGIC) {
- log_warning(cds)("Not a base shared archive: %s", _full_path);
+ if ((gen_header->_magic == CDS_ARCHIVE_MAGIC) ||
+ (gen_header->_magic == CDS_PREIMAGE_ARCHIVE_MAGIC && CDSConfig::is_dumping_final_static_archive())) {
+ // Good
+ } else {
+ if (CDSConfig::new_aot_flags_used()) {
+ log_warning(cds)("Not a valid %s %s", file_type, _full_path);
+ } else {
+ log_warning(cds)("Not a base shared archive: %s", _full_path);
+ }
return false;
}
} else {
@@ -597,7 +644,7 @@ bool FileMapInfo::init_from_file(int fd) {
if (header()->version() != CURRENT_CDS_ARCHIVE_VERSION) {
log_info(cds)("_version expected: 0x%x", CURRENT_CDS_ARCHIVE_VERSION);
log_info(cds)(" actual: 0x%x", header()->version());
- log_warning(cds)("The shared archive file has the wrong version.");
+ log_warning(cds)("The %s has the wrong version.", file_type);
return false;
}
@@ -609,7 +656,7 @@ bool FileMapInfo::init_from_file(int fd) {
log_info(cds)("_header_size: " UINT32_FORMAT, header_size);
log_info(cds)("base_archive_name_size: " UINT32_FORMAT, header()->base_archive_name_size());
log_info(cds)("base_archive_name_offset: " UINT32_FORMAT, header()->base_archive_name_offset());
- log_warning(cds)("The shared archive file has an incorrect header size.");
+ log_warning(cds)("The %s has an incorrect header size.", file_type);
return false;
}
}
@@ -626,8 +673,8 @@ bool FileMapInfo::init_from_file(int fd) {
if (strncmp(actual_ident, expected_ident, JVM_IDENT_MAX-1) != 0) {
log_info(cds)("_jvm_ident expected: %s", expected_ident);
log_info(cds)(" actual: %s", actual_ident);
- log_warning(cds)("The shared archive file was created by a different"
- " version or build of HotSpot");
+ log_warning(cds)("The %s was created by a different"
+ " version or build of HotSpot", file_type);
return false;
}
@@ -638,7 +685,7 @@ bool FileMapInfo::init_from_file(int fd) {
for (int i = 0; i < MetaspaceShared::n_regions; i++) {
FileMapRegion* r = region_at(i);
if (r->file_offset() > len || len - r->file_offset() < r->used()) {
- log_warning(cds)("The shared archive file has been truncated.");
+ log_warning(cds)("The %s has been truncated.", file_type);
return false;
}
}
@@ -658,18 +705,21 @@ bool FileMapInfo::open_for_read() {
if (_file_open) {
return true;
}
- log_info(cds)("trying to map %s", _full_path);
+ const char* file_type = CDSConfig::type_of_archive_being_loaded();
+ const char* info = CDSConfig::is_dumping_final_static_archive() ?
+ "AOTConfiguration file " : "";
+ log_info(cds)("trying to map %s%s", info, _full_path);
int fd = os::open(_full_path, O_RDONLY | O_BINARY, 0);
if (fd < 0) {
if (errno == ENOENT) {
- log_info(cds)("Specified shared archive not found (%s)", _full_path);
+ log_info(cds)("Specified %s not found (%s)", file_type, _full_path);
} else {
- log_warning(cds)("Failed to open shared archive file (%s)",
+ log_warning(cds)("Failed to open %s (%s)", file_type,
os::strerror(errno));
}
return false;
} else {
- log_info(cds)("Opened archive %s.", _full_path);
+ log_info(cds)("Opened %s %s.", file_type, _full_path);
}
_fd = fd;
@@ -682,20 +732,25 @@ bool FileMapInfo::open_for_read() {
void FileMapInfo::open_for_write() {
LogMessage(cds) msg;
if (msg.is_info()) {
- msg.info("Dumping shared data to file: ");
+ if (CDSConfig::is_dumping_preimage_static_archive()) {
+ msg.info("Writing binary AOTConfiguration file: ");
+ } else {
+ msg.info("Dumping shared data to file: ");
+ }
msg.info(" %s", _full_path);
}
#ifdef _WINDOWS // On Windows, need WRITE permission to remove the file.
- chmod(_full_path, _S_IREAD | _S_IWRITE);
+ chmod(_full_path, _S_IREAD | _S_IWRITE);
#endif
// Use remove() to delete the existing file because, on Unix, this will
// allow processes that have it open continued access to the file.
remove(_full_path);
- int fd = os::open(_full_path, O_RDWR | O_CREAT | O_TRUNC | O_BINARY, 0444);
+ int mode = CDSConfig::is_dumping_preimage_static_archive() ? 0666 : 0444;
+ int fd = os::open(_full_path, O_RDWR | O_CREAT | O_TRUNC | O_BINARY, mode);
if (fd < 0) {
- log_error(cds)("Unable to create shared archive file %s: (%s).", _full_path,
+ log_error(cds)("Unable to create %s %s: (%s).", CDSConfig::type_of_archive_being_written(), _full_path,
os::strerror(errno));
MetaspaceShared::writing_error();
return;
@@ -951,7 +1006,14 @@ void FileMapInfo::write_bytes(const void* buffer, size_t nbytes) {
// If the shared archive is corrupted, close it and remove it.
close();
remove(_full_path);
- MetaspaceShared::writing_error("Unable to write to shared archive file.");
+
+ if (CDSConfig::is_dumping_preimage_static_archive()) {
+ MetaspaceShared::writing_error("Unable to write to AOT configuration file.");
+ } else if (CDSConfig::new_aot_flags_used()) {
+ MetaspaceShared::writing_error("Unable to write to AOT cache.");
+ } else {
+ MetaspaceShared::writing_error("Unable to write to shared archive.");
+ }
}
_file_offset += nbytes;
}
@@ -1793,15 +1855,16 @@ int FileMapHeader::compute_crc() {
// This function should only be called during run time with UseSharedSpaces enabled.
bool FileMapHeader::validate() {
+ const char* file_type = CDSConfig::type_of_archive_being_loaded();
if (_obj_alignment != ObjectAlignmentInBytes) {
- log_info(cds)("The shared archive file's ObjectAlignmentInBytes of %d"
+ log_info(cds)("The %s's ObjectAlignmentInBytes of %d"
" does not equal the current ObjectAlignmentInBytes of %d.",
- _obj_alignment, ObjectAlignmentInBytes);
+ file_type, _obj_alignment, ObjectAlignmentInBytes);
return false;
}
if (_compact_strings != CompactStrings) {
- log_info(cds)("The shared archive file's CompactStrings setting (%s)"
- " does not equal the current CompactStrings setting (%s).",
+ log_info(cds)("The %s's CompactStrings setting (%s)"
+ " does not equal the current CompactStrings setting (%s).", file_type,
_compact_strings ? "enabled" : "disabled",
CompactStrings ? "enabled" : "disabled");
return false;
@@ -1825,8 +1888,8 @@ bool FileMapHeader::validate() {
if (!_verify_local && BytecodeVerificationLocal) {
// we cannot load boot classes, so there's no point of using the CDS archive
- log_info(cds)("The shared archive file's BytecodeVerificationLocal setting (%s)"
- " does not equal the current BytecodeVerificationLocal setting (%s).",
+ log_info(cds)("The %s's BytecodeVerificationLocal setting (%s)"
+ " does not equal the current BytecodeVerificationLocal setting (%s).", file_type,
_verify_local ? "enabled" : "disabled",
BytecodeVerificationLocal ? "enabled" : "disabled");
return false;
@@ -1837,8 +1900,8 @@ bool FileMapHeader::validate() {
if (_has_platform_or_app_classes
&& !_verify_remote // we didn't verify the archived platform/app classes
&& BytecodeVerificationRemote) { // but we want to verify all loaded platform/app classes
- log_info(cds)("The shared archive file was created with less restrictive "
- "verification setting than the current setting.");
+ log_info(cds)("The %s was created with less restrictive "
+ "verification setting than the current setting.", file_type);
// Pretend that we didn't have any archived platform/app classes, so they won't be loaded
// by SystemDictionaryShared.
_has_platform_or_app_classes = false;
@@ -1850,32 +1913,32 @@ bool FileMapHeader::validate() {
// while AllowArchivingWithJavaAgent is set during the current run.
if (_allow_archiving_with_java_agent && !AllowArchivingWithJavaAgent) {
log_warning(cds)("The setting of the AllowArchivingWithJavaAgent is different "
- "from the setting in the shared archive.");
+ "from the setting in the %s.", file_type);
return false;
}
if (_allow_archiving_with_java_agent) {
- log_warning(cds)("This archive was created with AllowArchivingWithJavaAgent. It should be used "
- "for testing purposes only and should not be used in a production environment");
+ log_warning(cds)("This %s was created with AllowArchivingWithJavaAgent. It should be used "
+ "for testing purposes only and should not be used in a production environment", file_type);
}
- log_info(cds)("Archive was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d",
- compressed_oops(), compressed_class_pointers(), compact_headers());
+ log_info(cds)("The %s was created with UseCompressedOops = %d, UseCompressedClassPointers = %d, UseCompactObjectHeaders = %d",
+ file_type, compressed_oops(), compressed_class_pointers(), compact_headers());
if (compressed_oops() != UseCompressedOops || compressed_class_pointers() != UseCompressedClassPointers) {
- log_warning(cds)("Unable to use shared archive.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
- "different from runtime, CDS will be disabled.");
+ log_warning(cds)("Unable to use %s.\nThe saved state of UseCompressedOops and UseCompressedClassPointers is "
+ "different from runtime, CDS will be disabled.", file_type);
return false;
}
if (compact_headers() != UseCompactObjectHeaders) {
- log_warning(cds)("Unable to use shared archive.\nThe shared archive file's UseCompactObjectHeaders setting (%s)"
- " does not equal the current UseCompactObjectHeaders setting (%s).",
+ log_warning(cds)("Unable to use %s.\nThe %s's UseCompactObjectHeaders setting (%s)"
+ " does not equal the current UseCompactObjectHeaders setting (%s).", file_type, file_type,
_compact_headers ? "enabled" : "disabled",
UseCompactObjectHeaders ? "enabled" : "disabled");
return false;
}
- if (!_use_optimized_module_handling) {
+ if (!_use_optimized_module_handling && !CDSConfig::is_dumping_final_static_archive()) {
CDSConfig::stop_using_optimized_module_handling();
log_info(cds)("optimized module handling: disabled because archive was created without optimized module handling");
}
@@ -1885,10 +1948,6 @@ bool FileMapHeader::validate() {
if (!_has_full_module_graph) {
CDSConfig::stop_using_full_module_graph("archive was created without full module graph");
}
-
- if (_has_archived_invokedynamic) {
- CDSConfig::set_has_archived_invokedynamic();
- }
}
return true;
diff --git a/src/hotspot/share/cds/filemap.hpp b/src/hotspot/share/cds/filemap.hpp
index 22f70635e17..50af87a5da7 100644
--- a/src/hotspot/share/cds/filemap.hpp
+++ b/src/hotspot/share/cds/filemap.hpp
@@ -140,7 +140,6 @@ private:
// some expensive operations.
bool _has_aot_linked_classes; // Was the CDS archive created with -XX:+AOTClassLinking
bool _has_full_module_graph; // Does this CDS archive contain the full archived module graph?
- bool _has_archived_invokedynamic; // Does the archive have aot-linked invokedynamic CP entries?
HeapRootSegments _heap_root_segments; // Heap root segments info
size_t _heap_oopmap_start_pos; // The first bit in the oopmap corresponds to this position in the heap.
size_t _heap_ptrmap_start_pos; // The first bit in the ptrmap corresponds to this position in the heap.
@@ -270,12 +269,15 @@ public:
FileMapHeader *header() const { return _header; }
static bool get_base_archive_name_from_header(const char* archive_name,
char** base_archive_name);
+ static bool is_preimage_static_archive(const char* file);
+
bool init_from_file(int fd);
void log_paths(const char* msg, int start_idx, int end_idx);
FileMapInfo(const char* full_apth, bool is_static);
~FileMapInfo();
+ static void free_current_info();
// Accessors
int compute_header_crc() const { return header()->compute_crc(); }
@@ -397,7 +399,7 @@ public:
// The offset of the (exclusive) end of the last core region in this archive, relative to SharedBaseAddress
size_t mapping_end_offset() const { return last_core_region()->mapping_end_offset(); }
- char* mapped_base() const { return first_core_region()->mapped_base(); }
+ char* mapped_base() const { return header()->mapped_base_address(); }
char* mapped_end() const { return last_core_region()->mapped_end(); }
// Non-zero if the archive needs to be mapped a non-default location due to ASLR.
diff --git a/src/hotspot/share/cds/finalImageRecipes.cpp b/src/hotspot/share/cds/finalImageRecipes.cpp
new file mode 100644
index 00000000000..55855679a1c
--- /dev/null
+++ b/src/hotspot/share/cds/finalImageRecipes.cpp
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "cds/aotConstantPoolResolver.hpp"
+#include "cds/archiveBuilder.hpp"
+#include "cds/archiveUtils.inline.hpp"
+#include "cds/cdsConfig.hpp"
+#include "cds/finalImageRecipes.hpp"
+#include "classfile/classLoader.hpp"
+#include "classfile/javaClasses.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "classfile/systemDictionaryShared.hpp"
+#include "classfile/vmClasses.hpp"
+#include "memory/oopFactory.hpp"
+#include "memory/resourceArea.hpp"
+#include "oops/constantPool.inline.hpp"
+#include "runtime/handles.inline.hpp"
+
+static FinalImageRecipes* _final_image_recipes = nullptr;
+
+void* FinalImageRecipes::operator new(size_t size) throw() {
+ return ArchiveBuilder::current()->ro_region_alloc(size);
+}
+
+void FinalImageRecipes::record_recipes_impl() {
+ assert(CDSConfig::is_dumping_preimage_static_archive(), "must be");
+ ResourceMark rm;
+ GrowableArray* klasses = ArchiveBuilder::current()->klasses();
+
+ // Record the indys that have been resolved in the training run. These indys will be
+ // resolved during the final image assembly.
+
+ GrowableArray tmp_indy_klasses;
+ GrowableArray*> tmp_indy_cp_indices;
+ int total_indys_to_resolve = 0;
+ for (int i = 0; i < klasses->length(); i++) {
+ Klass* k = klasses->at(i);
+ if (k->is_instance_klass()) {
+ InstanceKlass* ik = InstanceKlass::cast(k);
+ GrowableArray indices;
+
+ if (ik->constants()->cache() != nullptr) {
+ Array* tmp_indy_entries = ik->constants()->cache()->resolved_indy_entries();
+ if (tmp_indy_entries != nullptr) {
+ for (int i = 0; i < tmp_indy_entries->length(); i++) {
+ ResolvedIndyEntry* rie = tmp_indy_entries->adr_at(i);
+ int cp_index = rie->constant_pool_index();
+ if (rie->is_resolved()) {
+ indices.append(cp_index);
+ }
+ }
+ }
+ }
+
+ if (indices.length() > 0) {
+ tmp_indy_klasses.append(ArchiveBuilder::current()->get_buffered_addr(ik));
+ tmp_indy_cp_indices.append(ArchiveUtils::archive_array(&indices));
+ total_indys_to_resolve += indices.length();
+ }
+ }
+ }
+
+ _all_klasses = ArchiveUtils::archive_array(klasses);
+ ArchivePtrMarker::mark_pointer(&_all_klasses);
+
+ assert(tmp_indy_klasses.length() == tmp_indy_cp_indices.length(), "must be");
+ if (tmp_indy_klasses.length() > 0) {
+ _indy_klasses = ArchiveUtils::archive_array(&tmp_indy_klasses);
+ _indy_cp_indices = ArchiveUtils::archive_array(&tmp_indy_cp_indices);
+
+ ArchivePtrMarker::mark_pointer(&_indy_klasses);
+ ArchivePtrMarker::mark_pointer(&_indy_cp_indices);
+ }
+ log_info(cds)("%d indies in %d classes will be resolved in final CDS image", total_indys_to_resolve, tmp_indy_klasses.length());
+}
+
+void FinalImageRecipes::load_all_classes(TRAPS) {
+ assert(CDSConfig::is_dumping_final_static_archive(), "sanity");
+ Handle class_loader(THREAD, SystemDictionary::java_system_loader());
+ for (int i = 0; i < _all_klasses->length(); i++) {
+ Klass* k = _all_klasses->at(i);
+ if (k->is_instance_klass()) {
+ InstanceKlass* ik = InstanceKlass::cast(k);
+ if (!ik->is_shared_unregistered_class() && !ik->is_hidden()) {
+ Klass* actual = SystemDictionary::resolve_or_fail(ik->name(), class_loader, true, CHECK);
+ if (actual != ik) {
+ ResourceMark rm(THREAD);
+ log_error(cds)("Unable to resolve class from CDS archive: %s", ik->external_name());
+ log_error(cds)("Expected: " INTPTR_FORMAT ", actual: " INTPTR_FORMAT, p2i(ik), p2i(actual));
+ log_error(cds)("Please check if your VM command-line is the same as in the training run");
+ MetaspaceShared::unrecoverable_writing_error();
+ }
+ assert(ik->is_loaded(), "must be");
+ ik->link_class(CHECK);
+ }
+ }
+ }
+}
+
+void FinalImageRecipes::apply_recipes_for_invokedynamic(TRAPS) {
+ assert(CDSConfig::is_dumping_final_static_archive(), "must be");
+
+ if (CDSConfig::is_dumping_invokedynamic() && _indy_klasses != nullptr) {
+ assert(_indy_cp_indices != nullptr, "must be");
+ for (int i = 0; i < _indy_klasses->length(); i++) {
+ InstanceKlass* ik = _indy_klasses->at(i);
+ ConstantPool* cp = ik->constants();
+ Array* cp_indices = _indy_cp_indices->at(i);
+ GrowableArray preresolve_list(cp->length(), cp->length(), false);
+ for (int j = 0; j < cp_indices->length(); j++) {
+ preresolve_list.at_put(cp_indices->at(j), true);
+ }
+ AOTConstantPoolResolver::preresolve_indy_cp_entries(THREAD, ik, &preresolve_list);
+ }
+ }
+}
+
+void FinalImageRecipes::record_recipes() {
+ _final_image_recipes = new FinalImageRecipes();
+ _final_image_recipes->record_recipes_impl();
+}
+
+void FinalImageRecipes::apply_recipes(TRAPS) {
+ assert(CDSConfig::is_dumping_final_static_archive(), "must be");
+ if (_final_image_recipes != nullptr) {
+ _final_image_recipes->apply_recipes_impl(THREAD);
+ if (HAS_PENDING_EXCEPTION) {
+ log_error(cds)("%s: %s", PENDING_EXCEPTION->klass()->external_name(),
+ java_lang_String::as_utf8_string(java_lang_Throwable::message(PENDING_EXCEPTION)));
+ log_error(cds)("Please check if your VM command-line is the same as in the training run");
+ MetaspaceShared::unrecoverable_writing_error("Unexpected exception, use -Xlog:cds,exceptions=trace for detail");
+ }
+ }
+
+ // Set it to null as we don't need to write this table into the final image.
+ _final_image_recipes = nullptr;
+}
+
+void FinalImageRecipes::apply_recipes_impl(TRAPS) {
+ load_all_classes(CHECK);
+ apply_recipes_for_invokedynamic(CHECK);
+}
+
+void FinalImageRecipes::serialize(SerializeClosure* soc) {
+ soc->do_ptr((void**)&_final_image_recipes);
+}
diff --git a/src/hotspot/share/cds/finalImageRecipes.hpp b/src/hotspot/share/cds/finalImageRecipes.hpp
new file mode 100644
index 00000000000..f07d9787af9
--- /dev/null
+++ b/src/hotspot/share/cds/finalImageRecipes.hpp
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_CDS_FINALIMAGERECIPES_HPP
+#define SHARE_CDS_FINALIMAGERECIPES_HPP
+
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/exceptions.hpp"
+
+class InstanceKlass;
+class Klass;
+
+template class GrowableArray;
+template class Array;
+
+// This class is used for transferring information from the AOTConfiguration file (aka the "preimage")
+// to the JVM that creates the AOTCache (aka the "final image").
+// - The recipes are recorded when CDSConfig::is_dumping_preimage_static_archive() is true.
+// - The recipes are applied when CDSConfig::is_dumping_final_static_archive() is true.
+// The following information are recorded:
+// - The list of all classes that are stored in the AOTConfiguration file.
+// - The list of all classes that require AOT resolution of invokedynamic call sites.
+class FinalImageRecipes {
+ // A list of all the archived classes from the preimage. We want to transfer all of these
+ // into the final image.
+ Array* _all_klasses;
+
+ // The classes who have resolved at least one indy CP entry during the training run.
+ // _indy_cp_indices[i] is a list of all resolved CP entries for _indy_klasses[i].
+ Array* _indy_klasses;
+ Array*>* _indy_cp_indices;
+
+ FinalImageRecipes() : _indy_klasses(nullptr), _indy_cp_indices(nullptr) {}
+
+ void* operator new(size_t size) throw();
+
+ // Called when dumping preimage
+ void record_recipes_impl();
+
+ // Called when dumping final image
+ void apply_recipes_impl(TRAPS);
+ void load_all_classes(TRAPS);
+ void apply_recipes_for_invokedynamic(TRAPS);
+
+public:
+ static void serialize(SerializeClosure* soc);
+
+ // Called when dumping preimage
+ static void record_recipes();
+
+ // Called when dumping final image
+ static void apply_recipes(TRAPS);
+};
+
+#endif // SHARE_CDS_FINALIMAGERECIPES_HPP
diff --git a/src/hotspot/share/cds/heapShared.cpp b/src/hotspot/share/cds/heapShared.cpp
index 5aa456ea438..ce98b2b93b7 100644
--- a/src/hotspot/share/cds/heapShared.cpp
+++ b/src/hotspot/share/cds/heapShared.cpp
@@ -138,11 +138,10 @@ static ArchivableStaticFieldInfo fmg_archive_subgraph_entry_fields[] = {
KlassSubGraphInfo* HeapShared::_dump_time_special_subgraph;
ArchivedKlassSubGraphInfoRecord* HeapShared::_run_time_special_subgraph;
GrowableArrayCHeap* HeapShared::_pending_roots = nullptr;
-GrowableArrayCHeap* HeapShared::_root_segments;
+GrowableArrayCHeap* HeapShared::_root_segments = nullptr;
int HeapShared::_root_segment_max_size_elems;
OopHandle HeapShared::_scratch_basic_type_mirrors[T_VOID+1];
-MetaspaceObjToOopHandleTable* HeapShared::_scratch_java_mirror_table = nullptr;
-MetaspaceObjToOopHandleTable* HeapShared::_scratch_references_table = nullptr;
+MetaspaceObjToOopHandleTable* HeapShared::_scratch_objects_table = nullptr;
static bool is_subgraph_root_class_of(ArchivableStaticFieldInfo fields[], InstanceKlass* ik) {
for (int i = 0; fields[i].valid(); i++) {
@@ -226,10 +225,6 @@ int HeapShared::append_root(oop obj) {
// No GC should happen since we aren't scanning _pending_roots.
assert(Thread::current() == (Thread*)VMThread::vm_thread(), "should be in vm thread");
- if (_pending_roots == nullptr) {
- _pending_roots = new GrowableArrayCHeap(500);
- }
-
return _pending_roots->append(obj);
}
@@ -289,7 +284,7 @@ void HeapShared::clear_root(int index) {
}
}
-bool HeapShared::archive_object(oop obj, KlassSubGraphInfo* subgraph_info) {
+bool HeapShared::archive_object(oop obj, oop referrer, KlassSubGraphInfo* subgraph_info) {
assert(CDSConfig::is_dumping_heap(), "dump-time only");
assert(!obj->is_stackChunk(), "do not archive stack chunks");
@@ -305,7 +300,7 @@ bool HeapShared::archive_object(oop obj, KlassSubGraphInfo* subgraph_info) {
} else {
count_allocation(obj->size());
ArchiveHeapWriter::add_source_obj(obj);
- CachedOopInfo info = make_cached_oop_info(obj);
+ CachedOopInfo info = make_cached_oop_info(obj, referrer);
archived_object_cache()->put_when_absent(obj, info);
archived_object_cache()->maybe_grow();
mark_native_pointers(obj);
@@ -337,6 +332,12 @@ bool HeapShared::archive_object(oop obj, KlassSubGraphInfo* subgraph_info) {
if (mirror_k != nullptr) {
AOTArtifactFinder::add_cached_class(mirror_k);
}
+ } else if (java_lang_invoke_ResolvedMethodName::is_instance(obj)) {
+ Method* m = java_lang_invoke_ResolvedMethodName::vmtarget(obj);
+ if (m != nullptr) {
+ InstanceKlass* method_holder = m->method_holder();
+ AOTArtifactFinder::add_cached_class(method_holder);
+ }
}
}
@@ -393,15 +394,20 @@ public:
void HeapShared::add_scratch_resolved_references(ConstantPool* src, objArrayOop dest) {
if (SystemDictionaryShared::is_builtin_loader(src->pool_holder()->class_loader_data())) {
- _scratch_references_table->set_oop(src, dest);
+ _scratch_objects_table->set_oop(src, dest);
}
}
objArrayOop HeapShared::scratch_resolved_references(ConstantPool* src) {
- return (objArrayOop)_scratch_references_table->get_oop(src);
+ return (objArrayOop)_scratch_objects_table->get_oop(src);
}
-void HeapShared::init_scratch_objects(TRAPS) {
+void HeapShared::init_dumping() {
+ _scratch_objects_table = new (mtClass)MetaspaceObjToOopHandleTable();
+ _pending_roots = new GrowableArrayCHeap(500);
+}
+
+void HeapShared::init_scratch_objects_for_basic_type_mirrors(TRAPS) {
for (int i = T_BOOLEAN; i < T_VOID+1; i++) {
BasicType bt = (BasicType)i;
if (!is_reference_type(bt)) {
@@ -409,8 +415,6 @@ void HeapShared::init_scratch_objects(TRAPS) {
_scratch_basic_type_mirrors[i] = OopHandle(Universe::vm_global(), m);
}
}
- _scratch_java_mirror_table = new (mtClass)MetaspaceObjToOopHandleTable();
- _scratch_references_table = new (mtClass)MetaspaceObjToOopHandleTable();
}
// Given java_mirror that represents a (primitive or reference) type T,
@@ -445,24 +449,24 @@ oop HeapShared::scratch_java_mirror(BasicType t) {
}
oop HeapShared::scratch_java_mirror(Klass* k) {
- return _scratch_java_mirror_table->get_oop(k);
+ return _scratch_objects_table->get_oop(k);
}
void HeapShared::set_scratch_java_mirror(Klass* k, oop mirror) {
- _scratch_java_mirror_table->set_oop(k, mirror);
+ _scratch_objects_table->set_oop(k, mirror);
}
void HeapShared::remove_scratch_objects(Klass* k) {
// Klass is being deallocated. Java mirror can still be alive, and it should not
// point to dead klass. We need to break the link from mirror to the Klass.
// See how InstanceKlass::deallocate_contents does it for normal mirrors.
- oop mirror = _scratch_java_mirror_table->get_oop(k);
+ oop mirror = _scratch_objects_table->get_oop(k);
if (mirror != nullptr) {
java_lang_Class::set_klass(mirror, nullptr);
}
- _scratch_java_mirror_table->remove_oop(k);
+ _scratch_objects_table->remove_oop(k);
if (k->is_instance_klass()) {
- _scratch_references_table->remove(InstanceKlass::cast(k)->constants());
+ _scratch_objects_table->remove(InstanceKlass::cast(k)->constants());
}
}
@@ -489,7 +493,7 @@ bool HeapShared::is_string_concat_klass(InstanceKlass* ik) {
}
bool HeapShared::is_archivable_hidden_klass(InstanceKlass* ik) {
- return CDSConfig::is_dumping_invokedynamic() &&
+ return CDSConfig::is_dumping_method_handles() &&
(is_lambda_form_klass(ik) || is_lambda_proxy_klass(ik) || is_string_concat_klass(ik));
}
@@ -564,7 +568,7 @@ void HeapShared::copy_and_rescan_aot_inited_mirror(InstanceKlass* ik) {
assert(success, "sanity");
}
- if (log_is_enabled(Info, cds, init)) {
+ if (log_is_enabled(Debug, cds, init)) {
ResourceMark rm;
log_debug(cds, init)("copied %3d field(s) in aot-initialized mirror %s%s%s", nfields, ik->external_name(),
ik->is_hidden() ? " (hidden)" : "",
@@ -779,9 +783,12 @@ void KlassSubGraphInfo::add_subgraph_object_klass(Klass* orig_k) {
if (orig_k->is_instance_klass()) {
#ifdef ASSERT
InstanceKlass* ik = InstanceKlass::cast(orig_k);
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
+ // -XX:AOTInitTestClass must be used carefully in regression tests to
+ // include only classes that are safe to aot-initialize.
assert(ik->class_loader() == nullptr ||
- HeapShared::is_lambda_proxy_klass(ik),
+ HeapShared::is_lambda_proxy_klass(ik) ||
+ AOTClassInitializer::has_test_class(),
"we can archive only instances of boot classes or lambda proxy classes");
} else {
assert(ik->class_loader() == nullptr, "must be boot class");
@@ -826,13 +833,20 @@ void KlassSubGraphInfo::add_subgraph_object_klass(Klass* orig_k) {
}
void KlassSubGraphInfo::check_allowed_klass(InstanceKlass* ik) {
+#ifndef PRODUCT
+ if (AOTClassInitializer::has_test_class()) {
+ // The tests can cache arbitrary types of objects.
+ return;
+ }
+#endif
+
if (ik->module()->name() == vmSymbols::java_base()) {
assert(ik->package() != nullptr, "classes in java.base cannot be in unnamed package");
return;
}
const char* lambda_msg = "";
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
lambda_msg = ", or a lambda proxy class";
if (HeapShared::is_lambda_proxy_klass(ik) &&
(ik->class_loader() == nullptr ||
@@ -1105,7 +1119,7 @@ void HeapShared::resolve_classes_for_subgraph_of(JavaThread* current, Klass* k)
}
void HeapShared::initialize_java_lang_invoke(TRAPS) {
- if (CDSConfig::is_loading_invokedynamic() || CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_using_aot_linked_classes() || CDSConfig::is_dumping_method_handles()) {
resolve_or_init("java/lang/invoke/Invokers$Holder", true, CHECK);
resolve_or_init("java/lang/invoke/MethodHandle", true, CHECK);
resolve_or_init("java/lang/invoke/MethodHandleNatives", true, CHECK);
@@ -1349,33 +1363,39 @@ void HeapShared::clear_archived_roots_of(Klass* k) {
}
}
-class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
+// Push all oops that are referenced by _referencing_obj onto the _stack.
+class HeapShared::ReferentPusher: public BasicOopIterateClosure {
+ PendingOopStack* _stack;
+ GrowableArray _found_oop_fields;
int _level;
bool _record_klasses_only;
KlassSubGraphInfo* _subgraph_info;
oop _referencing_obj;
-
- // The following are for maintaining a stack for determining
- // CachedOopInfo::_referrer
- static WalkOopAndArchiveClosure* _current;
- WalkOopAndArchiveClosure* _last;
public:
- WalkOopAndArchiveClosure(int level,
+ ReferentPusher(PendingOopStack* stack,
+ int level,
bool record_klasses_only,
KlassSubGraphInfo* subgraph_info,
oop orig) :
+ _stack(stack),
+ _found_oop_fields(),
_level(level),
_record_klasses_only(record_klasses_only),
_subgraph_info(subgraph_info),
_referencing_obj(orig) {
- _last = _current;
- _current = this;
}
- ~WalkOopAndArchiveClosure() {
- _current = _last;
+ void do_oop(narrowOop *p) { ReferentPusher::do_oop_work(p); }
+ void do_oop( oop *p) { ReferentPusher::do_oop_work(p); }
+
+ ~ReferentPusher() {
+ while (_found_oop_fields.length() > 0) {
+ // This produces the exact same traversal order as the previous version
+ // of ReferentPusher that recurses on the C stack -- a depth-first search,
+ // walking the oop fields in _referencing_obj by ascending field offsets.
+ oop obj = _found_oop_fields.pop();
+ _stack->push(PendingOop(obj, _referencing_obj, _level + 1));
+ }
}
- void do_oop(narrowOop *p) { WalkOopAndArchiveClosure::do_oop_work(p); }
- void do_oop( oop *p) { WalkOopAndArchiveClosure::do_oop_work(p); }
protected:
template void do_oop_work(T *p) {
@@ -1395,20 +1415,15 @@ class WalkOopAndArchiveClosure: public BasicOopIterateClosure {
}
}
- bool success = HeapShared::archive_reachable_objects_from(
- _level + 1, _subgraph_info, obj);
- assert(success, "VM should have exited with unarchivable objects for _level > 1");
+ _found_oop_fields.push(obj);
}
}
public:
- static WalkOopAndArchiveClosure* current() { return _current; }
oop referencing_obj() { return _referencing_obj; }
KlassSubGraphInfo* subgraph_info() { return _subgraph_info; }
};
-WalkOopAndArchiveClosure* WalkOopAndArchiveClosure::_current = nullptr;
-
// Checks if an oop has any non-null oop fields
class PointsToOopsChecker : public BasicOopIterateClosure {
bool _result;
@@ -1424,9 +1439,7 @@ public:
bool result() { return _result; }
};
-HeapShared::CachedOopInfo HeapShared::make_cached_oop_info(oop obj) {
- WalkOopAndArchiveClosure* walker = WalkOopAndArchiveClosure::current();
- oop referrer = (walker == nullptr) ? nullptr : walker->referencing_obj();
+HeapShared::CachedOopInfo HeapShared::make_cached_oop_info(oop obj, oop referrer) {
PointsToOopsChecker points_to_oops_checker;
obj->oop_iterate(&points_to_oops_checker);
return CachedOopInfo(referrer, points_to_oops_checker.result());
@@ -1449,12 +1462,35 @@ void HeapShared::init_box_classes(TRAPS) {
// (1) If orig_obj has not been archived yet, archive it.
// (2) If orig_obj has not been seen yet (since start_recording_subgraph() was called),
// trace all objects that are reachable from it, and make sure these objects are archived.
-// (3) Record the klasses of all orig_obj and all reachable objects.
+// (3) Record the klasses of all objects that are reachable from orig_obj (including those that
+// were already archived when this function is called)
bool HeapShared::archive_reachable_objects_from(int level,
KlassSubGraphInfo* subgraph_info,
oop orig_obj) {
assert(orig_obj != nullptr, "must be");
+ PendingOopStack stack;
+ stack.push(PendingOop(orig_obj, nullptr, level));
+ while (stack.length() > 0) {
+ PendingOop po = stack.pop();
+ _object_being_archived = po;
+ bool status = walk_one_object(&stack, po.level(), subgraph_info, po.obj(), po.referrer());
+ _object_being_archived = PendingOop();
+
+ if (!status) {
+ // Don't archive a subgraph root that's too big. For archives static fields, that's OK
+ // as the Java code will take care of initializing this field dynamically.
+ assert(level == 1, "VM should have exited with unarchivable objects for _level > 1");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool HeapShared::walk_one_object(PendingOopStack* stack, int level, KlassSubGraphInfo* subgraph_info,
+ oop orig_obj, oop referrer) {
+ assert(orig_obj != nullptr, "must be");
if (!JavaClasses::is_supported_for_archiving(orig_obj)) {
// This object has injected fields that cannot be supported easily, so we disallow them for now.
// If you get an error here, you probably made a change in the JDK library that has added
@@ -1523,7 +1559,7 @@ bool HeapShared::archive_reachable_objects_from(int level,
bool record_klasses_only = already_archived;
if (!already_archived) {
++_num_new_archived_objs;
- if (!archive_object(orig_obj, subgraph_info)) {
+ if (!archive_object(orig_obj, referrer, subgraph_info)) {
// Skip archiving the sub-graph referenced from the current entry field.
ResourceMark rm;
log_error(cds, heap)(
@@ -1546,8 +1582,13 @@ bool HeapShared::archive_reachable_objects_from(int level,
Klass *orig_k = orig_obj->klass();
subgraph_info->add_subgraph_object_klass(orig_k);
- WalkOopAndArchiveClosure walker(level, record_klasses_only, subgraph_info, orig_obj);
- orig_obj->oop_iterate(&walker);
+ {
+ // Find all the oops that are referenced by orig_obj, push them onto the stack
+ // so we can work on them next.
+ ResourceMark rm;
+ ReferentPusher pusher(stack, level, record_klasses_only, subgraph_info, orig_obj);
+ orig_obj->oop_iterate(&pusher);
+ }
if (CDSConfig::is_initing_classes_at_dump_time()) {
// The enum klasses are archived with aot-initialized mirror.
@@ -1572,8 +1613,7 @@ bool HeapShared::archive_reachable_objects_from(int level,
// - No java.lang.Class instance (java mirror) can be included inside
// an archived sub-graph. Mirror can only be the sub-graph entry object.
//
-// The Java heap object sub-graph archiving process (see
-// WalkOopAndArchiveClosure):
+// The Java heap object sub-graph archiving process (see ReferentPusher):
//
// 1) Java object sub-graph archiving starts from a given static field
// within a Class instance (java mirror). If the static field is a
@@ -1721,6 +1761,7 @@ void HeapShared::check_special_subgraph_classes() {
}
HeapShared::SeenObjectsTable* HeapShared::_seen_objects_table = nullptr;
+HeapShared::PendingOop HeapShared::_object_being_archived;
int HeapShared::_num_new_walked_objs;
int HeapShared::_num_new_archived_objs;
int HeapShared::_num_old_recorded_klasses;
@@ -2042,10 +2083,11 @@ bool HeapShared::is_dumped_interned_string(oop o) {
void HeapShared::debug_trace() {
ResourceMark rm;
- WalkOopAndArchiveClosure* walker = WalkOopAndArchiveClosure::current();
- if (walker != nullptr) {
+ oop referrer = _object_being_archived.referrer();
+ if (referrer != nullptr) {
LogStream ls(Log(cds, heap)::error());
- CDSHeapVerifier::trace_to_root(&ls, walker->referencing_obj());
+ ls.print_cr("Reference trace");
+ CDSHeapVerifier::trace_to_root(&ls, referrer);
}
}
diff --git a/src/hotspot/share/cds/heapShared.hpp b/src/hotspot/share/cds/heapShared.hpp
index 11250f1a35c..04c9ae91381 100644
--- a/src/hotspot/share/cds/heapShared.hpp
+++ b/src/hotspot/share/cds/heapShared.hpp
@@ -234,7 +234,7 @@ private:
static DumpTimeKlassSubGraphInfoTable* _dump_time_subgraph_info_table;
static RunTimeKlassSubGraphInfoTable _run_time_subgraph_info_table;
- static CachedOopInfo make_cached_oop_info(oop obj);
+ static CachedOopInfo make_cached_oop_info(oop obj, oop referrer);
static ArchivedKlassSubGraphInfoRecord* archive_subgraph_info(KlassSubGraphInfo* info);
static void archive_object_subgraphs(ArchivableStaticFieldInfo fields[],
bool is_full_module_graph);
@@ -284,8 +284,7 @@ private:
static GrowableArrayCHeap* _root_segments;
static int _root_segment_max_size_elems;
static OopHandle _scratch_basic_type_mirrors[T_VOID+1];
- static MetaspaceObjToOopHandleTable* _scratch_java_mirror_table;
- static MetaspaceObjToOopHandleTable* _scratch_references_table;
+ static MetaspaceObjToOopHandleTable* _scratch_objects_table;
static void init_seen_objects_table() {
assert(_seen_objects_table == nullptr, "must be");
@@ -315,7 +314,7 @@ private:
static bool has_been_seen_during_subgraph_recording(oop obj);
static void set_has_been_seen_during_subgraph_recording(oop obj);
- static bool archive_object(oop obj, KlassSubGraphInfo* subgraph_info);
+ static bool archive_object(oop obj, oop referrer, KlassSubGraphInfo* subgraph_info);
static void resolve_classes_for_subgraphs(JavaThread* current, ArchivableStaticFieldInfo fields[]);
static void resolve_classes_for_subgraph_of(JavaThread* current, Klass* k);
@@ -341,6 +340,30 @@ private:
static void archive_strings();
static void archive_subgraphs();
+ // PendingOop and PendingOopStack are used for recursively discovering all cacheable
+ // heap objects. The recursion is done using PendingOopStack so we won't overflow the
+ // C stack with deep reference chains.
+ class PendingOop {
+ oop _obj;
+ oop _referrer;
+ int _level;
+
+ public:
+ PendingOop() : _obj(nullptr), _referrer(nullptr), _level(-1) {}
+ PendingOop(oop obj, oop referrer, int level) : _obj(obj), _referrer(referrer), _level(level) {}
+
+ oop obj() const { return _obj; }
+ oop referrer() const { return _referrer; }
+ int level() const { return _level; }
+ };
+
+ class ReferentPusher;
+ using PendingOopStack = GrowableArrayCHeap;
+
+ static PendingOop _object_being_archived;
+ static bool walk_one_object(PendingOopStack* stack, int level, KlassSubGraphInfo* subgraph_info,
+ oop orig_obj, oop referrer);
+
public:
static void reset_archived_object_states(TRAPS);
static void create_archived_object_cache() {
@@ -405,7 +428,8 @@ private:
static void write_heap(ArchiveHeapInfo* heap_info) NOT_CDS_JAVA_HEAP_RETURN;
static objArrayOop scratch_resolved_references(ConstantPool* src);
static void add_scratch_resolved_references(ConstantPool* src, objArrayOop dest) NOT_CDS_JAVA_HEAP_RETURN;
- static void init_scratch_objects(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
+ static void init_dumping() NOT_CDS_JAVA_HEAP_RETURN;
+ static void init_scratch_objects_for_basic_type_mirrors(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
static void init_box_classes(TRAPS) NOT_CDS_JAVA_HEAP_RETURN;
static bool is_heap_region(int idx) {
CDS_JAVA_HEAP_ONLY(return (idx == MetaspaceShared::hp);)
diff --git a/src/hotspot/share/cds/lambdaFormInvokers.cpp b/src/hotspot/share/cds/lambdaFormInvokers.cpp
index ee3e9cff782..7832de8c6cc 100644
--- a/src/hotspot/share/cds/lambdaFormInvokers.cpp
+++ b/src/hotspot/share/cds/lambdaFormInvokers.cpp
@@ -22,8 +22,10 @@
*
*/
+#include "cds/aotClassFilter.hpp"
#include "cds/archiveBuilder.hpp"
-#include "cds/lambdaFormInvokers.hpp"
+#include "cds/cdsConfig.hpp"
+#include "cds/lambdaFormInvokers.inline.hpp"
#include "cds/metaspaceShared.hpp"
#include "cds/regeneratedClasses.hpp"
#include "classfile/classFileStream.hpp"
@@ -88,30 +90,39 @@ class PrintLambdaFormMessage {
}
};
+class LambdaFormInvokersClassFilterMark : public AOTClassFilter::FilterMark {
+public:
+ bool is_aot_tooling_class(InstanceKlass* ik) {
+ if (ik->name()->index_of_at(0, "$Species_", 9) > 0) {
+ // Classes like java.lang.invoke.BoundMethodHandle$Species_L should be included in AOT cache
+ return false;
+ }
+ if (LambdaFormInvokers::may_be_regenerated_class(ik->name())) {
+ // Regenerated holder classes should be included in AOT cache.
+ return false;
+ }
+ // Treat all other classes loaded during LambdaFormInvokers::regenerate_holder_classes() as
+ // "AOT tooling classes".
+ return true;
+ }
+};
+
void LambdaFormInvokers::regenerate_holder_classes(TRAPS) {
+ if (!CDSConfig::is_dumping_regenerated_lambdaform_invokers()) {
+ return;
+ }
+
PrintLambdaFormMessage plm;
if (_lambdaform_lines == nullptr || _lambdaform_lines->length() == 0) {
log_info(cds)("Nothing to regenerate for holder classes");
return;
}
- if (CDSConfig::is_dumping_static_archive() && CDSConfig::is_dumping_invokedynamic()) {
- // Work around JDK-8310831, as some methods in lambda form holder classes may not get generated.
- log_info(cds)("Archived MethodHandles may refer to lambda form holder classes. Cannot regenerate.");
- return;
- }
-
- if (CDSConfig::is_dumping_dynamic_archive() && CDSConfig::is_dumping_aot_linked_classes() &&
- CDSConfig::is_using_aot_linked_classes()) {
- // The base archive may have some pre-resolved CP entries that point to the lambda form holder
- // classes in the base archive. If we generate new versions of these classes, those CP entries
- // will be pointing to invalid classes.
- log_info(cds)("Base archive already has aot-linked lambda form holder classes. Cannot regenerate.");
- return;
- }
-
ResourceMark rm(THREAD);
+ // Filter out AOT tooling classes like java.lang.invoke.GenerateJLIClassesHelper, etc.
+ LambdaFormInvokersClassFilterMark filter_mark;
+
Symbol* cds_name = vmSymbols::jdk_internal_misc_CDS();
Klass* cds_klass = SystemDictionary::resolve_or_null(cds_name, THREAD);
guarantee(cds_klass != nullptr, "jdk/internal/misc/CDS must exist!");
@@ -245,7 +256,7 @@ void LambdaFormInvokers::dump_static_archive_invokers() {
}
assert(index == count, "Should match");
}
- log_debug(cds)("Total LF lines stored into static archive: %d", count);
+ log_debug(cds)("Total LF lines stored into %s: %d", CDSConfig::type_of_archive_being_written(), count);
}
}
@@ -257,10 +268,20 @@ void LambdaFormInvokers::read_static_archive_invokers() {
char* str = line->adr_at(0);
append(str);
}
- log_debug(cds)("Total LF lines read from static archive: %d", _static_archive_invokers->length());
+ log_debug(cds)("Total LF lines read from %s: %d", CDSConfig::type_of_archive_being_loaded(), _static_archive_invokers->length());
}
}
void LambdaFormInvokers::serialize(SerializeClosure* soc) {
soc->do_ptr(&_static_archive_invokers);
+ if (soc->reading() && CDSConfig::is_dumping_final_static_archive()) {
+ if (!CDSConfig::is_dumping_aot_linked_classes()) {
+ // See CDSConfig::is_dumping_regenerated_lambdaform_invokers() -- a dynamic archive can
+ // regenerate lambda form invokers only if the base archive does not contain aot-linked classes.
+ // If so, we copy the contents of _static_archive_invokers (from the preimage) into
+ //_lambdaform_lines, which will be written as _static_archive_invokers into final static archive.
+ LambdaFormInvokers::read_static_archive_invokers();
+ }
+ _static_archive_invokers = nullptr;
+ }
}
diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
index 91f677055a6..6b2e432d559 100644
--- a/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
+++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.cpp
@@ -22,10 +22,17 @@
*
*/
+#include "cds/aotClassFilter.hpp"
#include "cds/archiveBuilder.hpp"
+#include "cds/cdsConfig.hpp"
+#include "cds/cdsProtectionDomain.hpp"
#include "cds/lambdaProxyClassDictionary.hpp"
#include "classfile/systemDictionaryShared.hpp"
+#include "interpreter/bootstrapInfo.hpp"
+#include "jfr/jfrEvents.hpp"
+#include "memory/metaspaceClosure.hpp"
#include "memory/resourceArea.hpp"
+#include "oops/klass.inline.hpp"
DumpTimeLambdaProxyClassInfo::~DumpTimeLambdaProxyClassInfo() {
if (_proxy_klasses != nullptr) {
@@ -82,3 +89,442 @@ void RunTimeLambdaProxyClassInfo::init(LambdaProxyClassKey& key, DumpTimeLambdaP
ArchiveBuilder::current()->write_pointer_in_buffer(&_proxy_klass_head,
info._proxy_klasses->at(0));
}
+
+DumpTimeLambdaProxyClassDictionary* LambdaProxyClassDictionary::_dumptime_table = nullptr;
+LambdaProxyClassDictionary LambdaProxyClassDictionary::_runtime_static_table; // for static CDS archive
+LambdaProxyClassDictionary LambdaProxyClassDictionary::_runtime_dynamic_table; // for dynamic CDS archive
+
+void LambdaProxyClassDictionary::dumptime_init() {
+ _dumptime_table = new (mtClass) DumpTimeLambdaProxyClassDictionary;
+}
+
+bool LambdaProxyClassDictionary::is_supported_invokedynamic(BootstrapInfo* bsi) {
+ LogTarget(Debug, cds, lambda) log;
+ if (bsi->arg_values() == nullptr || !bsi->arg_values()->is_objArray()) {
+ if (log.is_enabled()) {
+ LogStream log_stream(log);
+ log.print("bsi check failed");
+ log.print(" bsi->arg_values().not_null() %d", bsi->arg_values().not_null());
+ if (bsi->arg_values().not_null()) {
+ log.print(" bsi->arg_values()->is_objArray() %d", bsi->arg_values()->is_objArray());
+ bsi->print_msg_on(&log_stream);
+ }
+ }
+ return false;
+ }
+
+ Handle bsm = bsi->bsm();
+ if (bsm.is_null() || !java_lang_invoke_DirectMethodHandle::is_instance(bsm())) {
+ if (log.is_enabled()) {
+ log.print("bsm check failed");
+ log.print(" bsm.is_null() %d", bsm.is_null());
+ log.print(" java_lang_invoke_DirectMethodHandle::is_instance(bsm()) %d",
+ java_lang_invoke_DirectMethodHandle::is_instance(bsm()));
+ }
+ return false;
+ }
+
+ oop mn = java_lang_invoke_DirectMethodHandle::member(bsm());
+ Method* method = java_lang_invoke_MemberName::vmtarget(mn);
+ if (method->klass_name()->equals("java/lang/invoke/LambdaMetafactory") &&
+ method->name()->equals("metafactory") &&
+ method->signature()->equals("(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;"
+ "Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;"
+ "Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;")) {
+ return true;
+ } else {
+ if (log.is_enabled()) {
+ ResourceMark rm;
+ log.print("method check failed");
+ log.print(" klass_name() %s", method->klass_name()->as_C_string());
+ log.print(" name() %s", method->name()->as_C_string());
+ log.print(" signature() %s", method->signature()->as_C_string());
+ }
+ }
+
+ return false;
+}
+
+void LambdaProxyClassDictionary::add_lambda_proxy_class(InstanceKlass* caller_ik,
+ InstanceKlass* lambda_ik,
+ Symbol* invoked_name,
+ Symbol* invoked_type,
+ Symbol* method_type,
+ Method* member_method,
+ Symbol* instantiated_method_type,
+ TRAPS) {
+ if (!CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ // The lambda proxy classes will be stored as part of aot-resolved constant pool entries.
+ // There's no need to remember them in a separate table.
+ return;
+ }
+
+ if (CDSConfig::is_dumping_preimage_static_archive()) {
+ // Information about lambda proxies are recorded in FinalImageRecipes.
+ return;
+ }
+
+ assert(caller_ik->class_loader() == lambda_ik->class_loader(), "mismatched class loader");
+ assert(caller_ik->class_loader_data() == lambda_ik->class_loader_data(), "mismatched class loader data");
+ assert(java_lang_Class::class_data(lambda_ik->java_mirror()) == nullptr, "must not have class data");
+
+ MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
+
+ lambda_ik->assign_class_loader_type();
+ lambda_ik->set_shared_classpath_index(caller_ik->shared_classpath_index());
+ InstanceKlass* nest_host = caller_ik->nest_host(CHECK);
+ assert(nest_host != nullptr, "unexpected nullptr nest_host");
+
+ DumpTimeClassInfo* info = SystemDictionaryShared::get_info_locked(lambda_ik);
+ if (info != nullptr && !lambda_ik->is_non_strong_hidden() &&
+ SystemDictionaryShared::is_builtin(lambda_ik) &&
+ SystemDictionaryShared::is_builtin(caller_ik)
+ // Don't include the lambda proxy if its nest host is not in the "linked" state.
+ && nest_host->is_linked()) {
+ // Set _is_registered_lambda_proxy in DumpTimeClassInfo so that the lambda_ik
+ // won't be excluded during dumping of shared archive.
+ info->_is_registered_lambda_proxy = true;
+ info->set_nest_host(nest_host);
+
+ LambdaProxyClassKey key(caller_ik,
+ invoked_name,
+ invoked_type,
+ method_type,
+ member_method,
+ instantiated_method_type);
+ add_to_dumptime_table(key, lambda_ik);
+ }
+}
+
+bool LambdaProxyClassDictionary::is_registered_lambda_proxy_class(InstanceKlass* ik) {
+ DumpTimeClassInfo* info = SystemDictionaryShared::get_info_locked(ik);
+ bool result = (info != nullptr) ? info->_is_registered_lambda_proxy : false;
+ if (result) {
+ assert(CDSConfig::is_dumping_lambdas_in_legacy_mode(), "only used in legacy lambda proxy support");
+ }
+ return result;
+}
+
+void LambdaProxyClassDictionary::reset_registered_lambda_proxy_class(InstanceKlass* ik) {
+ DumpTimeClassInfo* info = SystemDictionaryShared::get_info_locked(ik);
+ if (info != nullptr) {
+ info->_is_registered_lambda_proxy = false;
+ info->set_excluded();
+ }
+}
+
+InstanceKlass* LambdaProxyClassDictionary::get_shared_nest_host(InstanceKlass* lambda_ik) {
+ assert(!CDSConfig::is_dumping_static_archive() && CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
+ RunTimeClassInfo* record = RunTimeClassInfo::get_for(lambda_ik);
+ return record->nest_host();
+}
+
+InstanceKlass* LambdaProxyClassDictionary::load_shared_lambda_proxy_class(InstanceKlass* caller_ik,
+ Symbol* invoked_name,
+ Symbol* invoked_type,
+ Symbol* method_type,
+ Method* member_method,
+ Symbol* instantiated_method_type,
+ TRAPS)
+{
+ InstanceKlass* lambda_ik = find_lambda_proxy_class(caller_ik, invoked_name, invoked_type,
+ method_type, member_method, instantiated_method_type);
+ if (lambda_ik == nullptr) {
+ return nullptr;
+ }
+ return load_and_init_lambda_proxy_class(lambda_ik, caller_ik, THREAD);
+}
+
+InstanceKlass* LambdaProxyClassDictionary::find_lambda_proxy_class(InstanceKlass* caller_ik,
+ Symbol* invoked_name,
+ Symbol* invoked_type,
+ Symbol* method_type,
+ Method* member_method,
+ Symbol* instantiated_method_type)
+{
+ assert(caller_ik != nullptr, "sanity");
+ assert(invoked_name != nullptr, "sanity");
+ assert(invoked_type != nullptr, "sanity");
+ assert(method_type != nullptr, "sanity");
+ assert(instantiated_method_type != nullptr, "sanity");
+
+ if (!caller_ik->is_shared() ||
+ !invoked_name->is_shared() ||
+ !invoked_type->is_shared() ||
+ !method_type->is_shared() ||
+ (member_method != nullptr && !member_method->is_shared()) ||
+ !instantiated_method_type->is_shared()) {
+ // These can't be represented as u4 offset, but we wouldn't have archived a lambda proxy in this case anyway.
+ return nullptr;
+ }
+
+ MutexLocker ml(CDSLambda_lock, Mutex::_no_safepoint_check_flag);
+ RunTimeLambdaProxyClassKey key =
+ RunTimeLambdaProxyClassKey::init_for_runtime(caller_ik, invoked_name, invoked_type,
+ method_type, member_method, instantiated_method_type);
+
+ unsigned hash = key.hash();
+ // Try to retrieve the lambda proxy class from static archive.
+ const RunTimeLambdaProxyClassInfo* info = _runtime_static_table.lookup(&key, hash, 0);
+ InstanceKlass* proxy_klass = find_lambda_proxy_class(info);
+ if (proxy_klass == nullptr) {
+ if (info != nullptr && log_is_enabled(Debug, cds)) {
+ ResourceMark rm;
+ log_debug(cds)("Used all static archived lambda proxy classes for: %s %s%s",
+ caller_ik->external_name(), invoked_name->as_C_string(), invoked_type->as_C_string());
+ }
+ } else {
+ return proxy_klass;
+ }
+
+ // Retrieving from static archive is unsuccessful, try dynamic archive.
+ info = _runtime_dynamic_table.lookup(&key, hash, 0);
+ proxy_klass = find_lambda_proxy_class(info);
+ if (proxy_klass == nullptr) {
+ if (info != nullptr && log_is_enabled(Debug, cds)) {
+ ResourceMark rm;
+ log_debug(cds)("Used all dynamic archived lambda proxy classes for: %s %s%s",
+ caller_ik->external_name(), invoked_name->as_C_string(), invoked_type->as_C_string());
+ }
+ }
+ return proxy_klass;
+}
+
+InstanceKlass* LambdaProxyClassDictionary::find_lambda_proxy_class(const RunTimeLambdaProxyClassInfo* info) {
+ InstanceKlass* proxy_klass = nullptr;
+ if (info != nullptr) {
+ InstanceKlass* curr_klass = info->proxy_klass_head();
+ InstanceKlass* prev_klass = curr_klass;
+ if (curr_klass->lambda_proxy_is_available()) {
+ while (curr_klass->next_link() != nullptr) {
+ prev_klass = curr_klass;
+ curr_klass = InstanceKlass::cast(curr_klass->next_link());
+ }
+ assert(curr_klass->is_hidden(), "must be");
+ assert(curr_klass->lambda_proxy_is_available(), "must be");
+
+ prev_klass->set_next_link(nullptr);
+ proxy_klass = curr_klass;
+ proxy_klass->clear_lambda_proxy_is_available();
+ if (log_is_enabled(Debug, cds)) {
+ ResourceMark rm;
+ log_debug(cds)("Loaded lambda proxy: %s ", proxy_klass->external_name());
+ }
+ }
+ }
+ return proxy_klass;
+}
+
+InstanceKlass* LambdaProxyClassDictionary::load_and_init_lambda_proxy_class(InstanceKlass* lambda_ik,
+ InstanceKlass* caller_ik, TRAPS) {
+ Handle class_loader(THREAD, caller_ik->class_loader());
+ Handle protection_domain;
+ PackageEntry* pkg_entry = caller_ik->package();
+ if (caller_ik->class_loader() != nullptr) {
+ protection_domain = CDSProtectionDomain::init_security_info(class_loader, caller_ik, pkg_entry, CHECK_NULL);
+ }
+
+ InstanceKlass* shared_nest_host = get_shared_nest_host(lambda_ik);
+ assert(shared_nest_host != nullptr, "unexpected nullptr _nest_host");
+ assert(shared_nest_host->is_shared(), "nest host must be in CDS archive");
+
+ Klass* resolved_nest_host = SystemDictionary::resolve_or_fail(shared_nest_host->name(), class_loader, true, CHECK_NULL);
+ if (resolved_nest_host != shared_nest_host) {
+ // The dynamically resolved nest_host is not the same as the one we used during dump time,
+ // so we cannot use lambda_ik.
+ return nullptr;
+ }
+
+ {
+ InstanceKlass* loaded_lambda =
+ SystemDictionary::load_shared_class(lambda_ik, class_loader, protection_domain,
+ nullptr, pkg_entry, CHECK_NULL);
+ if (loaded_lambda != lambda_ik) {
+ // changed by JVMTI
+ return nullptr;
+ }
+ }
+
+ assert(shared_nest_host->is_same_class_package(lambda_ik),
+ "lambda proxy class and its nest host must be in the same package");
+ // The lambda proxy class and its nest host have the same class loader and class loader data,
+ // as verified in add_lambda_proxy_class()
+ assert(shared_nest_host->class_loader() == class_loader(), "mismatched class loader");
+ assert(shared_nest_host->class_loader_data() == ClassLoaderData::class_loader_data(class_loader()), "mismatched class loader data");
+ lambda_ik->set_nest_host(shared_nest_host);
+
+ // Ensures the nest host is the same as the lambda proxy's
+ // nest host recorded at dump time.
+ InstanceKlass* nest_host = caller_ik->nest_host(THREAD);
+ assert(nest_host == shared_nest_host, "mismatched nest host");
+
+ EventClassLoad class_load_start_event;
+
+ // Add to class hierarchy, and do possible deoptimizations.
+ lambda_ik->add_to_hierarchy(THREAD);
+ // But, do not add to dictionary.
+
+ lambda_ik->link_class(CHECK_NULL);
+ // notify jvmti
+ if (JvmtiExport::should_post_class_load()) {
+ JvmtiExport::post_class_load(THREAD, lambda_ik);
+ }
+ if (class_load_start_event.should_commit()) {
+ SystemDictionary::post_class_load_event(&class_load_start_event, lambda_ik, ClassLoaderData::class_loader_data(class_loader()));
+ }
+
+ lambda_ik->initialize(CHECK_NULL);
+
+ return lambda_ik;
+}
+
+void LambdaProxyClassDictionary::dumptime_classes_do(MetaspaceClosure* it) {
+ _dumptime_table->iterate_all([&] (LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+ if (key.caller_ik()->is_loader_alive()) {
+ info.metaspace_pointers_do(it);
+ key.metaspace_pointers_do(it);
+ }
+ });
+}
+
+void LambdaProxyClassDictionary::add_to_dumptime_table(LambdaProxyClassKey& key,
+ InstanceKlass* proxy_klass) {
+ assert_lock_strong(DumpTimeTable_lock);
+
+ if (AOTClassFilter::is_aot_tooling_class(proxy_klass)) {
+ return;
+ }
+
+ bool created;
+ DumpTimeLambdaProxyClassInfo* info = _dumptime_table->put_if_absent(key, &created);
+ info->add_proxy_klass(proxy_klass);
+ if (created) {
+ ++_dumptime_table->_count;
+ }
+}
+
+class CopyLambdaProxyClassInfoToArchive : StackObj {
+ CompactHashtableWriter* _writer;
+ ArchiveBuilder* _builder;
+public:
+ CopyLambdaProxyClassInfoToArchive(CompactHashtableWriter* writer)
+ : _writer(writer), _builder(ArchiveBuilder::current()) {}
+ bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+ // In static dump, info._proxy_klasses->at(0) is already relocated to point to the archived class
+ // (not the original class).
+ ResourceMark rm;
+ log_info(cds,dynamic)("Archiving hidden %s", info._proxy_klasses->at(0)->external_name());
+ size_t byte_size = sizeof(RunTimeLambdaProxyClassInfo);
+ RunTimeLambdaProxyClassInfo* runtime_info =
+ (RunTimeLambdaProxyClassInfo*)ArchiveBuilder::ro_region_alloc(byte_size);
+ runtime_info->init(key, info);
+ unsigned int hash = runtime_info->hash();
+ u4 delta = _builder->any_to_offset_u4((void*)runtime_info);
+ _writer->add(hash, delta);
+ return true;
+ }
+};
+
+void LambdaProxyClassDictionary::write_dictionary(bool is_static_archive) {
+ LambdaProxyClassDictionary* dictionary = is_static_archive ? &_runtime_static_table : &_runtime_dynamic_table;
+ CompactHashtableStats stats;
+ dictionary->reset();
+ CompactHashtableWriter writer(_dumptime_table->_count, &stats);
+ CopyLambdaProxyClassInfoToArchive copy(&writer);
+ _dumptime_table->iterate(©);
+ writer.dump(dictionary, "lambda proxy class dictionary");
+}
+
+class AdjustLambdaProxyClassInfo : StackObj {
+public:
+ AdjustLambdaProxyClassInfo() {}
+ bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+ int len = info._proxy_klasses->length();
+ InstanceKlass* last_buff_k = nullptr;
+
+ for (int i = len - 1; i >= 0; i--) {
+ InstanceKlass* orig_k = info._proxy_klasses->at(i);
+ InstanceKlass* buff_k = ArchiveBuilder::current()->get_buffered_addr(orig_k);
+ assert(ArchiveBuilder::current()->is_in_buffer_space(buff_k), "must be");
+ buff_k->set_lambda_proxy_is_available();
+ buff_k->set_next_link(last_buff_k);
+ if (last_buff_k != nullptr) {
+ ArchivePtrMarker::mark_pointer(buff_k->next_link_addr());
+ }
+ last_buff_k = buff_k;
+ }
+
+ return true;
+ }
+};
+
+void LambdaProxyClassDictionary::adjust_dumptime_table() {
+ AdjustLambdaProxyClassInfo adjuster;
+ _dumptime_table->iterate(&adjuster);
+}
+
+class LambdaProxyClassDictionary::CleanupDumpTimeLambdaProxyClassTable: StackObj {
+ public:
+ bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
+ assert_lock_strong(DumpTimeTable_lock);
+ InstanceKlass* caller_ik = key.caller_ik();
+ InstanceKlass* nest_host = caller_ik->nest_host_not_null();
+
+ // If the caller class and/or nest_host are excluded, the associated lambda proxy
+ // must also be excluded.
+ bool always_exclude = SystemDictionaryShared::check_for_exclusion(caller_ik, nullptr) ||
+ SystemDictionaryShared::check_for_exclusion(nest_host, nullptr);
+
+ for (int i = info._proxy_klasses->length() - 1; i >= 0; i--) {
+ InstanceKlass* ik = info._proxy_klasses->at(i);
+ if (always_exclude || SystemDictionaryShared::check_for_exclusion(ik, nullptr)) {
+ LambdaProxyClassDictionary::reset_registered_lambda_proxy_class(ik);
+ info._proxy_klasses->remove_at(i);
+ }
+ }
+ return info._proxy_klasses->length() == 0 ? true /* delete the node*/ : false;
+ }
+};
+
+void LambdaProxyClassDictionary::cleanup_dumptime_table() {
+ assert_lock_strong(DumpTimeTable_lock);
+ CleanupDumpTimeLambdaProxyClassTable cleanup_proxy_classes;
+ _dumptime_table->unlink(&cleanup_proxy_classes);
+}
+
+class SharedLambdaDictionaryPrinter : StackObj {
+ outputStream* _st;
+ int _index;
+public:
+ SharedLambdaDictionaryPrinter(outputStream* st, int idx) : _st(st), _index(idx) {}
+
+ void do_value(const RunTimeLambdaProxyClassInfo* record) {
+ if (record->proxy_klass_head()->lambda_proxy_is_available()) {
+ ResourceMark rm;
+ Klass* k = record->proxy_klass_head();
+ while (k != nullptr) {
+ _st->print_cr("%4d: %s %s", _index++, k->external_name(),
+ SystemDictionaryShared::loader_type_for_shared_class(k));
+ k = k->next_link();
+ }
+ }
+ }
+};
+
+void LambdaProxyClassDictionary::print_on(const char* prefix,
+ outputStream* st,
+ int start_index,
+ bool is_static_archive) {
+ LambdaProxyClassDictionary* dictionary = is_static_archive ? &_runtime_static_table : &_runtime_dynamic_table;
+ if (!dictionary->empty()) {
+ st->print_cr("%sShared Lambda Dictionary", prefix);
+ SharedLambdaDictionaryPrinter ldp(st, start_index);
+ dictionary->iterate(&ldp);
+ }
+}
+
+void LambdaProxyClassDictionary::print_statistics(outputStream* st,
+ bool is_static_archive) {
+ LambdaProxyClassDictionary* dictionary = is_static_archive ? &_runtime_static_table : &_runtime_dynamic_table;
+ dictionary->print_table_statistics(st, "Lambda Shared Dictionary");
+}
diff --git a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
index bd163fc551d..814dda80827 100644
--- a/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
+++ b/src/hotspot/share/cds/lambdaProxyClassDictionary.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -24,13 +24,31 @@
#ifndef SHARE_CDS_LAMBDAPROXYCLASSINFO_HPP
#define SHARE_CDS_LAMBDAPROXYCLASSINFO_HPP
+
+#include "cds/archiveBuilder.hpp"
#include "cds/metaspaceShared.hpp"
#include "classfile/javaClasses.hpp"
+#include "memory/metaspaceClosure.hpp"
#include "utilities/growableArray.hpp"
#include "utilities/resourceHash.hpp"
+// This file contains *legacy* optimization for lambdas before JEP 483. May be removed in the future.
+//
+// The functionalties in this file are used only when CDSConfig::is_dumping_lambdas_in_legacy_mode()
+// returns true during the creation of a CDS archive.
+//
+// With the legacy optimization, generated lambda proxy classes (with names such as
+// java.util.ResourceBundle$Control$$Lambda/0x80000001d) are stored inside the CDS archive, accessible
+// by LambdaProxyClassDictionary::find_proxy_class(). This saves part of the time for resolving a
+// lambda call site (proxy class generation). However, a significant portion of the cost of
+// the lambda call site resolution still remains in the production run.
+//
+// In contrast, with JEP 483, the entire lambda call site (starting from the constant pool entry), is
+// resolved in the AOT cache assembly phase. No extra resolution is needed in the production run.
+
class InstanceKlass;
class Method;
+class MetaspaceClosure;
class Symbol;
class outputStream;
@@ -243,9 +261,75 @@ public:
int _count;
};
+// *Legacy* optimization for lambdas before JEP 483. May be removed in the future.
class LambdaProxyClassDictionary : public OffsetCompactHashtable<
RunTimeLambdaProxyClassKey*,
const RunTimeLambdaProxyClassInfo*,
- RunTimeLambdaProxyClassInfo::EQUALS> {};
+ RunTimeLambdaProxyClassInfo::EQUALS>
+{
+private:
+ class CleanupDumpTimeLambdaProxyClassTable;
+ static DumpTimeLambdaProxyClassDictionary* _dumptime_table;
+ static LambdaProxyClassDictionary _runtime_static_table; // for static CDS archive
+ static LambdaProxyClassDictionary _runtime_dynamic_table; // for dynamic CDS archive
+
+ static void add_to_dumptime_table(LambdaProxyClassKey& key,
+ InstanceKlass* proxy_klass);
+ static InstanceKlass* find_lambda_proxy_class(const RunTimeLambdaProxyClassInfo* info);
+ static InstanceKlass* find_lambda_proxy_class(InstanceKlass* caller_ik,
+ Symbol* invoked_name,
+ Symbol* invoked_type,
+ Symbol* method_type,
+ Method* member_method,
+ Symbol* instantiated_method_type);
+ static InstanceKlass* load_and_init_lambda_proxy_class(InstanceKlass* lambda_ik,
+ InstanceKlass* caller_ik, TRAPS);
+ static void reset_registered_lambda_proxy_class(InstanceKlass* ik);
+ static InstanceKlass* get_shared_nest_host(InstanceKlass* lambda_ik);
+
+public:
+ static void dumptime_init();
+ static void dumptime_classes_do(MetaspaceClosure* it);
+ static void add_lambda_proxy_class(InstanceKlass* caller_ik,
+ InstanceKlass* lambda_ik,
+ Symbol* invoked_name,
+ Symbol* invoked_type,
+ Symbol* method_type,
+ Method* member_method,
+ Symbol* instantiated_method_type,
+ TRAPS);
+ static bool is_supported_invokedynamic(BootstrapInfo* bsi);
+ static bool is_registered_lambda_proxy_class(InstanceKlass* ik);
+ static InstanceKlass* load_shared_lambda_proxy_class(InstanceKlass* caller_ik,
+ Symbol* invoked_name,
+ Symbol* invoked_type,
+ Symbol* method_type,
+ Method* member_method,
+ Symbol* instantiated_method_type,
+ TRAPS);
+ static void write_dictionary(bool is_static_archive);
+ static void adjust_dumptime_table();
+ static void cleanup_dumptime_table();
+
+ static void reset_dictionary(bool is_static_archive) {
+ if (is_static_archive) {
+ _runtime_static_table.reset();
+ } else {
+ _runtime_dynamic_table.reset();
+ }
+ }
+
+ static void serialize(SerializeClosure* soc, bool is_static_archive) {
+ if (is_static_archive) {
+ _runtime_static_table.serialize_header(soc);
+ } else {
+ _runtime_dynamic_table.serialize_header(soc);
+ }
+ }
+
+ static void print_on(const char* prefix, outputStream* st,
+ int start_index, bool is_static_archive);
+ static void print_statistics(outputStream* st, bool is_static_archive);
+};
#endif // SHARE_CDS_LAMBDAPROXYCLASSINFO_HPP
diff --git a/src/hotspot/share/cds/metaspaceShared.cpp b/src/hotspot/share/cds/metaspaceShared.cpp
index b95d524cb1d..85916ced3cf 100644
--- a/src/hotspot/share/cds/metaspaceShared.cpp
+++ b/src/hotspot/share/cds/metaspaceShared.cpp
@@ -23,6 +23,7 @@
*/
#include "cds/aotArtifactFinder.hpp"
+#include "cds/aotClassInitializer.hpp"
#include "cds/aotClassLinker.hpp"
#include "cds/aotClassLocation.hpp"
#include "cds/aotConstantPoolResolver.hpp"
@@ -39,8 +40,10 @@
#include "cds/dumpAllocStats.hpp"
#include "cds/dynamicArchive.hpp"
#include "cds/filemap.hpp"
+#include "cds/finalImageRecipes.hpp"
#include "cds/heapShared.hpp"
#include "cds/lambdaFormInvokers.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/metaspaceShared.hpp"
#include "classfile/classLoaderDataGraph.hpp"
#include "classfile/classLoaderDataShared.hpp"
@@ -94,6 +97,8 @@
#include "utilities/ostream.hpp"
#include "utilities/resourceHash.hpp"
+#include
+
ReservedSpace MetaspaceShared::_symbol_rs;
VirtualSpace MetaspaceShared::_symbol_vs;
bool MetaspaceShared::_archive_loading_failed = false;
@@ -146,11 +151,15 @@ size_t MetaspaceShared::core_region_alignment() {
return os::cds_core_region_alignment();
}
+size_t MetaspaceShared::protection_zone_size() {
+ return os::cds_core_region_alignment();
+}
+
static bool shared_base_valid(char* shared_base) {
// We check user input for SharedBaseAddress at dump time.
// At CDS runtime, "shared_base" will be the (attempted) mapping start. It will also
- // be the encoding base, since the the headers of archived base objects (and with Lilliput,
+ // be the encoding base, since the headers of archived base objects (and with Lilliput,
// the prototype mark words) carry pre-computed narrow Klass IDs that refer to the mapping
// start as base.
//
@@ -215,30 +224,9 @@ void MetaspaceShared::dump_loaded_classes(const char* file_name, TRAPS) {
}
}
-// If p is not aligned, move it up to the next address that's aligned with alignment.
-// If this is not possible (because p is too high), return nullptr. Example:
-// p = 0xffffffffffff0000, alignment= 0x10000 => return nullptr.
-static char* align_up_or_null(char* p, size_t alignment) {
- assert(p != nullptr, "sanity");
- if (is_aligned(p, alignment)) {
- return p;
- }
-
- char* down = align_down(p, alignment);
- if (max_uintx - uintx(down) < uintx(alignment)) {
- // Run out of address space to align up.
- return nullptr;
- }
-
- char* aligned = align_up(p, alignment);
- assert(aligned >= p, "sanity");
- assert(aligned != nullptr, "sanity");
- return aligned;
-}
-
static bool shared_base_too_high(char* specified_base, char* aligned_base, size_t cds_max) {
- // Caller should have checked if align_up_or_null( returns nullptr (comparing specified_base
- // with nullptr is UB).
+ // Caller should have checked that aligned_base was successfully aligned and is not nullptr.
+ // Comparing specified_base with nullptr is UB.
assert(aligned_base != nullptr, "sanity");
assert(aligned_base >= specified_base, "sanity");
@@ -262,7 +250,9 @@ static char* compute_shared_base(size_t cds_max) {
return specified_base;
}
- char* aligned_base = align_up_or_null(specified_base, alignment);
+ char* aligned_base = can_align_up(specified_base, alignment)
+ ? align_up(specified_base, alignment)
+ : nullptr;
if (aligned_base != specified_base) {
log_info(cds)("SharedBaseAddress (" INTPTR_FORMAT ") aligned up to " INTPTR_FORMAT,
@@ -489,6 +479,7 @@ void MetaspaceShared::serialize(SerializeClosure* soc) {
HeapShared::serialize_tables(soc);
SystemDictionaryShared::serialize_dictionary_headers(soc);
AOTLinkedClassBulkLoader::serialize(soc, true);
+ FinalImageRecipes::serialize(soc);
InstanceMirrorKlass::serialize_offsets(soc);
// Dump/restore well known classes (pointers)
@@ -609,6 +600,9 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables(AOTClassLocationConfig*&
SystemDictionaryShared::write_to_archive();
cl_config = AOTClassLocationConfig::dumptime()->write_to_archive();
AOTClassLinker::write_to_archive();
+ if (CDSConfig::is_dumping_preimage_static_archive()) {
+ FinalImageRecipes::record_recipes();
+ }
MetaspaceShared::write_method_handle_intrinsics();
// Write lambform lines into archive
@@ -624,12 +618,14 @@ char* VM_PopulateDumpSharedSpace::dump_read_only_tables(AOTClassLocationConfig*&
}
void VM_PopulateDumpSharedSpace::doit() {
- guarantee(!CDSConfig::is_using_archive(), "We should not be using an archive when we dump");
+ if (!CDSConfig::is_dumping_final_static_archive()) {
+ guarantee(!CDSConfig::is_using_archive(), "We should not be using an archive when we dump");
+ }
DEBUG_ONLY(SystemDictionaryShared::NoClassLoadingMark nclm);
_pending_method_handle_intrinsics = new (mtClassShared) GrowableArray(256, mtClassShared);
- if (CDSConfig::is_dumping_aot_linked_classes()) {
+ if (CDSConfig::is_dumping_method_handles()) {
// When dumping AOT-linked classes, some classes may have direct references to a method handle
// intrinsic. The easiest thing is to save all of them into the AOT cache.
SystemDictionary::get_all_method_handle_intrinsics(_pending_method_handle_intrinsics);
@@ -672,14 +668,23 @@ void VM_PopulateDumpSharedSpace::doit() {
AOTClassLocationConfig* cl_config;
char* serialized_data = dump_read_only_tables(cl_config);
- SystemDictionaryShared::adjust_lambda_proxy_class_dictionary();
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ log_info(cds)("Adjust lambda proxy class dictionary");
+ LambdaProxyClassDictionary::adjust_dumptime_table();
+ }
// The vtable clones contain addresses of the current process.
// We don't want to write these addresses into the archive.
CppVtables::zero_archived_vtables();
// Write the archive file
- const char* static_archive = CDSConfig::static_archive_path();
+ const char* static_archive;
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ static_archive = AOTCache;
+ FileMapInfo::free_current_info();
+ } else {
+ static_archive = CDSConfig::static_archive_path();
+ }
assert(static_archive != nullptr, "SharedArchiveFile not set?");
_map_info = new FileMapInfo(static_archive, true);
_map_info->populate_header(MetaspaceShared::core_region_alignment());
@@ -689,25 +694,37 @@ void VM_PopulateDumpSharedSpace::doit() {
_map_info->header()->set_class_location_config(cl_config);
}
-class CollectCLDClosure : public CLDClosure {
- GrowableArray _loaded_cld;
- GrowableArray _loaded_cld_handles; // keep the CLDs alive
- Thread* _current_thread;
+class CollectClassesForLinking : public KlassClosure {
+ GrowableArray _mirrors;
+
public:
- CollectCLDClosure(Thread* thread) : _current_thread(thread) {}
- ~CollectCLDClosure() {
- for (int i = 0; i < _loaded_cld_handles.length(); i++) {
- _loaded_cld_handles.at(i).release(Universe::vm_global());
+ CollectClassesForLinking() : _mirrors() {
+ // ClassLoaderDataGraph::loaded_classes_do_keepalive() requires ClassLoaderDataGraph_lock.
+ // We cannot link the classes while holding this lock (or else we may run into deadlock).
+ // Therefore, we need to first collect all the classes, keeping them alive by
+ // holding onto their java_mirrors in global OopHandles. We then link the classes after
+ // releasing the lock.
+ MutexLocker lock(ClassLoaderDataGraph_lock);
+ ClassLoaderDataGraph::loaded_classes_do_keepalive(this);
+ }
+
+ ~CollectClassesForLinking() {
+ for (int i = 0; i < _mirrors.length(); i++) {
+ _mirrors.at(i).release(Universe::vm_global());
}
}
+
void do_cld(ClassLoaderData* cld) {
assert(cld->is_alive(), "must be");
- _loaded_cld.append(cld);
- _loaded_cld_handles.append(OopHandle(Universe::vm_global(), cld->holder()));
}
- int nof_cld() const { return _loaded_cld.length(); }
- ClassLoaderData* cld_at(int index) { return _loaded_cld.at(index); }
+ void do_klass(Klass* k) {
+ if (k->is_instance_klass()) {
+ _mirrors.append(OopHandle(Universe::vm_global(), k->java_mirror()));
+ }
+ }
+
+ const GrowableArray* mirrors() const { return &_mirrors; }
};
// Check if we can eagerly link this class at dump time, so we can avoid the
@@ -731,44 +748,20 @@ bool MetaspaceShared::may_be_eagerly_linked(InstanceKlass* ik) {
return true;
}
-bool MetaspaceShared::link_class_for_cds(InstanceKlass* ik, TRAPS) {
- // Link the class to cause the bytecodes to be rewritten and the
- // cpcache to be created. Class verification is done according
- // to -Xverify setting.
- bool res = MetaspaceShared::try_link_class(THREAD, ik);
- AOTConstantPoolResolver::dumptime_resolve_constants(ik, CHECK_(false));
- return res;
-}
-
-void MetaspaceShared::link_shared_classes(bool jcmd_request, TRAPS) {
+void MetaspaceShared::link_shared_classes(TRAPS) {
AOTClassLinker::initialize();
-
- if (!jcmd_request) {
- LambdaFormInvokers::regenerate_holder_classes(CHECK);
- }
-
- // Collect all loaded ClassLoaderData.
- CollectCLDClosure collect_cld(THREAD);
- {
- // ClassLoaderDataGraph::loaded_cld_do requires ClassLoaderDataGraph_lock.
- // We cannot link the classes while holding this lock (or else we may run into deadlock).
- // Therefore, we need to first collect all the CLDs, and then link their classes after
- // releasing the lock.
- MutexLocker lock(ClassLoaderDataGraph_lock);
- ClassLoaderDataGraph::loaded_cld_do(&collect_cld);
- }
+ AOTClassInitializer::init_test_class(CHECK);
while (true) {
+ ResourceMark rm(THREAD);
+ CollectClassesForLinking collect_classes;
bool has_linked = false;
- for (int i = 0; i < collect_cld.nof_cld(); i++) {
- ClassLoaderData* cld = collect_cld.cld_at(i);
- for (Klass* klass = cld->klasses(); klass != nullptr; klass = klass->next_link()) {
- if (klass->is_instance_klass()) {
- InstanceKlass* ik = InstanceKlass::cast(klass);
- if (may_be_eagerly_linked(ik)) {
- has_linked |= link_class_for_cds(ik, CHECK);
- }
- }
+ const GrowableArray* mirrors = collect_classes.mirrors();
+ for (int i = 0; i < mirrors->length(); i++) {
+ OopHandle mirror = mirrors->at(i);
+ InstanceKlass* ik = InstanceKlass::cast(java_lang_Class::as_Klass(mirror.resolve()));
+ if (may_be_eagerly_linked(ik)) {
+ has_linked |= try_link_class(THREAD, ik);
}
}
@@ -778,6 +771,22 @@ void MetaspaceShared::link_shared_classes(bool jcmd_request, TRAPS) {
// Class linking includes verification which may load more classes.
// Keep scanning until we have linked no more classes.
}
+
+ // Resolve constant pool entries -- we don't load any new classes during this stage
+ {
+ ResourceMark rm(THREAD);
+ CollectClassesForLinking collect_classes;
+ const GrowableArray* mirrors = collect_classes.mirrors();
+ for (int i = 0; i < mirrors->length(); i++) {
+ OopHandle mirror = mirrors->at(i);
+ InstanceKlass* ik = InstanceKlass::cast(java_lang_Class::as_Klass(mirror.resolve()));
+ AOTConstantPoolResolver::dumptime_resolve_constants(ik, CHECK);
+ }
+ }
+
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ FinalImageRecipes::apply_recipes(CHECK);
+ }
}
void MetaspaceShared::prepare_for_dumping() {
@@ -804,13 +813,26 @@ void MetaspaceShared::preload_and_dump(TRAPS) {
}
}
- if (!CDSConfig::old_cds_flags_used()) {
- // The JLI launcher only recognizes the "old" -Xshare:dump flag.
- // When the new -XX:AOTMode=create flag is used, we can't return
- // to the JLI launcher, as the launcher will fail when trying to
- // run the main class, which is not what we want.
- tty->print_cr("AOTCache creation is complete: %s", AOTCache);
- vm_exit(0);
+ if (CDSConfig::new_aot_flags_used()) {
+ if (CDSConfig::is_dumping_preimage_static_archive()) {
+ // We are in the JVM that runs the training run. Continue execution,
+ // so that it can finish all clean-up and return the correct exit
+ // code to the OS.
+ tty->print_cr("AOTConfiguration recorded: %s", AOTConfiguration);
+ } else {
+ // The JLI launcher only recognizes the "old" -Xshare:dump flag.
+ // When the new -XX:AOTMode=create flag is used, we can't return
+ // to the JLI launcher, as the launcher will fail when trying to
+ // run the main class, which is not what we want.
+ struct stat st;
+ if (os::stat(AOTCache, &st) != 0) {
+ tty->print_cr("AOTCache creation failed: %s", AOTCache);
+ vm_exit(0);
+ } else {
+ tty->print_cr("AOTCache creation is complete: %s " INT64_FORMAT " bytes", AOTCache, (int64_t)(st.st_size));
+ vm_exit(0);
+ }
+ }
}
}
@@ -839,37 +861,16 @@ void MetaspaceShared::adjust_heap_sizes_for_dumping() {
#endif // INCLUDE_CDS_JAVA_HEAP && _LP64
void MetaspaceShared::get_default_classlist(char* default_classlist, const size_t buf_size) {
- // Construct the path to the class list (in jre/lib)
- // Walk up two directories from the location of the VM and
- // optionally tack on "lib" (depending on platform)
- os::jvm_path(default_classlist, (jint)(buf_size));
- for (int i = 0; i < 3; i++) {
- char *end = strrchr(default_classlist, *os::file_separator());
- if (end != nullptr) *end = '\0';
- }
- size_t classlist_path_len = strlen(default_classlist);
- if (classlist_path_len >= 3) {
- if (strcmp(default_classlist + classlist_path_len - 3, "lib") != 0) {
- if (classlist_path_len < buf_size - 4) {
- jio_snprintf(default_classlist + classlist_path_len,
- buf_size - classlist_path_len,
- "%slib", os::file_separator());
- classlist_path_len += 4;
- }
- }
- }
- if (classlist_path_len < buf_size - 10) {
- jio_snprintf(default_classlist + classlist_path_len,
- buf_size - classlist_path_len,
- "%sclasslist", os::file_separator());
- }
+ const char* filesep = os::file_separator();
+ jio_snprintf(default_classlist, buf_size, "%s%slib%sclasslist",
+ Arguments::get_java_home(), filesep, filesep);
}
void MetaspaceShared::preload_classes(TRAPS) {
char default_classlist[JVM_MAXPATHLEN];
const char* classlist_path;
- get_default_classlist(default_classlist, sizeof(default_classlist));
+ get_default_classlist(default_classlist, JVM_MAXPATHLEN);
if (SharedClassListFile == nullptr) {
classlist_path = default_classlist;
} else {
@@ -910,12 +911,46 @@ void MetaspaceShared::exercise_runtime_cds_code(TRAPS) {
}
void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS) {
- preload_classes(CHECK);
+ if (CDSConfig::is_dumping_classic_static_archive()) {
+ // We are running with -Xshare:dump
+ preload_classes(CHECK);
- if (SharedArchiveConfigFile) {
- log_info(cds)("Reading extra data from %s ...", SharedArchiveConfigFile);
- read_extra_data(THREAD, SharedArchiveConfigFile);
- log_info(cds)("Reading extra data: done.");
+ if (SharedArchiveConfigFile) {
+ log_info(cds)("Reading extra data from %s ...", SharedArchiveConfigFile);
+ read_extra_data(THREAD, SharedArchiveConfigFile);
+ log_info(cds)("Reading extra data: done.");
+ }
+ }
+
+ if (CDSConfig::is_dumping_preimage_static_archive()) {
+ log_info(cds)("Reading lambda form invokers from JDK default classlist ...");
+ char default_classlist[JVM_MAXPATHLEN];
+ get_default_classlist(default_classlist, JVM_MAXPATHLEN);
+ struct stat statbuf;
+ if (os::stat(default_classlist, &statbuf) == 0) {
+ ClassListParser::parse_classlist(default_classlist,
+ ClassListParser::_parse_lambda_forms_invokers_only, CHECK);
+ }
+ }
+
+#if INCLUDE_CDS_JAVA_HEAP
+ if (CDSConfig::is_dumping_heap()) {
+ assert(CDSConfig::allow_only_single_java_thread(), "Required");
+ if (!HeapShared::is_archived_boot_layer_available(THREAD)) {
+ log_info(cds)("archivedBootLayer not available, disabling full module graph");
+ CDSConfig::stop_dumping_full_module_graph();
+ }
+ // Do this before link_shared_classes(), as the following line may load new classes.
+ HeapShared::init_for_dumping(CHECK);
+ }
+#endif
+
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ if (ExtraSharedClassListFile) {
+ log_info(cds)("Loading extra classes from %s ...", ExtraSharedClassListFile);
+ ClassListParser::parse_classlist(ExtraSharedClassListFile,
+ ClassListParser::_parse_all, CHECK);
+ }
}
// Rewrite and link classes
@@ -925,23 +960,22 @@ void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS
// were not explicitly specified in the classlist. E.g., if an interface implemented by class K
// fails verification, all other interfaces that were not specified in the classlist but
// are implemented by K are not verified.
- link_shared_classes(false/*not from jcmd*/, CHECK);
+ link_shared_classes(CHECK);
log_info(cds)("Rewriting and linking classes: done");
+ if (CDSConfig::is_dumping_regenerated_lambdaform_invokers()) {
+ LambdaFormInvokers::regenerate_holder_classes(CHECK);
+ }
+
#if INCLUDE_CDS_JAVA_HEAP
if (CDSConfig::is_dumping_heap()) {
- if (!HeapShared::is_archived_boot_layer_available(THREAD)) {
- log_info(cds)("archivedBootLayer not available, disabling full module graph");
- CDSConfig::stop_dumping_full_module_graph();
- }
- HeapShared::init_for_dumping(CHECK);
ArchiveHeapWriter::init();
if (CDSConfig::is_dumping_full_module_graph()) {
ClassLoaderDataShared::ensure_module_entry_tables_exist();
HeapShared::reset_archived_object_states(CHECK);
}
- if (CDSConfig::is_dumping_invokedynamic()) {
+ if (CDSConfig::is_dumping_method_handles()) {
// This assert means that the MethodType and MethodTypeForm tables won't be
// updated concurrently when we are saving their contents into a side table.
assert(CDSConfig::allow_only_single_java_thread(), "Required");
@@ -951,12 +985,15 @@ void MetaspaceShared::preload_and_dump_impl(StaticArchiveBuilder& builder, TRAPS
vmSymbols::createArchivedObjects(),
vmSymbols::void_method_signature(),
CHECK);
+ }
+ if (CDSConfig::is_initing_classes_at_dump_time()) {
// java.lang.Class::reflectionFactory cannot be archived yet. We set this field
// to null, and it will be initialized again at runtime.
log_debug(cds)("Resetting Class::reflectionFactory");
TempNewSymbol method_name = SymbolTable::new_symbol("resetArchivedStates");
Symbol* method_sig = vmSymbols::void_method_signature();
+ JavaValue result(T_VOID);
JavaCalls::call_static(&result, vmClasses::Class_klass(),
method_name, method_sig, CHECK);
@@ -993,8 +1030,8 @@ bool MetaspaceShared::write_static_archive(ArchiveBuilder* builder, FileMapInfo*
builder->write_archive(map_info, heap_info);
if (AllowArchivingWithJavaAgent) {
- log_warning(cds)("This archive was created with AllowArchivingWithJavaAgent. It should be used "
- "for testing purposes only and should not be used in a production environment");
+ log_warning(cds)("This %s was created with AllowArchivingWithJavaAgent. It should be used "
+ "for testing purposes only and should not be used in a production environment", CDSConfig::type_of_archive_being_loaded());
}
return true;
}
@@ -1004,7 +1041,13 @@ bool MetaspaceShared::try_link_class(JavaThread* current, InstanceKlass* ik) {
ExceptionMark em(current);
JavaThread* THREAD = current; // For exception macros.
assert(CDSConfig::is_dumping_archive(), "sanity");
- if (!ik->is_shared() && ik->is_loaded() && !ik->is_linked() && ik->can_be_verified_at_dumptime() &&
+
+ if (ik->is_shared() && !CDSConfig::is_dumping_final_static_archive()) {
+ assert(CDSConfig::is_dumping_dynamic_archive(), "must be");
+ return false;
+ }
+
+ if (ik->is_loaded() && !ik->is_linked() && ik->can_be_verified_at_dumptime() &&
!SystemDictionaryShared::has_class_failed_verification(ik)) {
bool saved = BytecodeVerificationLocal;
if (ik->is_shared_unregistered_class() && ik->class_loader() == nullptr) {
@@ -1072,11 +1115,18 @@ bool MetaspaceShared::is_shared_static(void* p) {
// - When -XX:+RequireSharedSpaces is specified, AND the JVM cannot load the archive(s) due
// to version or classpath mismatch.
void MetaspaceShared::unrecoverable_loading_error(const char* message) {
- log_error(cds)("An error has occurred while processing the shared archive file.");
+ log_error(cds)("An error has occurred while processing the %s.", CDSConfig::type_of_archive_being_loaded());
if (message != nullptr) {
log_error(cds)("%s", message);
}
- vm_exit_during_initialization("Unable to use shared archive.", nullptr);
+
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ vm_exit_during_initialization("Must be a valid AOT configuration generated by the current JVM", AOTConfiguration);
+ } else if (CDSConfig::new_aot_flags_used()) {
+ vm_exit_during_initialization("Unable to use AOT cache.", nullptr);
+ } else {
+ vm_exit_during_initialization("Unable to use shared archive.", nullptr);
+ }
}
// This function is called when the JVM is unable to write the specified CDS archive due to an
@@ -1106,6 +1156,8 @@ void MetaspaceShared::initialize_runtime_shared_and_meta_spaces() {
log_info(cds)("Core region alignment: %zu", static_mapinfo->core_region_alignment());
dynamic_mapinfo = open_dynamic_archive();
+ log_info(cds)("ArchiveRelocationMode: %d", ArchiveRelocationMode);
+
// First try to map at the requested address
result = map_archives(static_mapinfo, dynamic_mapinfo, true);
if (result == MAP_ARCHIVE_MMAP_FAILURE) {
@@ -1223,6 +1275,7 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
ReservedSpace total_space_rs, archive_space_rs, class_space_rs;
MapArchiveResult result = MAP_ARCHIVE_OTHER_FAILURE;
+ size_t prot_zone_size = 0;
char* mapped_base_address = reserve_address_space_for_archives(static_mapinfo,
dynamic_mapinfo,
use_requested_addr,
@@ -1234,14 +1287,21 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
log_debug(cds)("Failed to reserve spaces (use_requested_addr=%u)", (unsigned)use_requested_addr);
} else {
+ if (Metaspace::using_class_space()) {
+ prot_zone_size = protection_zone_size();
+ }
+
#ifdef ASSERT
// Some sanity checks after reserving address spaces for archives
// and class space.
assert(archive_space_rs.is_reserved(), "Sanity");
if (Metaspace::using_class_space()) {
+ assert(archive_space_rs.base() == mapped_base_address &&
+ archive_space_rs.size() > protection_zone_size(),
+ "Archive space must lead and include the protection zone");
// Class space must closely follow the archive space. Both spaces
// must be aligned correctly.
- assert(class_space_rs.is_reserved(),
+ assert(class_space_rs.is_reserved() && class_space_rs.size() > 0,
"A class space should have been reserved");
assert(class_space_rs.base() >= archive_space_rs.end(),
"class space should follow the cds archive space");
@@ -1254,8 +1314,9 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
}
#endif // ASSERT
- log_info(cds)("Reserved archive_space_rs [" INTPTR_FORMAT " - " INTPTR_FORMAT "] (%zu) bytes",
- p2i(archive_space_rs.base()), p2i(archive_space_rs.end()), archive_space_rs.size());
+ log_info(cds)("Reserved archive_space_rs [" INTPTR_FORMAT " - " INTPTR_FORMAT "] (%zu) bytes%s",
+ p2i(archive_space_rs.base()), p2i(archive_space_rs.end()), archive_space_rs.size(),
+ (prot_zone_size > 0 ? " (includes protection zone)" : ""));
log_info(cds)("Reserved class_space_rs [" INTPTR_FORMAT " - " INTPTR_FORMAT "] (%zu) bytes",
p2i(class_space_rs.base()), p2i(class_space_rs.end()), class_space_rs.size());
@@ -1282,8 +1343,35 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
MemoryReserver::release(archive_space_rs);
// Mark as not reserved
archive_space_rs = {};
+ // The protection zone is part of the archive:
+ // See comment above, the Windows way of loading CDS is to mmap the individual
+ // parts of the archive into the address region we just vacated. The protection
+ // zone will not be mapped (and, in fact, does not exist as physical region in
+ // the archive). Therefore, after removing the archive space above, we must
+ // re-reserve the protection zone part lest something else gets mapped into that
+ // area later.
+ if (prot_zone_size > 0) {
+ assert(prot_zone_size >= os::vm_allocation_granularity(), "must be"); // not just page size!
+ char* p = os::attempt_reserve_memory_at(mapped_base_address, prot_zone_size,
+ false, MemTag::mtClassShared);
+ assert(p == mapped_base_address || p == nullptr, "must be");
+ if (p == nullptr) {
+ log_debug(cds)("Failed to re-reserve protection zone");
+ return MAP_ARCHIVE_MMAP_FAILURE;
+ }
+ }
}
}
+
+ if (prot_zone_size > 0) {
+ os::commit_memory(mapped_base_address, prot_zone_size, false); // will later be protected
+ // Before mapping the core regions into the newly established address space, we mark
+ // start and the end of the future protection zone with canaries. That way we easily
+ // catch mapping errors (accidentally mapping data into the future protection zone).
+ *(mapped_base_address) = 'P';
+ *(mapped_base_address + prot_zone_size - 1) = 'P';
+ }
+
MapArchiveResult static_result = map_archive(static_mapinfo, mapped_base_address, archive_space_rs);
MapArchiveResult dynamic_result = (static_result == MAP_ARCHIVE_SUCCESS) ?
map_archive(dynamic_mapinfo, mapped_base_address, archive_space_rs) : MAP_ARCHIVE_OTHER_FAILURE;
@@ -1327,38 +1415,40 @@ MapArchiveResult MetaspaceShared::map_archives(FileMapInfo* static_mapinfo, File
if (result == MAP_ARCHIVE_SUCCESS) {
SharedBaseAddress = (size_t)mapped_base_address;
#ifdef _LP64
- if (Metaspace::using_class_space()) {
- // Set up ccs in metaspace.
- Metaspace::initialize_class_space(class_space_rs);
+ if (Metaspace::using_class_space()) {
+ assert(prot_zone_size > 0 &&
+ *(mapped_base_address) == 'P' &&
+ *(mapped_base_address + prot_zone_size - 1) == 'P',
+ "Protection zone was overwritten?");
+ // Set up ccs in metaspace.
+ Metaspace::initialize_class_space(class_space_rs);
- // Set up compressed Klass pointer encoding: the encoding range must
- // cover both archive and class space.
- address cds_base = (address)static_mapinfo->mapped_base();
- address ccs_end = (address)class_space_rs.end();
- assert(ccs_end > cds_base, "Sanity check");
- if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
- // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
- // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
- // - every archived Klass' prototype (only if +UseCompactObjectHeaders)
- //
- // In order for those IDs to still be valid, we need to dictate base and shift: base should be the
- // mapping start, shift the shift used at archive generation time.
- address precomputed_narrow_klass_base = cds_base;
- const int precomputed_narrow_klass_shift = ArchiveBuilder::precomputed_narrow_klass_shift();
- CompressedKlassPointers::initialize_for_given_encoding(
- cds_base, ccs_end - cds_base, // Klass range
- precomputed_narrow_klass_base, precomputed_narrow_klass_shift // precomputed encoding, see ArchiveBuilder
- );
- } else {
- // Let JVM freely chose encoding base and shift
- CompressedKlassPointers::initialize (
- cds_base, ccs_end - cds_base // Klass range
- );
- }
- // map_or_load_heap_region() compares the current narrow oop and klass encodings
- // with the archived ones, so it must be done after all encodings are determined.
- static_mapinfo->map_or_load_heap_region();
- }
+ // Set up compressed Klass pointer encoding: the encoding range must
+ // cover both archive and class space.
+ const address encoding_base = (address)mapped_base_address;
+ const address klass_range_start = encoding_base + prot_zone_size;
+ const size_t klass_range_size = (address)class_space_rs.end() - klass_range_start;
+ if (INCLUDE_CDS_JAVA_HEAP || UseCompactObjectHeaders) {
+ // The CDS archive may contain narrow Klass IDs that were precomputed at archive generation time:
+ // - every archived java object header (only if INCLUDE_CDS_JAVA_HEAP)
+ // - every archived Klass' prototype (only if +UseCompactObjectHeaders)
+ //
+ // In order for those IDs to still be valid, we need to dictate base and shift: base should be the
+ // mapping start (including protection zone), shift should be the shift used at archive generation time.
+ CompressedKlassPointers::initialize_for_given_encoding(
+ klass_range_start, klass_range_size,
+ encoding_base, ArchiveBuilder::precomputed_narrow_klass_shift() // precomputed encoding, see ArchiveBuilder
+ );
+ } else {
+ // Let JVM freely choose encoding base and shift
+ CompressedKlassPointers::initialize(klass_range_start, klass_range_size);
+ }
+ CompressedKlassPointers::establish_protection_zone(encoding_base, prot_zone_size);
+
+ // map_or_load_heap_region() compares the current narrow oop and klass encodings
+ // with the archived ones, so it must be done after all encodings are determined.
+ static_mapinfo->map_or_load_heap_region();
+ }
#endif // _LP64
log_info(cds)("initial optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
log_info(cds)("initial full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
@@ -1440,7 +1530,6 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
const size_t archive_space_alignment = core_region_alignment();
// Size and requested location of the archive_space_rs (for both static and dynamic archives)
- assert(static_mapinfo->mapping_base_offset() == 0, "Must be");
size_t archive_end_offset = (dynamic_mapinfo == nullptr) ? static_mapinfo->mapping_end_offset() : dynamic_mapinfo->mapping_end_offset();
size_t archive_space_size = align_up(archive_end_offset, archive_space_alignment);
@@ -1461,7 +1550,7 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
assert(base_address == nullptr ||
(address)archive_space_rs.base() == base_address, "Sanity");
// Register archive space with NMT.
- MemTracker::record_virtual_memory_tag(archive_space_rs.base(), mtClassShared);
+ MemTracker::record_virtual_memory_tag(archive_space_rs, mtClassShared);
return archive_space_rs.base();
}
return nullptr;
@@ -1534,9 +1623,8 @@ char* MetaspaceShared::reserve_address_space_for_archives(FileMapInfo* static_ma
release_reserved_spaces(total_space_rs, archive_space_rs, class_space_rs);
return nullptr;
}
- // NMT: fix up the space tags
- MemTracker::record_virtual_memory_tag(archive_space_rs.base(), mtClassShared);
- MemTracker::record_virtual_memory_tag(class_space_rs.base(), mtClass);
+ MemTracker::record_virtual_memory_tag(archive_space_rs, mtClassShared);
+ MemTracker::record_virtual_memory_tag(class_space_rs, mtClass);
} else {
if (use_archive_base_addr && base_address != nullptr) {
total_space_rs = MemoryReserver::reserve((char*) base_address,
diff --git a/src/hotspot/share/cds/metaspaceShared.hpp b/src/hotspot/share/cds/metaspaceShared.hpp
index 6d5f273041a..8881dc6d6fd 100644
--- a/src/hotspot/share/cds/metaspaceShared.hpp
+++ b/src/hotspot/share/cds/metaspaceShared.hpp
@@ -131,14 +131,14 @@ public:
}
static bool try_link_class(JavaThread* current, InstanceKlass* ik);
- static void link_shared_classes(bool jcmd_request, TRAPS) NOT_CDS_RETURN;
- static bool link_class_for_cds(InstanceKlass* ik, TRAPS) NOT_CDS_RETURN_(false);
+ static void link_shared_classes(TRAPS) NOT_CDS_RETURN;
static bool may_be_eagerly_linked(InstanceKlass* ik) NOT_CDS_RETURN_(false);
#if INCLUDE_CDS
// Alignment for the 2 core CDS regions (RW/RO) only.
// (Heap region alignments are decided by GC).
static size_t core_region_alignment();
+ static size_t protection_zone_size();
static void rewrite_nofast_bytecodes_and_calculate_fingerprints(Thread* thread, InstanceKlass* ik);
// print loaded classes names to file.
static void dump_loaded_classes(const char* file_name, TRAPS);
diff --git a/src/hotspot/share/cds/runTimeClassInfo.hpp b/src/hotspot/share/cds/runTimeClassInfo.hpp
index ca60e11736d..8ad2efcbccb 100644
--- a/src/hotspot/share/cds/runTimeClassInfo.hpp
+++ b/src/hotspot/share/cds/runTimeClassInfo.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2021, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,13 @@ class Method;
class Symbol;
class RunTimeClassInfo {
-public:
+ public:
+ enum : char {
+ FROM_FIELD_IS_PROTECTED = 1 << 0,
+ FROM_IS_ARRAY = 1 << 1,
+ FROM_IS_OBJECT = 1 << 2
+ };
+
struct CrcInfo {
int _clsfile_size;
int _clsfile_crc32;
@@ -202,6 +208,17 @@ public:
return verifier_constraint_flags()[i];
}
+ bool from_field_is_protected(int i) {
+ return (verifier_constraint_flag(i) & FROM_FIELD_IS_PROTECTED) != 0;
+ }
+
+ bool from_is_array(int i) {
+ return (verifier_constraint_flag(i) & FROM_IS_ARRAY) != 0;
+ }
+ bool from_is_object(int i) {
+ return (verifier_constraint_flag(i) & FROM_IS_OBJECT) != 0;
+ }
+
int num_enum_klass_static_fields(int i) const {
return enum_klass_static_fields_addr()->_num;
}
diff --git a/src/hotspot/share/ci/bcEscapeAnalyzer.cpp b/src/hotspot/share/ci/bcEscapeAnalyzer.cpp
index cf7b839c982..712f7af4139 100644
--- a/src/hotspot/share/ci/bcEscapeAnalyzer.cpp
+++ b/src/hotspot/share/ci/bcEscapeAnalyzer.cpp
@@ -22,12 +22,12 @@
*
*/
-#include "classfile/vmIntrinsics.hpp"
#include "ci/bcEscapeAnalyzer.hpp"
#include "ci/ciConstant.hpp"
#include "ci/ciField.hpp"
#include "ci/ciMethodBlocks.hpp"
#include "ci/ciStreams.hpp"
+#include "classfile/vmIntrinsics.hpp"
#include "compiler/compiler_globals.hpp"
#include "interpreter/bytecode.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/ci/bcEscapeAnalyzer.hpp b/src/hotspot/share/ci/bcEscapeAnalyzer.hpp
index 925635f4fdc..27fb5cf1323 100644
--- a/src/hotspot/share/ci/bcEscapeAnalyzer.hpp
+++ b/src/hotspot/share/ci/bcEscapeAnalyzer.hpp
@@ -26,9 +26,9 @@
#define SHARE_CI_BCESCAPEANALYZER_HPP
#ifdef COMPILER2
-#include "ci/ciObject.hpp"
#include "ci/ciMethod.hpp"
#include "ci/ciMethodData.hpp"
+#include "ci/ciObject.hpp"
#include "code/dependencies.hpp"
#include "libadt/vectset.hpp"
#include "memory/allocation.hpp"
diff --git a/src/hotspot/share/ci/ciCallSite.cpp b/src/hotspot/share/ci/ciCallSite.cpp
index 3271e4ea7c7..c46758c0512 100644
--- a/src/hotspot/share/ci/ciCallSite.cpp
+++ b/src/hotspot/share/ci/ciCallSite.cpp
@@ -22,9 +22,9 @@
*
*/
-#include "classfile/javaClasses.inline.hpp"
#include "ci/ciCallSite.hpp"
#include "ci/ciUtilities.inline.hpp"
+#include "classfile/javaClasses.inline.hpp"
// ciCallSite
diff --git a/src/hotspot/share/ci/ciClassList.hpp b/src/hotspot/share/ci/ciClassList.hpp
index ccc99cfa211..618a052765e 100644
--- a/src/hotspot/share/ci/ciClassList.hpp
+++ b/src/hotspot/share/ci/ciClassList.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -75,13 +75,11 @@ class ciTypeArrayKlass;
// Everyone gives access to ciObjectFactory
#define CI_PACKAGE_ACCESS \
friend class ciObjectFactory; \
-friend class VMStructs;
// These are the packages that have access to ciEnv
// Any more access must be given explicitly.
#define CI_PACKAGE_ACCESS_TO \
friend class ciObjectFactory; \
-friend class VMStructs; \
friend class ciCallSite; \
friend class ciConstantPoolCache; \
friend class ciField; \
diff --git a/src/hotspot/share/ci/ciConstant.hpp b/src/hotspot/share/ci/ciConstant.hpp
index 1d0f60407ed..f07d0a170df 100644
--- a/src/hotspot/share/ci/ciConstant.hpp
+++ b/src/hotspot/share/ci/ciConstant.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,6 @@
//
// This class represents a constant value.
class ciConstant {
- friend class VMStructs;
private:
friend class ciEnv;
friend class ciField;
diff --git a/src/hotspot/share/ci/ciEnv.cpp b/src/hotspot/share/ci/ciEnv.cpp
index e87c5ba08e9..3991773d86f 100644
--- a/src/hotspot/share/ci/ciEnv.cpp
+++ b/src/hotspot/share/ci/ciEnv.cpp
@@ -42,8 +42,8 @@
#include "compiler/compilationLog.hpp"
#include "compiler/compilationPolicy.hpp"
#include "compiler/compileBroker.hpp"
-#include "compiler/compilerEvent.hpp"
#include "compiler/compileLog.hpp"
+#include "compiler/compilerEvent.hpp"
#include "compiler/compileTask.hpp"
#include "compiler/disassembler.hpp"
#include "gc/shared/collectedHeap.inline.hpp"
@@ -106,7 +106,7 @@ static bool firstEnv = true;
// ------------------------------------------------------------------
// ciEnv::ciEnv
ciEnv::ciEnv(CompileTask* task)
- : _ciEnv_arena(mtCompiler) {
+ : _ciEnv_arena(mtCompiler, Arena::Tag::tag_cienv) {
VM_ENTRY_MARK;
// Set up ciEnv::current immediately, for the sake of ciObjectFactory, etc.
@@ -238,7 +238,7 @@ public:
}
};
-ciEnv::ciEnv(Arena* arena) : _ciEnv_arena(mtCompiler) {
+ciEnv::ciEnv(Arena* arena) : _ciEnv_arena(mtCompiler, Arena::Tag::tag_cienv) {
ASSERT_IN_VM;
// Set up ciEnv::current immediately, for the sake of ciObjectFactory, etc.
diff --git a/src/hotspot/share/ci/ciEnv.hpp b/src/hotspot/share/ci/ciEnv.hpp
index 6c66633ee17..a22975c7453 100644
--- a/src/hotspot/share/ci/ciEnv.hpp
+++ b/src/hotspot/share/ci/ciEnv.hpp
@@ -31,9 +31,9 @@
#include "code/debugInfoRec.hpp"
#include "code/dependencies.hpp"
#include "code/exceptionHandlerTable.hpp"
+#include "compiler/cHeapStringHolder.hpp"
#include "compiler/compiler_globals.hpp"
#include "compiler/compilerThread.hpp"
-#include "compiler/cHeapStringHolder.hpp"
#include "oops/methodData.hpp"
#include "runtime/javaThread.hpp"
diff --git a/src/hotspot/share/ci/ciInstance.cpp b/src/hotspot/share/ci/ciInstance.cpp
index edd8f8b0f40..ad456ba6726 100644
--- a/src/hotspot/share/ci/ciInstance.cpp
+++ b/src/hotspot/share/ci/ciInstance.cpp
@@ -22,13 +22,13 @@
*
*/
-#include "classfile/javaClasses.inline.hpp"
#include "ci/ciConstant.hpp"
#include "ci/ciField.hpp"
#include "ci/ciInstance.hpp"
#include "ci/ciInstanceKlass.hpp"
#include "ci/ciNullObject.hpp"
#include "ci/ciUtilities.inline.hpp"
+#include "classfile/javaClasses.inline.hpp"
#include "classfile/vmClasses.hpp"
#include "oops/oop.inline.hpp"
diff --git a/src/hotspot/share/ci/ciInstanceKlass.cpp b/src/hotspot/share/ci/ciInstanceKlass.cpp
index 5799f30ad67..a24a68f874b 100644
--- a/src/hotspot/share/ci/ciInstanceKlass.cpp
+++ b/src/hotspot/share/ci/ciInstanceKlass.cpp
@@ -31,10 +31,10 @@
#include "memory/allocation.hpp"
#include "memory/allocation.inline.hpp"
#include "memory/resourceArea.hpp"
+#include "oops/fieldStreams.inline.hpp"
#include "oops/instanceKlass.inline.hpp"
#include "oops/klass.inline.hpp"
#include "oops/oop.inline.hpp"
-#include "oops/fieldStreams.inline.hpp"
#include "runtime/fieldDescriptor.inline.hpp"
#include "runtime/handles.inline.hpp"
#include "runtime/jniHandles.inline.hpp"
@@ -400,9 +400,6 @@ ciField* ciInstanceKlass::get_field_by_offset(int field_offset, bool is_static)
int field_off = field->offset_in_bytes();
if (field_off == field_offset)
return field;
- if (field_off > field_offset)
- break;
- // could do binary search or check bins, but probably not worth it
}
return nullptr;
}
@@ -431,11 +428,6 @@ ciField* ciInstanceKlass::get_field_by_name(ciSymbol* name, ciSymbol* signature,
}
-static int sort_field_by_offset(ciField** a, ciField** b) {
- return (*a)->offset_in_bytes() - (*b)->offset_in_bytes();
- // (no worries about 32-bit overflow...)
-}
-
// ------------------------------------------------------------------
// ciInstanceKlass::compute_nonstatic_fields
int ciInstanceKlass::compute_nonstatic_fields() {
@@ -476,9 +468,6 @@ int ciInstanceKlass::compute_nonstatic_fields() {
int flen = fields->length();
- // Now sort them by offset, ascending.
- // (In principle, they could mix with superclass fields.)
- fields->sort(sort_field_by_offset);
_nonstatic_fields = fields;
return flen;
}
diff --git a/src/hotspot/share/ci/ciInstanceKlass.hpp b/src/hotspot/share/ci/ciInstanceKlass.hpp
index 9c1c416780d..69b73152d37 100644
--- a/src/hotspot/share/ci/ciInstanceKlass.hpp
+++ b/src/hotspot/share/ci/ciInstanceKlass.hpp
@@ -67,7 +67,7 @@ private:
ciInstance* _java_mirror;
ciConstantPoolCache* _field_cache; // cached map index->field
- GrowableArray* _nonstatic_fields;
+ GrowableArray* _nonstatic_fields; // ordered by JavaFieldStream
int _has_injected_fields; // any non static injected fields? lazily initialized.
// The possible values of the _implementor fall into following three cases:
diff --git a/src/hotspot/share/ci/ciMetadata.hpp b/src/hotspot/share/ci/ciMetadata.hpp
index 62b67d65483..2e232c59188 100644
--- a/src/hotspot/share/ci/ciMetadata.hpp
+++ b/src/hotspot/share/ci/ciMetadata.hpp
@@ -103,7 +103,7 @@ class ciMetadata: public ciBaseObject {
Metadata* constant_encoding() { return _metadata; }
- bool equals(ciMetadata* obj) const { return (this == obj); }
+ bool equals(const ciMetadata* obj) const { return (this == obj); }
uint hash() { return ident() * 31; } // ???
diff --git a/src/hotspot/share/ci/ciMethod.cpp b/src/hotspot/share/ci/ciMethod.cpp
index 96cb0bb9040..3b2670c3eb0 100644
--- a/src/hotspot/share/ci/ciMethod.cpp
+++ b/src/hotspot/share/ci/ciMethod.cpp
@@ -28,9 +28,9 @@
#include "ci/ciMethod.hpp"
#include "ci/ciMethodBlocks.hpp"
#include "ci/ciMethodData.hpp"
+#include "ci/ciReplay.hpp"
#include "ci/ciStreams.hpp"
#include "ci/ciSymbol.hpp"
-#include "ci/ciReplay.hpp"
#include "ci/ciSymbols.hpp"
#include "ci/ciUtilities.inline.hpp"
#include "compiler/abstractCompiler.hpp"
diff --git a/src/hotspot/share/ci/ciObjectFactory.cpp b/src/hotspot/share/ci/ciObjectFactory.cpp
index 35ec27b8aaa..1fa590e4ad3 100644
--- a/src/hotspot/share/ci/ciObjectFactory.cpp
+++ b/src/hotspot/share/ci/ciObjectFactory.cpp
@@ -108,7 +108,7 @@ void ciObjectFactory::initialize() {
// This Arena is long lived and exists in the resource mark of the
// compiler thread that initializes the initial ciObjectFactory which
// creates the shared ciObjects that all later ciObjectFactories use.
- Arena* arena = new (mtCompiler) Arena(mtCompiler);
+ Arena* arena = new (mtCompiler) Arena(mtCompiler, Arena::Tag::tag_cienv);
ciEnv initial(arena);
ciEnv* env = ciEnv::current();
env->_factory->init_shared_objects();
diff --git a/src/hotspot/share/ci/ciObjectFactory.hpp b/src/hotspot/share/ci/ciObjectFactory.hpp
index f1f6ae24545..d95a7d1ff22 100644
--- a/src/hotspot/share/ci/ciObjectFactory.hpp
+++ b/src/hotspot/share/ci/ciObjectFactory.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -37,7 +37,6 @@
// which ensures that for each oop, at most one ciObject is created.
// This invariant allows efficient implementation of ciObject.
class ciObjectFactory : public ArenaObj {
- friend class VMStructs;
friend class ciEnv;
private:
diff --git a/src/hotspot/share/ci/ciReplay.cpp b/src/hotspot/share/ci/ciReplay.cpp
index 1385bb637a9..d490b75919d 100644
--- a/src/hotspot/share/ci/ciReplay.cpp
+++ b/src/hotspot/share/ci/ciReplay.cpp
@@ -22,10 +22,10 @@
*
*/
+#include "ci/ciKlass.hpp"
#include "ci/ciMethodData.hpp"
#include "ci/ciReplay.hpp"
#include "ci/ciSymbol.hpp"
-#include "ci/ciKlass.hpp"
#include "ci/ciUtilities.inline.hpp"
#include "classfile/javaClasses.hpp"
#include "classfile/symbolTable.hpp"
@@ -802,7 +802,7 @@ class CompileReplay : public StackObj {
// Make sure the existence of a prior compile doesn't stop this one
nmethod* nm = (entry_bci != InvocationEntryBci) ? method->lookup_osr_nmethod_for(entry_bci, comp_level, true) : method->code();
if (nm != nullptr) {
- nm->make_not_entrant();
+ nm->make_not_entrant("CI replay");
}
replay_state = this;
CompileBroker::compile_method(methodHandle(THREAD, method), entry_bci, comp_level,
diff --git a/src/hotspot/share/ci/ciStreams.cpp b/src/hotspot/share/ci/ciStreams.cpp
index 259e72a3412..0859805b843 100644
--- a/src/hotspot/share/ci/ciStreams.cpp
+++ b/src/hotspot/share/ci/ciStreams.cpp
@@ -126,7 +126,7 @@ Bytecodes::Code ciBytecodeStream::next_wide_or_table(Bytecodes::Code bc) {
}
default:
- fatal("unhandled bytecode");
+ fatal("unhandled bytecode : Current Method = %s, BCI = %d, OPCODE = %s (0x%X)", _method->name()->as_utf8(), cur_bci(), Bytecodes::name(bc), bc);
}
return bc;
}
diff --git a/src/hotspot/share/ci/ciTypeFlow.cpp b/src/hotspot/share/ci/ciTypeFlow.cpp
index 3caca6424bc..234b4611ea1 100644
--- a/src/hotspot/share/ci/ciTypeFlow.cpp
+++ b/src/hotspot/share/ci/ciTypeFlow.cpp
@@ -500,8 +500,8 @@ bool ciTypeFlow::StateVector::meet_exception(ciInstanceKlass* exc,
bool different = false;
// Meet locals from incoming array.
- Cell limit = local(_outer->max_locals()-1);
- for (Cell c = start_cell(); c <= limit; c = next_cell(c)) {
+ Cell limit = local_limit_cell();
+ for (Cell c = start_cell(); c < limit; c = next_cell(c)) {
ciType* t1 = type_at(c);
ciType* t2 = incoming->type_at(c);
if (!t1->equals(t2)) {
diff --git a/src/hotspot/share/ci/ciTypeFlow.hpp b/src/hotspot/share/ci/ciTypeFlow.hpp
index 2e727985307..92db6253aa0 100644
--- a/src/hotspot/share/ci/ciTypeFlow.hpp
+++ b/src/hotspot/share/ci/ciTypeFlow.hpp
@@ -213,9 +213,12 @@ public:
return (Cell)(outer()->max_locals() + stack_size());
}
+ Cell local_limit_cell() const { return (Cell) outer()->max_locals(); }
+
// Cell creation
Cell local(int lnum) const {
assert(lnum < outer()->max_locals(), "index check");
+ assert(Cell_0 <= lnum && lnum <= Cell_max, "out of Cell's range");
return (Cell)(lnum);
}
diff --git a/src/hotspot/share/ci/ciUtilities.cpp b/src/hotspot/share/ci/ciUtilities.cpp
index 1a91ded9371..0c5b4d9824f 100644
--- a/src/hotspot/share/ci/ciUtilities.cpp
+++ b/src/hotspot/share/ci/ciUtilities.cpp
@@ -23,9 +23,10 @@
*/
#include "ci/ciUtilities.hpp"
-#include "gc/shared/cardTableBarrierSet.hpp"
#include "gc/shared/cardTable.hpp"
+#include "gc/shared/cardTableBarrierSet.hpp"
#include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/gc_globals.hpp"
// ciUtilities
//
@@ -45,5 +46,6 @@ CardTable::CardValue* ci_card_table_address() {
BarrierSet* bs = BarrierSet::barrier_set();
CardTableBarrierSet* ctbs = barrier_set_cast(bs);
CardTable* ct = ctbs->card_table();
+ assert(!UseShenandoahGC, "Shenandoah byte_map_base is not constant.");
return ct->byte_map_base();
}
diff --git a/src/hotspot/share/classfile/classFileParser.cpp b/src/hotspot/share/classfile/classFileParser.cpp
index ae66ee22511..3e6246d4aee 100644
--- a/src/hotspot/share/classfile/classFileParser.cpp
+++ b/src/hotspot/share/classfile/classFileParser.cpp
@@ -5333,7 +5333,8 @@ ClassFileParser::ClassFileParser(ClassFileStream* stream,
assert(0 == _access_flags.as_unsigned_short(), "invariant");
// Figure out whether we can skip format checking (matching classic VM behavior)
- _need_verify = Verifier::should_verify_for(_loader_data->class_loader());
+ // Always verify CFLH bytes from the user agents.
+ _need_verify = stream->from_class_file_load_hook() ? true : Verifier::should_verify_for(_loader_data->class_loader());
// synch back verification state to stream to check for truncation.
stream->set_need_verify(_need_verify);
diff --git a/src/hotspot/share/classfile/classFileStream.cpp b/src/hotspot/share/classfile/classFileStream.cpp
index 0fc3e5b390c..382bd58af2b 100644
--- a/src/hotspot/share/classfile/classFileStream.cpp
+++ b/src/hotspot/share/classfile/classFileStream.cpp
@@ -34,13 +34,15 @@ void ClassFileStream::truncated_file_error(TRAPS) const {
ClassFileStream::ClassFileStream(const u1* buffer,
int length,
const char* source,
- bool from_boot_loader_modules_image) :
+ bool from_boot_loader_modules_image,
+ bool from_class_file_load_hook) :
_buffer_start(buffer),
_buffer_end(buffer + length),
_current(buffer),
_source(source),
_need_verify(true), // may be reset by ClassFileParser when this stream is parsed.
- _from_boot_loader_modules_image(from_boot_loader_modules_image) {
+ _from_boot_loader_modules_image(from_boot_loader_modules_image),
+ _from_class_file_load_hook(from_class_file_load_hook) {
assert(buffer != nullptr, "caller should throw NPE");
}
@@ -68,5 +70,6 @@ const ClassFileStream* ClassFileStream::clone() const {
return new ClassFileStream(new_buffer_start,
length(),
clone_source(),
- from_boot_loader_modules_image());
+ from_boot_loader_modules_image(),
+ from_class_file_load_hook());
}
diff --git a/src/hotspot/share/classfile/classFileStream.hpp b/src/hotspot/share/classfile/classFileStream.hpp
index 67d66bb74a1..135a34bdeee 100644
--- a/src/hotspot/share/classfile/classFileStream.hpp
+++ b/src/hotspot/share/classfile/classFileStream.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -45,7 +45,9 @@ class ClassFileStream: public ResourceObj {
const char* const _source; // Source of stream (directory name, ZIP/JAR archive name)
bool _need_verify; // True if we need to verify and check truncation of stream bytes.
bool _from_boot_loader_modules_image; // True if this was created by ClassPathImageEntry.
- void truncated_file_error(TRAPS) const ;
+ bool _from_class_file_load_hook; // True if this is from CFLH (needs verification)
+
+ void truncated_file_error(TRAPS) const;
protected:
const u1* clone_buffer() const;
@@ -55,7 +57,8 @@ class ClassFileStream: public ResourceObj {
ClassFileStream(const u1* buffer,
int length,
const char* source,
- bool from_boot_loader_modules_image = false);
+ bool from_boot_loader_modules_image = false,
+ bool from_class_file_load_hook = false);
virtual const ClassFileStream* clone() const;
@@ -154,6 +157,8 @@ class ClassFileStream: public ResourceObj {
bool at_eos() const { return _current == _buffer_end; }
uint64_t compute_fingerprint() const;
+
+ bool from_class_file_load_hook() const { return _from_class_file_load_hook; }
};
#endif // SHARE_CLASSFILE_CLASSFILESTREAM_HPP
diff --git a/src/hotspot/share/classfile/compactHashtable.cpp b/src/hotspot/share/classfile/compactHashtable.cpp
index 5a3c6998904..8d50e8136a3 100644
--- a/src/hotspot/share/classfile/compactHashtable.cpp
+++ b/src/hotspot/share/classfile/compactHashtable.cpp
@@ -72,19 +72,6 @@ CompactHashtableWriter::~CompactHashtableWriter() {
FREE_C_HEAP_ARRAY(GrowableArray*, _buckets);
}
-size_t CompactHashtableWriter::estimate_size(int num_entries) {
- int num_buckets = calculate_num_buckets(num_entries);
- size_t bucket_bytes = ArchiveBuilder::ro_array_bytesize(num_buckets + 1);
-
- // In worst case, we have no VALUE_ONLY_BUCKET_TYPE, so each entry takes 2 slots
- int entries_space = 2 * num_entries;
- size_t entry_bytes = ArchiveBuilder::ro_array_bytesize(entries_space);
-
- return bucket_bytes
- + entry_bytes
- + SimpleCompactHashtable::calculate_header_size();
-}
-
// Add a symbol entry to the temporary hash table
void CompactHashtableWriter::add(unsigned int hash, u4 value) {
int index = hash % _num_buckets;
diff --git a/src/hotspot/share/classfile/compactHashtable.hpp b/src/hotspot/share/classfile/compactHashtable.hpp
index 73e9f7fc092..2985f0f9a1a 100644
--- a/src/hotspot/share/classfile/compactHashtable.hpp
+++ b/src/hotspot/share/classfile/compactHashtable.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -134,8 +134,6 @@ private:
public:
void dump(SimpleCompactHashtable *cht, const char* table_name);
-
- static size_t estimate_size(int num_entries);
};
#endif // INCLUDE_CDS
@@ -243,7 +241,6 @@ template <
bool (*EQUALS)(V value, K key, int len)
>
class CompactHashtable : public SimpleCompactHashtable {
- friend class VMStructs;
V decode(u4 offset) const {
return DECODE(_base_address, offset);
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
index 1af9bb83361..b6007923907 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.cpp
@@ -126,17 +126,19 @@ void FieldLayout::initialize_static_layout() {
}
}
-void FieldLayout::initialize_instance_layout(const InstanceKlass* super_klass) {
+void FieldLayout::initialize_instance_layout(const InstanceKlass* super_klass, bool& super_ends_with_oop) {
if (super_klass == nullptr) {
+ super_ends_with_oop = false;
_blocks = new LayoutRawBlock(LayoutRawBlock::EMPTY, INT_MAX);
_blocks->set_offset(0);
_last = _blocks;
_start = _blocks;
insert(first_empty_block(), new LayoutRawBlock(LayoutRawBlock::RESERVED, instanceOopDesc::base_offset_in_bytes()));
} else {
- bool has_fields = reconstruct_layout(super_klass);
+ bool super_has_instance_fields = false;
+ reconstruct_layout(super_klass, super_has_instance_fields, super_ends_with_oop);
fill_holes(super_klass);
- if (!super_klass->has_contended_annotations() || !has_fields) {
+ if (!super_klass->has_contended_annotations() || !super_has_instance_fields) {
_start = _blocks; // start allocating fields from the first empty block
} else {
_start = _last; // append fields at the end of the reconstructed layout
@@ -293,15 +295,21 @@ LayoutRawBlock* FieldLayout::insert_field_block(LayoutRawBlock* slot, LayoutRawB
return block;
}
-bool FieldLayout::reconstruct_layout(const InstanceKlass* ik) {
- bool has_instance_fields = false;
+void FieldLayout::reconstruct_layout(const InstanceKlass* ik, bool& has_instance_fields, bool& ends_with_oop) {
+ has_instance_fields = ends_with_oop = false;
GrowableArray* all_fields = new GrowableArray(32);
+ BasicType last_type;
+ int last_offset = -1;
while (ik != nullptr) {
for (AllFieldStream fs(ik->fieldinfo_stream(), ik->constants()); !fs.done(); fs.next()) {
BasicType type = Signature::basic_type(fs.signature());
// distinction between static and non-static fields is missing
if (fs.access_flags().is_static()) continue;
has_instance_fields = true;
+ if (fs.offset() > last_offset) {
+ last_offset = fs.offset();
+ last_type = type;
+ }
int size = type2aelembytes(type);
// INHERITED blocks are marked as non-reference because oop_maps are handled by their holder class
LayoutRawBlock* block = new LayoutRawBlock(fs.index(), LayoutRawBlock::INHERITED, size, size, false);
@@ -310,6 +318,11 @@ bool FieldLayout::reconstruct_layout(const InstanceKlass* ik) {
}
ik = ik->super() == nullptr ? nullptr : InstanceKlass::cast(ik->super());
}
+ assert(last_offset == -1 || last_offset > 0, "Sanity");
+ if (last_offset > 0 &&
+ (last_type == BasicType::T_ARRAY || last_type == BasicType::T_OBJECT)) {
+ ends_with_oop = true;
+ }
all_fields->sort(LayoutRawBlock::compare_offset);
_blocks = new LayoutRawBlock(LayoutRawBlock::RESERVED, instanceOopDesc::base_offset_in_bytes());
@@ -323,7 +336,6 @@ bool FieldLayout::reconstruct_layout(const InstanceKlass* ik) {
_last = b;
}
_start = _blocks;
- return has_instance_fields;
}
// Called during the reconstruction of a layout, after fields from super
@@ -516,7 +528,7 @@ FieldGroup* FieldLayoutBuilder::get_or_create_contended_group(int g) {
void FieldLayoutBuilder::prologue() {
_layout = new FieldLayout(_field_info, _constant_pool);
const InstanceKlass* super_klass = _super_klass;
- _layout->initialize_instance_layout(super_klass);
+ _layout->initialize_instance_layout(super_klass, _super_ends_with_oop);
if (super_klass != nullptr) {
_has_nonstatic_fields = super_klass->has_nonstatic_fields();
}
@@ -594,8 +606,14 @@ void FieldLayoutBuilder::insert_contended_padding(LayoutRawBlock* slot) {
// Computation of regular classes layout is an evolution of the previous default layout
// (FieldAllocationStyle 1):
// - primitive fields are allocated first (from the biggest to the smallest)
-// - then oop fields are allocated, either in existing gaps or at the end of
-// the layout
+// - oop fields are allocated, either in existing gaps or at the end of
+// the layout. We allocate oops in a single block to have a single oop map entry.
+// - if the super class ended with an oop, we lead with oops. That will cause the
+// trailing oop map entry of the super class and the oop map entry of this class
+// to be folded into a single entry later. Correspondingly, if the super class
+// ends with a primitive field, we gain nothing by leading with oops; therefore
+// we let oop fields trail, thus giving future derived classes the chance to apply
+// the same trick.
void FieldLayoutBuilder::compute_regular_layout() {
bool need_tail_padding = false;
prologue();
@@ -608,8 +626,14 @@ void FieldLayoutBuilder::compute_regular_layout() {
insert_contended_padding(_layout->start());
need_tail_padding = true;
}
- _layout->add(_root_group->primitive_fields());
- _layout->add(_root_group->oop_fields());
+
+ if (_super_ends_with_oop) {
+ _layout->add(_root_group->oop_fields());
+ _layout->add(_root_group->primitive_fields());
+ } else {
+ _layout->add(_root_group->primitive_fields());
+ _layout->add(_root_group->oop_fields());
+ }
if (!_contended_groups.is_empty()) {
for (int i = 0; i < _contended_groups.length(); i++) {
diff --git a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
index 9b0d80b2a55..82bbaefc623 100644
--- a/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
+++ b/src/hotspot/share/classfile/fieldLayoutBuilder.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -169,7 +169,7 @@ class FieldLayout : public ResourceObj {
public:
FieldLayout(GrowableArray* field_info, ConstantPool* cp);
void initialize_static_layout();
- void initialize_instance_layout(const InstanceKlass* ik);
+ void initialize_instance_layout(const InstanceKlass* ik, bool& super_ends_with_oop);
LayoutRawBlock* first_empty_block() {
LayoutRawBlock* block = _start;
@@ -188,7 +188,7 @@ class FieldLayout : public ResourceObj {
void add_field_at_offset(LayoutRawBlock* blocks, int offset, LayoutRawBlock* start = nullptr);
void add_contiguously(GrowableArray* list, LayoutRawBlock* start = nullptr);
LayoutRawBlock* insert_field_block(LayoutRawBlock* slot, LayoutRawBlock* block);
- bool reconstruct_layout(const InstanceKlass* ik);
+ void reconstruct_layout(const InstanceKlass* ik, bool& has_instance_fields, bool& ends_with_oop);
void fill_holes(const InstanceKlass* ik);
LayoutRawBlock* insert(LayoutRawBlock* slot, LayoutRawBlock* block);
void remove(LayoutRawBlock* block);
@@ -237,6 +237,7 @@ class FieldLayoutBuilder : public ResourceObj {
int _alignment;
bool _has_nonstatic_fields;
bool _is_contended; // is a contended class?
+ bool _super_ends_with_oop;
public:
FieldLayoutBuilder(const Symbol* classname, const InstanceKlass* super_klass, ConstantPool* constant_pool,
diff --git a/src/hotspot/share/classfile/javaClasses.cpp b/src/hotspot/share/classfile/javaClasses.cpp
index a224ef481b0..a2ad1dce5e4 100644
--- a/src/hotspot/share/classfile/javaClasses.cpp
+++ b/src/hotspot/share/classfile/javaClasses.cpp
@@ -5452,8 +5452,8 @@ void JavaClasses::serialize_offsets(SerializeClosure* soc) {
bool JavaClasses::is_supported_for_archiving(oop obj) {
Klass* klass = obj->klass();
- if (!CDSConfig::is_dumping_invokedynamic()) {
- // These are supported by CDS only when CDSConfig::is_dumping_invokedynamic() is enabled.
+ if (!CDSConfig::is_dumping_method_handles()) {
+ // These are supported by CDS only when CDSConfig::is_dumping_method_handles() is enabled.
if (klass == vmClasses::ResolvedMethodName_klass() ||
klass == vmClasses::MemberName_klass()) {
return false;
diff --git a/src/hotspot/share/classfile/klassFactory.cpp b/src/hotspot/share/classfile/klassFactory.cpp
index 52da747d8b6..28bb176ed0a 100644
--- a/src/hotspot/share/classfile/klassFactory.cpp
+++ b/src/hotspot/share/classfile/klassFactory.cpp
@@ -76,7 +76,9 @@ InstanceKlass* KlassFactory::check_shared_class_file_load_hook(
s2 path_index = ik->shared_classpath_index();
ClassFileStream* stream = new ClassFileStream(ptr,
pointer_delta_as_int(end_ptr, ptr),
- cfs->source());
+ cfs->source(),
+ /* from_boot_loader_modules_image */ false,
+ /* from_class_file_load_hook */ true);
ClassLoadInfo cl_info(protection_domain);
ClassFileParser parser(stream,
class_name,
@@ -155,7 +157,9 @@ static ClassFileStream* check_class_file_load_hook(ClassFileStream* stream,
// Set new class file stream using JVMTI agent modified class file data.
stream = new ClassFileStream(ptr,
pointer_delta_as_int(end_ptr, ptr),
- stream->source());
+ stream->source(),
+ /* from_boot_loader_modules_image */ false,
+ /* from_class_file_load_hook */ true);
}
}
diff --git a/src/hotspot/share/classfile/moduleEntry.cpp b/src/hotspot/share/classfile/moduleEntry.cpp
index 55363d7f41f..18f2abfd5f4 100644
--- a/src/hotspot/share/classfile/moduleEntry.cpp
+++ b/src/hotspot/share/classfile/moduleEntry.cpp
@@ -413,7 +413,13 @@ ModuleEntry* ModuleEntry::allocate_archived_entry() const {
_archive_modules_entries->put(this, archived_entry);
DEBUG_ONLY(_num_archived_module_entries++);
- assert(archived_entry->shared_protection_domain() == nullptr, "never set during -Xshare:dump");
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ OopHandle null_handle;
+ archived_entry->_shared_pd = null_handle;
+ } else {
+ assert(archived_entry->shared_protection_domain() == nullptr, "never set during -Xshare:dump");
+ }
+
// Clear handles and restore at run time. Handles cannot be archived.
OopHandle null_handle;
archived_entry->_module = null_handle;
diff --git a/src/hotspot/share/classfile/moduleEntry.hpp b/src/hotspot/share/classfile/moduleEntry.hpp
index 9ad7bde8954..c2d07a78512 100644
--- a/src/hotspot/share/classfile/moduleEntry.hpp
+++ b/src/hotspot/share/classfile/moduleEntry.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 2016, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -207,7 +207,6 @@ public:
void load_from_archive(ClassLoaderData* loader_data);
void restore_archived_oops(ClassLoaderData* loader_data);
void clear_archived_oops();
- void update_oops_in_archived_module(int root_oop_index);
static void verify_archived_module_entries() PRODUCT_RETURN;
#endif
};
diff --git a/src/hotspot/share/classfile/modules.cpp b/src/hotspot/share/classfile/modules.cpp
index 950ca699a8e..3f2ff90ccab 100644
--- a/src/hotspot/share/classfile/modules.cpp
+++ b/src/hotspot/share/classfile/modules.cpp
@@ -541,32 +541,81 @@ void Modules::verify_archived_modules() {
ModuleEntry::verify_archived_module_entries();
}
-char* Modules::_archived_main_module_name = nullptr;
-char* Modules::_archived_addmods_names = nullptr;
-char* Modules::_archived_native_access_flags = nullptr;
+class Modules::ArchivedProperty {
+ const char* _prop;
+ const bool _numbered;
+ const char* _archived_value;
-void Modules::dump_main_module_name() {
- const char* module_name = Arguments::get_property("jdk.module.main");
- if (module_name != nullptr) {
- _archived_main_module_name = ArchiveBuilder::current()->ro_strdup(module_name);
+ const char* get_flattened_value() const {
+ if (_numbered) {
+ return get_numbered_property_as_sorted_string();
+ } else {
+ return Arguments::get_property(_prop);
+ }
}
+
+ void runtime_check() const;
+ const char* get_numbered_property_as_sorted_string() const;
+
+public:
+ ArchivedProperty(const char* prop, bool numbered)
+ : _prop(prop), _numbered(numbered), _archived_value(nullptr) {}
+
+ void dump() {
+ ResourceMark rm;
+ const char* str = get_flattened_value();
+ if (str != nullptr) {
+ _archived_value = ArchiveBuilder::current()->ro_strdup(str);
+ }
+ }
+
+ void serialize(SerializeClosure* soc) {
+ soc->do_ptr(&_archived_value);
+ if (soc->reading()) {
+ runtime_check();
+ // Don't hold onto the pointer, in case we might decide to unmap the archive.
+ _archived_value = nullptr;
+ }
+ }
+};
+
+Modules::ArchivedProperty Modules::_archived_props[] = {
+ // numbered
+ {"jdk.module.main", false},
+
+ // non-numbered
+ {"jdk.module.addexports", true}, // --add-exports
+ {"jdk.module.addmods", true}, // --add-modules
+ {"jdk.module.enable.native.access", true}, // --enable-native-access
+};
+
+constexpr size_t Modules::num_archived_props() {
+ return sizeof(_archived_props) / sizeof(_archived_props[0]);
}
-void Modules::check_archived_flag_consistency(char* archived_flag, const char* runtime_flag, const char* property) {
- log_info(cds)("%s %s", property,
- archived_flag != nullptr ? archived_flag : "(null)");
+Modules::ArchivedProperty& Modules::archived_prop(size_t i) {
+ assert(i < num_archived_props(), "oob");
+ return _archived_props[i];
+}
+
+void Modules::ArchivedProperty::runtime_check() const {
+ ResourceMark rm;
+ const char* runtime_value = get_flattened_value();
+ log_info(cds)("archived module property %s: %s", _prop,
+ _archived_value != nullptr ? _archived_value : "(null)");
+
bool disable = false;
- if (runtime_flag == nullptr) {
- if (archived_flag != nullptr) {
- log_info(cds)("Mismatched values for property %s: %s specified during dump time but not during runtime", property, archived_flag);
+ if (runtime_value == nullptr) {
+ if (_archived_value != nullptr) {
+ log_info(cds)("Mismatched values for property %s: %s specified during dump time but not during runtime", _prop, _archived_value);
disable = true;
}
} else {
- if (archived_flag == nullptr) {
- log_info(cds)("Mismatched values for property %s: %s specified during runtime but not during dump time", property, runtime_flag);
+ if (_archived_value == nullptr) {
+ log_info(cds)("Mismatched values for property %s: %s specified during runtime but not during dump time", _prop, runtime_value);
disable = true;
- } else if (strcmp(runtime_flag, archived_flag) != 0) {
- log_info(cds)("Mismatched values for property %s: runtime %s dump time %s", property, runtime_flag, archived_flag);
+ } else if (strcmp(runtime_value, _archived_value) != 0) {
+ log_info(cds)("Mismatched values for property %s: runtime %s dump time %s", _prop, runtime_value, _archived_value);
disable = true;
}
}
@@ -575,98 +624,20 @@ void Modules::check_archived_flag_consistency(char* archived_flag, const char* r
log_info(cds)("Disabling optimized module handling");
CDSConfig::stop_using_optimized_module_handling();
}
- log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
- log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
-}
-
-void Modules::dump_archived_module_info() {
- // Write module name into archive
- CDS_JAVA_HEAP_ONLY(Modules::dump_main_module_name();)
- // Write module names from --add-modules into archive
- CDS_JAVA_HEAP_ONLY(Modules::dump_addmods_names();)
- // Write native enable-native-access flag into archive
- CDS_JAVA_HEAP_ONLY(Modules::dump_native_access_flag());
-}
-
-void Modules::serialize_archived_module_info(SerializeClosure* soc) {
- CDS_JAVA_HEAP_ONLY(Modules::serialize(soc);)
- CDS_JAVA_HEAP_ONLY(Modules::serialize_addmods_names(soc);)
- CDS_JAVA_HEAP_ONLY(Modules::serialize_native_access_flags(soc);)
-}
-
-void Modules::serialize(SerializeClosure* soc) {
- soc->do_ptr(&_archived_main_module_name);
- if (soc->reading()) {
- const char* runtime_main_module = Arguments::get_property("jdk.module.main");
- log_info(cds)("_archived_main_module_name %s",
- _archived_main_module_name != nullptr ? _archived_main_module_name : "(null)");
-
- check_archived_flag_consistency(_archived_main_module_name, runtime_main_module, "jdk.module.main");
-
- // Don't hold onto the pointer, in case we might decide to unmap the archive.
- _archived_main_module_name = nullptr;
- }
-}
-
-void Modules::dump_native_access_flag() {
- ResourceMark rm;
- const char* native_access_names = get_native_access_flags_as_sorted_string();
- if (native_access_names != nullptr) {
- _archived_native_access_flags = ArchiveBuilder::current()->ro_strdup(native_access_names);
- }
}
// Caller needs ResourceMark
-const char* Modules::get_native_access_flags_as_sorted_string() {
- return get_numbered_property_as_sorted_string("jdk.module.enable.native.access");
-}
-
-void Modules::serialize_native_access_flags(SerializeClosure* soc) {
- soc->do_ptr(&_archived_native_access_flags);
- if (soc->reading()) {
- ResourceMark rm;
- check_archived_flag_consistency(_archived_native_access_flags, get_native_access_flags_as_sorted_string(), "jdk.module.enable.native.access");
-
- // Don't hold onto the pointer, in case we might decide to unmap the archive.
- _archived_native_access_flags = nullptr;
- }
-}
-
-void Modules::dump_addmods_names() {
- ResourceMark rm;
- const char* addmods_names = get_addmods_names_as_sorted_string();
- if (addmods_names != nullptr) {
- _archived_addmods_names = ArchiveBuilder::current()->ro_strdup(addmods_names);
- }
-}
-
-// Caller needs ResourceMark
-const char* Modules::get_addmods_names_as_sorted_string() {
- return get_numbered_property_as_sorted_string("jdk.module.addmods");
-}
-
-void Modules::serialize_addmods_names(SerializeClosure* soc) {
- soc->do_ptr(&_archived_addmods_names);
- if (soc->reading()) {
- ResourceMark rm;
- check_archived_flag_consistency(_archived_addmods_names, get_addmods_names_as_sorted_string(), "jdk.module.addmods");
-
- // Don't hold onto the pointer, in case we might decide to unmap the archive.
- _archived_addmods_names = nullptr;
- }
-}
-
-// Caller needs ResourceMark
-const char* Modules::get_numbered_property_as_sorted_string(const char* property) {
+const char* Modules::ArchivedProperty::get_numbered_property_as_sorted_string() const {
+ assert(_numbered, "sanity");
// theoretical string size limit for decimal int, but the following loop will end much sooner due to
// OS command-line size limit.
const int max_digits = 10;
const int extra_symbols_count = 2; // includes '.', '\0'
- size_t prop_len = strlen(property) + max_digits + extra_symbols_count;
+ size_t prop_len = strlen(_prop) + max_digits + extra_symbols_count;
char* prop_name = resource_allocate_bytes(prop_len);
GrowableArray list;
for (unsigned int i = 0;; i++) {
- jio_snprintf(prop_name, prop_len, "%s.%d", property, i);
+ jio_snprintf(prop_name, prop_len, "%s.%d", _prop, i);
const char* prop_value = Arguments::get_property(prop_name);
if (prop_value == nullptr) {
break;
@@ -713,6 +684,22 @@ const char* Modules::get_numbered_property_as_sorted_string(const char* property
return (st.size() > 0) ? st.as_string() : nullptr; // Example: "java.base,java.compiler"
}
+void Modules::dump_archived_module_info() {
+ for (size_t i = 0; i < num_archived_props(); i++) {
+ archived_prop(i).dump();
+ }
+}
+
+void Modules::serialize_archived_module_info(SerializeClosure* soc) {
+ for (size_t i = 0; i < num_archived_props(); i++) {
+ archived_prop(i).serialize(soc);
+ }
+ if (soc->reading()) {
+ log_info(cds)("optimized module handling: %s", CDSConfig::is_using_optimized_module_handling() ? "enabled" : "disabled");
+ log_info(cds)("full module graph: %s", CDSConfig::is_using_full_module_graph() ? "enabled" : "disabled");
+ }
+}
+
void Modules::define_archived_modules(Handle h_platform_loader, Handle h_system_loader, TRAPS) {
assert(CDSConfig::is_using_full_module_graph(), "must be");
@@ -934,7 +921,6 @@ jobject Modules::get_module(jclass clazz, TRAPS) {
LogTarget(Debug,module) lt;
if (lt.is_enabled()) {
- ResourceMark rm(THREAD);
LogStream ls(lt);
Klass* klass = java_lang_Class::as_Klass(mirror);
oop module_name = java_lang_Module::name(module);
@@ -945,6 +931,7 @@ jobject Modules::get_module(jclass clazz, TRAPS) {
ls.print("get_module(): Unnamed Module");
}
if (klass != nullptr) {
+ ResourceMark rm(THREAD);
ls.print_cr(" for class %s", klass->external_name());
} else {
ls.print_cr(" for primitive class");
diff --git a/src/hotspot/share/classfile/modules.hpp b/src/hotspot/share/classfile/modules.hpp
index 2c23ab5f6ea..33a5ec77ccb 100644
--- a/src/hotspot/share/classfile/modules.hpp
+++ b/src/hotspot/share/classfile/modules.hpp
@@ -59,23 +59,15 @@ public:
static void verify_archived_modules() NOT_CDS_JAVA_HEAP_RETURN;
static void dump_archived_module_info() NOT_CDS_JAVA_HEAP_RETURN;
static void serialize_archived_module_info(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
- static void dump_main_module_name() NOT_CDS_JAVA_HEAP_RETURN;
- static void serialize(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
- static void check_archived_flag_consistency(char* archived_flag, const char* runtime_flag, const char* property) NOT_CDS_JAVA_HEAP_RETURN;
- static void dump_native_access_flag() NOT_CDS_JAVA_HEAP_RETURN;
- static const char* get_native_access_flags_as_sorted_string() NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
- static void serialize_native_access_flags(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
-
- static void dump_addmods_names() NOT_CDS_JAVA_HEAP_RETURN;
- static const char* get_addmods_names_as_sorted_string() NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
- static void serialize_addmods_names(SerializeClosure* soc) NOT_CDS_JAVA_HEAP_RETURN;
-
- static const char* get_numbered_property_as_sorted_string(const char* property) NOT_CDS_JAVA_HEAP_RETURN_(nullptr);
#if INCLUDE_CDS_JAVA_HEAP
- static char* _archived_main_module_name;
- static char* _archived_addmods_names;
- static char* _archived_native_access_flags;
+private:
+ class ArchivedProperty;
+
+ static ArchivedProperty _archived_props[];
+ static constexpr size_t num_archived_props();
+ static ArchivedProperty& archived_prop(size_t i);
+public:
#endif
// Provides the java.lang.Module for the unnamed module defined
diff --git a/src/hotspot/share/classfile/stringTable.cpp b/src/hotspot/share/classfile/stringTable.cpp
index 22ccb4e4b04..1076ffb2dd7 100644
--- a/src/hotspot/share/classfile/stringTable.cpp
+++ b/src/hotspot/share/classfile/stringTable.cpp
@@ -744,13 +744,29 @@ struct SizeFunc : StackObj {
TableStatistics StringTable::get_table_statistics() {
static TableStatistics ts;
SizeFunc sz;
- ts = _local_table->statistics_get(Thread::current(), sz, ts);
+
+ Thread* jt = Thread::current();
+ StringTableHash::StatisticsTask sts(_local_table);
+ if (!sts.prepare(jt)) {
+ return ts; // return old table statistics
+ }
+ {
+ TraceTime timer("GetStatistics", TRACETIME_LOG(Debug, stringtable, perf));
+ while (sts.do_task(jt, sz)) {
+ sts.pause(jt);
+ if (jt->is_Java_thread()) {
+ ThreadBlockInVM tbivm(JavaThread::cast(jt));
+ }
+ sts.cont(jt);
+ }
+ }
+ ts = sts.done(jt);
return ts;
}
void StringTable::print_table_statistics(outputStream* st) {
- SizeFunc sz;
- _local_table->statistics_to(Thread::current(), sz, st, "StringTable");
+ TableStatistics ts = get_table_statistics();
+ ts.print(st, "StringTable");
#if INCLUDE_CDS_JAVA_HEAP
if (!_shared_table.empty()) {
_shared_table.print_table_statistics(st, "Shared String Table");
diff --git a/src/hotspot/share/classfile/stringTable.hpp b/src/hotspot/share/classfile/stringTable.hpp
index 0d58e09f208..15539009f66 100644
--- a/src/hotspot/share/classfile/stringTable.hpp
+++ b/src/hotspot/share/classfile/stringTable.hpp
@@ -40,7 +40,6 @@ class SerializeClosure;
class StringTableConfig;
class StringTable : AllStatic {
- friend class VMStructs;
friend class StringTableConfig;
static volatile bool _has_work;
diff --git a/src/hotspot/share/classfile/symbolTable.cpp b/src/hotspot/share/classfile/symbolTable.cpp
index 84c0a5a39de..1fd28a1d4e9 100644
--- a/src/hotspot/share/classfile/symbolTable.cpp
+++ b/src/hotspot/share/classfile/symbolTable.cpp
@@ -67,12 +67,7 @@ inline bool symbol_equals_compact_hashtable_entry(Symbol* value, const char* key
static OffsetCompactHashtable<
const char*, Symbol*,
symbol_equals_compact_hashtable_entry
-> _shared_table;
-
-static OffsetCompactHashtable<
- const char*, Symbol*,
- symbol_equals_compact_hashtable_entry
-> _dynamic_shared_table;
+> _shared_table, _dynamic_shared_table, _shared_table_for_dumping;
// --------------------------------------------------------------------------
@@ -572,23 +567,35 @@ Symbol* SymbolTable::new_permanent_symbol(const char* name) {
return sym;
}
-struct SizeFunc : StackObj {
- size_t operator()(Symbol* value) {
+TableStatistics SymbolTable::get_table_statistics() {
+ static TableStatistics ts;
+ auto sz = [&](Symbol* value) {
assert(value != nullptr, "expected valid value");
return (value)->size() * HeapWordSize;
};
+
+ Thread* jt = Thread::current();
+ SymbolTableHash::StatisticsTask sts(_local_table);
+ if (!sts.prepare(jt)) {
+ return ts; // return old table statistics
+ }
+ {
+ TraceTime timer("GetStatistics", TRACETIME_LOG(Debug, symboltable, perf));
+ while (sts.do_task(jt, sz)) {
+ sts.pause(jt);
+ if (jt->is_Java_thread()) {
+ ThreadBlockInVM tbivm(JavaThread::cast(jt));
+ }
+ sts.cont(jt);
+ }
+ }
+ ts = sts.done(jt);
+ return ts;
};
-TableStatistics SymbolTable::get_table_statistics() {
- static TableStatistics ts;
- SizeFunc sz;
- ts = _local_table->statistics_get(Thread::current(), sz, ts);
- return ts;
-}
-
void SymbolTable::print_table_statistics(outputStream* st) {
- SizeFunc sz;
- _local_table->statistics_to(Thread::current(), sz, st, "SymbolTable");
+ TableStatistics ts = get_table_statistics();
+ ts.print(st, "SymbolTable");
if (!_shared_table.empty()) {
_shared_table.print_table_statistics(st, "Shared Symbol Table");
@@ -693,39 +700,27 @@ void SymbolTable::copy_shared_symbol_table(GrowableArray* symbols,
}
}
-size_t SymbolTable::estimate_size_for_archive() {
- if (_items_count > (size_t)max_jint) {
- fatal("Too many symbols to be archived: %zu", _items_count);
- }
- return CompactHashtableWriter::estimate_size(int(_items_count));
-}
-
void SymbolTable::write_to_archive(GrowableArray* symbols) {
CompactHashtableWriter writer(int(_items_count), ArchiveBuilder::symbol_stats());
copy_shared_symbol_table(symbols, &writer);
- if (CDSConfig::is_dumping_static_archive()) {
- _shared_table.reset();
- writer.dump(&_shared_table, "symbol");
- } else {
- assert(CDSConfig::is_dumping_dynamic_archive(), "must be");
- _dynamic_shared_table.reset();
- writer.dump(&_dynamic_shared_table, "symbol");
- }
+ _shared_table_for_dumping.reset();
+ writer.dump(&_shared_table_for_dumping, "symbol");
}
void SymbolTable::serialize_shared_table_header(SerializeClosure* soc,
bool is_static_archive) {
OffsetCompactHashtable * table;
- if (is_static_archive) {
- table = &_shared_table;
+ if (soc->reading()) {
+ if (is_static_archive) {
+ table = &_shared_table;
+ } else {
+ table = &_dynamic_shared_table;
+ }
} else {
- table = &_dynamic_shared_table;
+ table = &_shared_table_for_dumping;
}
+
table->serialize_header(soc);
- if (soc->writing()) {
- // Sanity. Make sure we don't use the shared table at dump time
- table->reset();
- }
}
#endif //INCLUDE_CDS
diff --git a/src/hotspot/share/classfile/symbolTable.hpp b/src/hotspot/share/classfile/symbolTable.hpp
index a1e585e97de..cc861812dc3 100644
--- a/src/hotspot/share/classfile/symbolTable.hpp
+++ b/src/hotspot/share/classfile/symbolTable.hpp
@@ -1,5 +1,5 @@
/*
- * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2025, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -47,7 +47,6 @@ class constantPoolHandle;
class SymbolClosure;
class SymbolTable : public AllStatic {
- friend class VMStructs;
friend class Symbol;
friend class ClassFileParser;
friend class SymbolTableConfig;
@@ -161,7 +160,6 @@ private:
static void copy_shared_symbol_table(GrowableArray* symbols,
CompactHashtableWriter* ch_table);
public:
- static size_t estimate_size_for_archive() NOT_CDS_RETURN_(0);
static void write_to_archive(GrowableArray* symbols) NOT_CDS_RETURN;
static void serialize_shared_table_header(SerializeClosure* soc,
bool is_static_archive = true) NOT_CDS_RETURN;
diff --git a/src/hotspot/share/classfile/systemDictionary.cpp b/src/hotspot/share/classfile/systemDictionary.cpp
index 68f9556099f..0ef4f0dcd18 100644
--- a/src/hotspot/share/classfile/systemDictionary.cpp
+++ b/src/hotspot/share/classfile/systemDictionary.cpp
@@ -1073,38 +1073,6 @@ bool SystemDictionary::check_shared_class_super_types(InstanceKlass* ik, Handle
return true;
}
-InstanceKlass* SystemDictionary::load_shared_lambda_proxy_class(InstanceKlass* ik,
- Handle class_loader,
- Handle protection_domain,
- PackageEntry* pkg_entry,
- TRAPS) {
- InstanceKlass* shared_nest_host = SystemDictionaryShared::get_shared_nest_host(ik);
- assert(shared_nest_host->is_shared(), "nest host must be in CDS archive");
- Symbol* cn = shared_nest_host->name();
- Klass *s = resolve_or_fail(cn, class_loader, true, CHECK_NULL);
- if (s != shared_nest_host) {
- // The dynamically resolved nest_host is not the same as the one we used during dump time,
- // so we cannot use ik.
- return nullptr;
- } else {
- assert(s->is_shared(), "must be");
- }
-
- InstanceKlass* loaded_ik = load_shared_class(ik, class_loader, protection_domain, nullptr, pkg_entry, CHECK_NULL);
-
- if (loaded_ik != nullptr) {
- assert(shared_nest_host->is_same_class_package(ik),
- "lambda proxy class and its nest host must be in the same package");
- // The lambda proxy class and its nest host have the same class loader and class loader data,
- // as verified in SystemDictionaryShared::add_lambda_proxy_class()
- assert(shared_nest_host->class_loader() == class_loader(), "mismatched class loader");
- assert(shared_nest_host->class_loader_data() == class_loader_data(class_loader), "mismatched class loader data");
- ik->set_nest_host(shared_nest_host);
- }
-
- return loaded_ik;
-}
-
InstanceKlass* SystemDictionary::load_shared_class(InstanceKlass* ik,
Handle class_loader,
Handle protection_domain,
@@ -1112,6 +1080,7 @@ InstanceKlass* SystemDictionary::load_shared_class(InstanceKlass* ik,
PackageEntry* pkg_entry,
TRAPS) {
assert(ik != nullptr, "sanity");
+ assert(ik->is_shared(), "sanity");
assert(!ik->is_unshareable_info_restored(), "shared class can be restored only once");
assert(Atomic::add(&ik->_shared_class_load_count, 1) == 1, "shared class loaded more than once");
Symbol* class_name = ik->name();
@@ -1130,7 +1099,7 @@ InstanceKlass* SystemDictionary::load_shared_class(InstanceKlass* ik,
InstanceKlass* new_ik = nullptr;
// CFLH check is skipped for VM hidden classes (see KlassFactory::create_from_stream).
// It will be skipped for shared VM hidden lambda proxy classes.
- if (!SystemDictionaryShared::is_hidden_lambda_proxy(ik)) {
+ if (!ik->is_hidden()) {
new_ik = KlassFactory::check_shared_class_file_load_hook(
ik, class_name, class_loader, protection_domain, cfs, CHECK_NULL);
}
@@ -1177,6 +1146,10 @@ void SystemDictionary::load_shared_class_misc(InstanceKlass* ik, ClassLoaderData
// notify a class loaded from shared object
ClassLoadingService::notify_class_loaded(ik, true /* shared class */);
+
+ if (CDSConfig::is_dumping_final_static_archive()) {
+ SystemDictionaryShared::init_dumptime_info_from_preimage(ik);
+ }
}
#endif // INCLUDE_CDS
diff --git a/src/hotspot/share/classfile/systemDictionary.hpp b/src/hotspot/share/classfile/systemDictionary.hpp
index 7432166ba4e..8cf2cd83b82 100644
--- a/src/hotspot/share/classfile/systemDictionary.hpp
+++ b/src/hotspot/share/classfile/systemDictionary.hpp
@@ -78,8 +78,8 @@ template class GrowableArray;
class SystemDictionary : AllStatic {
friend class AOTLinkedClassBulkLoader;
friend class BootstrapInfo;
+ friend class LambdaProxyClassDictionary;
friend class vmClasses;
- friend class VMStructs;
public:
@@ -326,16 +326,11 @@ private:
static void restore_archived_method_handle_intrinsics_impl(TRAPS) NOT_CDS_RETURN;
protected:
- // Used by SystemDictionaryShared
+ // Used by SystemDictionaryShared and LambdaProxyClassDictionary
static bool add_loader_constraint(Symbol* name, Klass* klass_being_linked, Handle loader1,
Handle loader2);
static void post_class_load_event(EventClassLoad* event, const InstanceKlass* k, const ClassLoaderData* init_cld);
- static InstanceKlass* load_shared_lambda_proxy_class(InstanceKlass* ik,
- Handle class_loader,
- Handle protection_domain,
- PackageEntry* pkg_entry,
- TRAPS);
static InstanceKlass* load_shared_class(InstanceKlass* ik,
Handle class_loader,
Handle protection_domain,
@@ -346,6 +341,7 @@ protected:
static InstanceKlass* find_or_define_instance_class(Symbol* class_name,
Handle class_loader,
InstanceKlass* k, TRAPS);
+
public:
static bool is_system_class_loader(oop class_loader);
static bool is_platform_class_loader(oop class_loader);
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.cpp b/src/hotspot/share/classfile/systemDictionaryShared.cpp
index b8e24bb2caa..2fbee502b93 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.cpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.cpp
@@ -22,8 +22,9 @@
*
*/
+#include "cds/aotClassFilter.hpp"
+#include "cds/aotClassLocation.hpp"
#include "cds/archiveBuilder.hpp"
-#include "cds/archiveHeapLoader.hpp"
#include "cds/archiveUtils.hpp"
#include "cds/cdsConfig.hpp"
#include "cds/cdsProtectionDomain.hpp"
@@ -33,6 +34,8 @@
#include "cds/dynamicArchive.hpp"
#include "cds/filemap.hpp"
#include "cds/heapShared.hpp"
+#include "cds/lambdaProxyClassDictionary.hpp"
+#include "cds/lambdaFormInvokers.inline.hpp"
#include "cds/metaspaceShared.hpp"
#include "cds/runTimeClassInfo.hpp"
#include "cds/unregisteredClasses.hpp"
@@ -50,7 +53,6 @@
#include "classfile/verificationType.hpp"
#include "classfile/vmClasses.hpp"
#include "classfile/vmSymbols.hpp"
-#include "interpreter/bootstrapInfo.hpp"
#include "jfr/jfrEvents.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
@@ -80,7 +82,6 @@ SystemDictionaryShared::ArchiveInfo SystemDictionaryShared::_static_archive;
SystemDictionaryShared::ArchiveInfo SystemDictionaryShared::_dynamic_archive;
DumpTimeSharedClassTable* SystemDictionaryShared::_dumptime_table = nullptr;
-DumpTimeLambdaProxyClassDictionary* SystemDictionaryShared::_dumptime_lambda_proxy_class_dictionary = nullptr;
// Used by NoClassLoadingMark
DEBUG_ONLY(bool SystemDictionaryShared::_class_loading_may_happen = true;)
@@ -193,23 +194,20 @@ InstanceKlass* SystemDictionaryShared::acquire_class_for_current_thread(
// k must not be a shared class.
DumpTimeClassInfo* SystemDictionaryShared::get_info(InstanceKlass* k) {
MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
- assert(!k->is_shared(), "sanity");
return get_info_locked(k);
}
DumpTimeClassInfo* SystemDictionaryShared::get_info_locked(InstanceKlass* k) {
assert_lock_strong(DumpTimeTable_lock);
- assert(!k->is_shared(), "sanity");
DumpTimeClassInfo* info = _dumptime_table->get_info(k);
assert(info != nullptr, "must be");
return info;
}
bool SystemDictionaryShared::check_for_exclusion(InstanceKlass* k, DumpTimeClassInfo* info) {
- if (MetaspaceShared::is_in_shared_metaspace(k)) {
+ if (CDSConfig::is_dumping_dynamic_archive() && MetaspaceShared::is_in_shared_metaspace(k)) {
// We have reached a super type that's already in the base archive. Treat it
// as "not excluded".
- assert(CDSConfig::is_dumping_dynamic_archive(), "must be");
return false;
}
@@ -245,38 +243,17 @@ bool SystemDictionaryShared::is_jfr_event_class(InstanceKlass *k) {
return false;
}
-bool SystemDictionaryShared::is_registered_lambda_proxy_class(InstanceKlass* ik) {
- DumpTimeClassInfo* info = _dumptime_table->get(ik);
- bool result = (info != nullptr) ? info->_is_registered_lambda_proxy : false;
- if (result) {
- assert(!CDSConfig::is_dumping_invokedynamic(), "only used in legacy lambda proxy support");
- }
- return result;
-}
-
-void SystemDictionaryShared::reset_registered_lambda_proxy_class(InstanceKlass* ik) {
- DumpTimeClassInfo* info = _dumptime_table->get(ik);
- if (info != nullptr) {
- info->_is_registered_lambda_proxy = false;
- info->set_excluded();
- }
-}
-
bool SystemDictionaryShared::is_early_klass(InstanceKlass* ik) {
DumpTimeClassInfo* info = _dumptime_table->get(ik);
return (info != nullptr) ? info->is_early_klass() : false;
}
-bool SystemDictionaryShared::is_hidden_lambda_proxy(InstanceKlass* ik) {
- assert(ik->is_shared(), "applicable to only a shared class");
- if (ik->is_hidden()) {
- return true;
- } else {
- return false;
- }
-}
-
bool SystemDictionaryShared::check_for_exclusion_impl(InstanceKlass* k) {
+ if (CDSConfig::is_dumping_final_static_archive() && k->is_shared_unregistered_class()
+ && k->is_shared()) {
+ return false; // Do not exclude: unregistered classes are passed from preimage to final image.
+ }
+
if (k->is_in_error_state()) {
return warn_excluded(k, "In error state");
}
@@ -292,7 +269,7 @@ bool SystemDictionaryShared::check_for_exclusion_impl(InstanceKlass* k) {
if (!k->is_hidden() && k->shared_classpath_index() < 0 && is_builtin(k)) {
if (k->name()->starts_with("java/lang/invoke/BoundMethodHandle$Species_")) {
// This class is dynamically generated by the JDK
- if (CDSConfig::is_dumping_aot_linked_classes()) {
+ if (CDSConfig::is_dumping_method_handles()) {
k->set_shared_classpath_index(0);
} else {
ResourceMark rm;
@@ -329,6 +306,10 @@ bool SystemDictionaryShared::check_for_exclusion_impl(InstanceKlass* k) {
// class may fail to verify in AOTLinkedClassBulkLoader::init_required_classes_for_loader(),
// causing the JVM to fail at bootstrap.
return warn_excluded(k, "Unlinked class not supported by AOTClassLinking");
+ } else if (CDSConfig::is_dumping_preimage_static_archive()) {
+ // When dumping the final static archive, we will unconditionally load and link all
+ // classes from tje preimage. We don't want to get a VerifyError when linking this class.
+ return warn_excluded(k, "Unlinked class not supported by AOTConfiguration");
}
} else {
if (!k->can_be_verified_at_dumptime()) {
@@ -495,15 +476,20 @@ void SystemDictionaryShared::set_shared_class_misc_info(InstanceKlass* k, ClassF
void SystemDictionaryShared::initialize() {
if (CDSConfig::is_dumping_archive()) {
_dumptime_table = new (mtClass) DumpTimeSharedClassTable;
- _dumptime_lambda_proxy_class_dictionary =
- new (mtClass) DumpTimeLambdaProxyClassDictionary;
+ LambdaProxyClassDictionary::dumptime_init();
+ if (CDSConfig::is_dumping_heap()) {
+ HeapShared::init_dumping();
+ }
}
}
void SystemDictionaryShared::init_dumptime_info(InstanceKlass* k) {
MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
assert(SystemDictionaryShared::class_loading_may_happen(), "sanity");
- _dumptime_table->allocate_info(k);
+ DumpTimeClassInfo* info = _dumptime_table->allocate_info(k);
+ if (AOTClassFilter::is_aot_tooling_class(k)) {
+ info->set_is_aot_tooling_class();
+ }
}
void SystemDictionaryShared::remove_dumptime_info(InstanceKlass* k) {
@@ -537,6 +523,18 @@ void SystemDictionaryShared::handle_class_unloading(InstanceKlass* klass) {
}
}
+void SystemDictionaryShared::init_dumptime_info_from_preimage(InstanceKlass* k) {
+ init_dumptime_info(k);
+ copy_verification_constraints_from_preimage(k);
+ copy_linking_constraints_from_preimage(k);
+
+ if (SystemDictionary::is_platform_class_loader(k->class_loader())) {
+ AOTClassLocationConfig::dumptime_set_has_platform_classes();
+ } else if (SystemDictionary::is_system_class_loader(k->class_loader())) {
+ AOTClassLocationConfig::dumptime_set_has_app_classes();
+ }
+}
+
// Check if a class or any of its supertypes has been redefined.
bool SystemDictionaryShared::has_been_redefined(InstanceKlass* k) {
if (k->has_been_redefined()) {
@@ -565,8 +563,8 @@ void SystemDictionaryShared::validate_before_archiving(InstanceKlass* k) {
guarantee(!info->is_excluded(), "Should not attempt to archive excluded class %s", name);
if (is_builtin(k)) {
if (k->is_hidden()) {
- if (!CDSConfig::is_dumping_invokedynamic()) {
- assert(is_registered_lambda_proxy_class(k), "unexpected hidden class %s", name);
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ assert(LambdaProxyClassDictionary::is_registered_lambda_proxy_class(k), "unexpected hidden class %s", name);
}
}
guarantee(!k->is_shared_unregistered_class(),
@@ -683,7 +681,9 @@ void SystemDictionaryShared::finish_exclusion_checks() {
});
_dumptime_table->update_counts();
- cleanup_lambda_proxy_class_dictionary();
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ LambdaProxyClassDictionary::cleanup_dumptime_table();
+ }
}
bool SystemDictionaryShared::is_excluded_class(InstanceKlass* k) {
@@ -719,23 +719,22 @@ bool SystemDictionaryShared::has_class_failed_verification(InstanceKlass* ik) {
return (p == nullptr) ? false : p->failed_verification();
}
-void SystemDictionaryShared::dumptime_classes_do(class MetaspaceClosure* it) {
+void SystemDictionaryShared::dumptime_classes_do(MetaspaceClosure* it) {
assert_lock_strong(DumpTimeTable_lock);
auto do_klass = [&] (InstanceKlass* k, DumpTimeClassInfo& info) {
- if (k->is_loader_alive() && !info.is_excluded()) {
+ if (CDSConfig::is_dumping_final_static_archive() && !k->is_loaded()) {
+ assert(k->is_shared_unregistered_class(), "must be");
+ info.metaspace_pointers_do(it);
+ } else if (k->is_loader_alive() && !info.is_excluded()) {
info.metaspace_pointers_do(it);
}
};
_dumptime_table->iterate_all_live_classes(do_klass);
- auto do_lambda = [&] (LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
- if (key.caller_ik()->is_loader_alive()) {
- info.metaspace_pointers_do(it);
- key.metaspace_pointers_do(it);
- }
- };
- _dumptime_lambda_proxy_class_dictionary->iterate_all(do_lambda);
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ LambdaProxyClassDictionary::dumptime_classes_do(it);
+ }
}
bool SystemDictionaryShared::add_verification_constraint(InstanceKlass* k, Symbol* name,
@@ -770,193 +769,9 @@ void SystemDictionaryShared::add_enum_klass_static_field(InstanceKlass* ik, int
info->add_enum_klass_static_field(root_index);
}
-void SystemDictionaryShared::add_to_dump_time_lambda_proxy_class_dictionary(LambdaProxyClassKey& key,
- InstanceKlass* proxy_klass) {
- assert_lock_strong(DumpTimeTable_lock);
-
- bool created;
- DumpTimeLambdaProxyClassInfo* info = _dumptime_lambda_proxy_class_dictionary->put_if_absent(key, &created);
- info->add_proxy_klass(proxy_klass);
- if (created) {
- ++_dumptime_lambda_proxy_class_dictionary->_count;
- }
-}
-
-void SystemDictionaryShared::add_lambda_proxy_class(InstanceKlass* caller_ik,
- InstanceKlass* lambda_ik,
- Symbol* invoked_name,
- Symbol* invoked_type,
- Symbol* method_type,
- Method* member_method,
- Symbol* instantiated_method_type,
- TRAPS) {
- if (CDSConfig::is_dumping_invokedynamic()) {
- // The lambda proxy classes will be stored as part of aot-resolved constant pool entries.
- // There's no need to remember them in a separate table.
- return;
- }
-
- assert(caller_ik->class_loader() == lambda_ik->class_loader(), "mismatched class loader");
- assert(caller_ik->class_loader_data() == lambda_ik->class_loader_data(), "mismatched class loader data");
- assert(java_lang_Class::class_data(lambda_ik->java_mirror()) == nullptr, "must not have class data");
-
- MutexLocker ml(DumpTimeTable_lock, Mutex::_no_safepoint_check_flag);
-
- lambda_ik->assign_class_loader_type();
- lambda_ik->set_shared_classpath_index(caller_ik->shared_classpath_index());
- InstanceKlass* nest_host = caller_ik->nest_host(CHECK);
- assert(nest_host != nullptr, "unexpected nullptr nest_host");
-
- DumpTimeClassInfo* info = _dumptime_table->get(lambda_ik);
- if (info != nullptr && !lambda_ik->is_non_strong_hidden() && is_builtin(lambda_ik) && is_builtin(caller_ik)
- // Don't include the lambda proxy if its nest host is not in the "linked" state.
- && nest_host->is_linked()) {
- // Set _is_registered_lambda_proxy in DumpTimeClassInfo so that the lambda_ik
- // won't be excluded during dumping of shared archive.
- info->_is_registered_lambda_proxy = true;
- info->set_nest_host(nest_host);
-
- LambdaProxyClassKey key(caller_ik,
- invoked_name,
- invoked_type,
- method_type,
- member_method,
- instantiated_method_type);
- add_to_dump_time_lambda_proxy_class_dictionary(key, lambda_ik);
- }
-}
-
-InstanceKlass* SystemDictionaryShared::get_shared_lambda_proxy_class(InstanceKlass* caller_ik,
- Symbol* invoked_name,
- Symbol* invoked_type,
- Symbol* method_type,
- Method* member_method,
- Symbol* instantiated_method_type) {
- assert(caller_ik != nullptr, "sanity");
- assert(invoked_name != nullptr, "sanity");
- assert(invoked_type != nullptr, "sanity");
- assert(method_type != nullptr, "sanity");
- assert(instantiated_method_type != nullptr, "sanity");
-
- if (!caller_ik->is_shared() ||
- !invoked_name->is_shared() ||
- !invoked_type->is_shared() ||
- !method_type->is_shared() ||
- (member_method != nullptr && !member_method->is_shared()) ||
- !instantiated_method_type->is_shared()) {
- // These can't be represented as u4 offset, but we wouldn't have archived a lambda proxy in this case anyway.
- return nullptr;
- }
-
- MutexLocker ml(CDSLambda_lock, Mutex::_no_safepoint_check_flag);
- RunTimeLambdaProxyClassKey key =
- RunTimeLambdaProxyClassKey::init_for_runtime(caller_ik, invoked_name, invoked_type,
- method_type, member_method, instantiated_method_type);
-
- // Try to retrieve the lambda proxy class from static archive.
- const RunTimeLambdaProxyClassInfo* info = _static_archive.lookup_lambda_proxy_class(&key);
- InstanceKlass* proxy_klass = retrieve_lambda_proxy_class(info);
- if (proxy_klass == nullptr) {
- if (info != nullptr && log_is_enabled(Debug, cds)) {
- ResourceMark rm;
- log_debug(cds)("Used all static archived lambda proxy classes for: %s %s%s",
- caller_ik->external_name(), invoked_name->as_C_string(), invoked_type->as_C_string());
- }
- } else {
- return proxy_klass;
- }
-
- // Retrieving from static archive is unsuccessful, try dynamic archive.
- info = _dynamic_archive.lookup_lambda_proxy_class(&key);
- proxy_klass = retrieve_lambda_proxy_class(info);
- if (proxy_klass == nullptr) {
- if (info != nullptr && log_is_enabled(Debug, cds)) {
- ResourceMark rm;
- log_debug(cds)("Used all dynamic archived lambda proxy classes for: %s %s%s",
- caller_ik->external_name(), invoked_name->as_C_string(), invoked_type->as_C_string());
- }
- }
- return proxy_klass;
-}
-
-InstanceKlass* SystemDictionaryShared::retrieve_lambda_proxy_class(const RunTimeLambdaProxyClassInfo* info) {
- InstanceKlass* proxy_klass = nullptr;
- if (info != nullptr) {
- InstanceKlass* curr_klass = info->proxy_klass_head();
- InstanceKlass* prev_klass = curr_klass;
- if (curr_klass->lambda_proxy_is_available()) {
- while (curr_klass->next_link() != nullptr) {
- prev_klass = curr_klass;
- curr_klass = InstanceKlass::cast(curr_klass->next_link());
- }
- assert(curr_klass->is_hidden(), "must be");
- assert(curr_klass->lambda_proxy_is_available(), "must be");
-
- prev_klass->set_next_link(nullptr);
- proxy_klass = curr_klass;
- proxy_klass->clear_lambda_proxy_is_available();
- if (log_is_enabled(Debug, cds)) {
- ResourceMark rm;
- log_debug(cds)("Loaded lambda proxy: %s ", proxy_klass->external_name());
- }
- }
- }
- return proxy_klass;
-}
-
-InstanceKlass* SystemDictionaryShared::get_shared_nest_host(InstanceKlass* lambda_ik) {
- assert(!CDSConfig::is_dumping_static_archive() && CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
- RunTimeClassInfo* record = RunTimeClassInfo::get_for(lambda_ik);
- return record->nest_host();
-}
-
-InstanceKlass* SystemDictionaryShared::prepare_shared_lambda_proxy_class(InstanceKlass* lambda_ik,
- InstanceKlass* caller_ik, TRAPS) {
- Handle class_loader(THREAD, caller_ik->class_loader());
- Handle protection_domain;
- PackageEntry* pkg_entry = caller_ik->package();
- if (caller_ik->class_loader() != nullptr) {
- protection_domain = CDSProtectionDomain::init_security_info(class_loader, caller_ik, pkg_entry, CHECK_NULL);
- }
-
- InstanceKlass* shared_nest_host = get_shared_nest_host(lambda_ik);
- assert(shared_nest_host != nullptr, "unexpected nullptr _nest_host");
-
- InstanceKlass* loaded_lambda =
- SystemDictionary::load_shared_lambda_proxy_class(lambda_ik, class_loader, protection_domain, pkg_entry, CHECK_NULL);
-
- if (loaded_lambda == nullptr) {
- return nullptr;
- }
-
- // Ensures the nest host is the same as the lambda proxy's
- // nest host recorded at dump time.
- InstanceKlass* nest_host = caller_ik->nest_host(THREAD);
- assert(nest_host == shared_nest_host, "mismatched nest host");
-
- EventClassLoad class_load_start_event;
-
- // Add to class hierarchy, and do possible deoptimizations.
- loaded_lambda->add_to_hierarchy(THREAD);
- // But, do not add to dictionary.
-
- loaded_lambda->link_class(CHECK_NULL);
- // notify jvmti
- if (JvmtiExport::should_post_class_load()) {
- JvmtiExport::post_class_load(THREAD, loaded_lambda);
- }
- if (class_load_start_event.should_commit()) {
- SystemDictionary::post_class_load_event(&class_load_start_event, loaded_lambda, ClassLoaderData::class_loader_data(class_loader()));
- }
-
- loaded_lambda->initialize(CHECK_NULL);
-
- return loaded_lambda;
-}
-
void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass,
TRAPS) {
- assert(!CDSConfig::is_dumping_static_archive() && CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
+ assert(CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
RunTimeClassInfo* record = RunTimeClassInfo::get_for(klass);
int length = record->num_verifier_constraints();
@@ -965,21 +780,16 @@ void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass
RunTimeClassInfo::RTVerifierConstraint* vc = record->verifier_constraint_at(i);
Symbol* name = vc->name();
Symbol* from_name = vc->from_name();
- char c = record->verifier_constraint_flag(i);
if (log_is_enabled(Trace, cds, verification)) {
ResourceMark rm(THREAD);
log_trace(cds, verification)("check_verification_constraint: %s: %s must be subclass of %s [0x%x]",
klass->external_name(), from_name->as_klass_external_name(),
- name->as_klass_external_name(), c);
+ name->as_klass_external_name(), record->verifier_constraint_flag(i));
}
- bool from_field_is_protected = (c & SystemDictionaryShared::FROM_FIELD_IS_PROTECTED) ? true : false;
- bool from_is_array = (c & SystemDictionaryShared::FROM_IS_ARRAY) ? true : false;
- bool from_is_object = (c & SystemDictionaryShared::FROM_IS_OBJECT) ? true : false;
-
- bool ok = VerificationType::resolve_and_check_assignability(klass, name,
- from_name, from_field_is_protected, from_is_array, from_is_object, CHECK);
+ bool ok = VerificationType::resolve_and_check_assignability(klass, name, from_name,
+ record->from_field_is_protected(i), record->from_is_array(i), record->from_is_object(i), CHECK);
if (!ok) {
ResourceMark rm(THREAD);
stringStream ss;
@@ -995,6 +805,24 @@ void SystemDictionaryShared::check_verification_constraints(InstanceKlass* klass
}
}
+void SystemDictionaryShared::copy_verification_constraints_from_preimage(InstanceKlass* klass) {
+ assert(CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
+ DumpTimeClassInfo* dt_info = get_info(klass);
+ RunTimeClassInfo* rt_info = RunTimeClassInfo::get_for(klass); // from preimage
+
+ int length = rt_info->num_verifier_constraints();
+ if (length > 0) {
+ for (int i = 0; i < length; i++) {
+ RunTimeClassInfo::RTVerifierConstraint* vc = rt_info->verifier_constraint_at(i);
+ Symbol* name = vc->name();
+ Symbol* from_name = vc->from_name();
+
+ dt_info->add_verification_constraint(klass, name, from_name,
+ rt_info->from_field_is_protected(i), rt_info->from_is_array(i), rt_info->from_is_object(i));
+ }
+ }
+}
+
static oop get_class_loader_by(char type) {
if (type == (char)ClassLoader::BOOT_LOADER) {
return (oop)nullptr;
@@ -1066,7 +894,7 @@ void SystemDictionaryShared::record_linking_constraint(Symbol* name, InstanceKla
// returns true IFF there's no need to re-initialize the i/v-tables for klass for
// the purpose of checking class loader constraints.
bool SystemDictionaryShared::check_linking_constraints(Thread* current, InstanceKlass* klass) {
- assert(!CDSConfig::is_dumping_static_archive() && CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
+ assert(CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
LogTarget(Info, class, loader, constraints) log;
if (klass->is_shared_boot_class()) {
// No class loader constraint check performed for boot classes.
@@ -1112,90 +940,22 @@ bool SystemDictionaryShared::check_linking_constraints(Thread* current, Instance
return false;
}
-bool SystemDictionaryShared::is_supported_invokedynamic(BootstrapInfo* bsi) {
- LogTarget(Debug, cds, lambda) log;
- if (bsi->arg_values() == nullptr || !bsi->arg_values()->is_objArray()) {
- if (log.is_enabled()) {
- LogStream log_stream(log);
- log.print("bsi check failed");
- log.print(" bsi->arg_values().not_null() %d", bsi->arg_values().not_null());
- if (bsi->arg_values().not_null()) {
- log.print(" bsi->arg_values()->is_objArray() %d", bsi->arg_values()->is_objArray());
- bsi->print_msg_on(&log_stream);
+void SystemDictionaryShared::copy_linking_constraints_from_preimage(InstanceKlass* klass) {
+ assert(CDSConfig::is_using_archive(), "called at run time with CDS enabled only");
+ JavaThread* current = JavaThread::current();
+ if (klass->is_shared_platform_class() || klass->is_shared_app_class()) {
+ RunTimeClassInfo* rt_info = RunTimeClassInfo::get_for(klass); // from preimage
+
+ if (rt_info->num_loader_constraints() > 0) {
+ for (int i = 0; i < rt_info->num_loader_constraints(); i++) {
+ RunTimeClassInfo::RTLoaderConstraint* lc = rt_info->loader_constraint_at(i);
+ Symbol* name = lc->constraint_name();
+ Handle loader1(current, get_class_loader_by(lc->_loader_type1));
+ Handle loader2(current, get_class_loader_by(lc->_loader_type2));
+ record_linking_constraint(name, klass, loader1, loader2);
}
}
- return false;
}
-
- Handle bsm = bsi->bsm();
- if (bsm.is_null() || !java_lang_invoke_DirectMethodHandle::is_instance(bsm())) {
- if (log.is_enabled()) {
- log.print("bsm check failed");
- log.print(" bsm.is_null() %d", bsm.is_null());
- log.print(" java_lang_invoke_DirectMethodHandle::is_instance(bsm()) %d",
- java_lang_invoke_DirectMethodHandle::is_instance(bsm()));
- }
- return false;
- }
-
- oop mn = java_lang_invoke_DirectMethodHandle::member(bsm());
- Method* method = java_lang_invoke_MemberName::vmtarget(mn);
- if (method->klass_name()->equals("java/lang/invoke/LambdaMetafactory") &&
- method->name()->equals("metafactory") &&
- method->signature()->equals("(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;"
- "Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;"
- "Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;")) {
- return true;
- } else {
- if (log.is_enabled()) {
- ResourceMark rm;
- log.print("method check failed");
- log.print(" klass_name() %s", method->klass_name()->as_C_string());
- log.print(" name() %s", method->name()->as_C_string());
- log.print(" signature() %s", method->signature()->as_C_string());
- }
- }
-
- return false;
-}
-
-class EstimateSizeForArchive : StackObj {
- size_t _shared_class_info_size;
- int _num_builtin_klasses;
- int _num_unregistered_klasses;
-
-public:
- EstimateSizeForArchive() {
- _shared_class_info_size = 0;
- _num_builtin_klasses = 0;
- _num_unregistered_klasses = 0;
- }
-
- void do_entry(InstanceKlass* k, DumpTimeClassInfo& info) {
- if (!info.is_excluded()) {
- size_t byte_size = info.runtime_info_bytesize();
- _shared_class_info_size += align_up(byte_size, SharedSpaceObjectAlignment);
- }
- }
-
- size_t total() {
- return _shared_class_info_size;
- }
-};
-
-size_t SystemDictionaryShared::estimate_size_for_archive() {
- EstimateSizeForArchive est;
- _dumptime_table->iterate_all_live_classes(&est);
- size_t total_size = est.total() +
- CompactHashtableWriter::estimate_size(_dumptime_table->count_of(true)) +
- CompactHashtableWriter::estimate_size(_dumptime_table->count_of(false));
-
- size_t bytesize = align_up(sizeof(RunTimeLambdaProxyClassInfo), SharedSpaceObjectAlignment);
- total_size +=
- (bytesize * _dumptime_lambda_proxy_class_dictionary->_count) +
- CompactHashtableWriter::estimate_size(_dumptime_lambda_proxy_class_dictionary->_count);
-
- return total_size;
}
unsigned int SystemDictionaryShared::hash_for_shared_dictionary(address ptr) {
@@ -1213,51 +973,6 @@ unsigned int SystemDictionaryShared::hash_for_shared_dictionary(address ptr) {
}
}
-class CopyLambdaProxyClassInfoToArchive : StackObj {
- CompactHashtableWriter* _writer;
- ArchiveBuilder* _builder;
-public:
- CopyLambdaProxyClassInfoToArchive(CompactHashtableWriter* writer)
- : _writer(writer), _builder(ArchiveBuilder::current()) {}
- bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
- // In static dump, info._proxy_klasses->at(0) is already relocated to point to the archived class
- // (not the original class).
- ResourceMark rm;
- log_info(cds,dynamic)("Archiving hidden %s", info._proxy_klasses->at(0)->external_name());
- size_t byte_size = sizeof(RunTimeLambdaProxyClassInfo);
- RunTimeLambdaProxyClassInfo* runtime_info =
- (RunTimeLambdaProxyClassInfo*)ArchiveBuilder::ro_region_alloc(byte_size);
- runtime_info->init(key, info);
- unsigned int hash = runtime_info->hash();
- u4 delta = _builder->any_to_offset_u4((void*)runtime_info);
- _writer->add(hash, delta);
- return true;
- }
-};
-
-class AdjustLambdaProxyClassInfo : StackObj {
-public:
- AdjustLambdaProxyClassInfo() {}
- bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
- int len = info._proxy_klasses->length();
- InstanceKlass* last_buff_k = nullptr;
-
- for (int i = len - 1; i >= 0; i--) {
- InstanceKlass* orig_k = info._proxy_klasses->at(i);
- InstanceKlass* buff_k = ArchiveBuilder::current()->get_buffered_addr(orig_k);
- assert(ArchiveBuilder::current()->is_in_buffer_space(buff_k), "must be");
- buff_k->set_lambda_proxy_is_available();
- buff_k->set_next_link(last_buff_k);
- if (last_buff_k != nullptr) {
- ArchivePtrMarker::mark_pointer(buff_k->next_link_addr());
- }
- last_buff_k = buff_k;
- }
-
- return true;
- }
-};
-
class CopySharedClassInfoToArchive : StackObj {
CompactHashtableWriter* _writer;
bool _is_builtin;
@@ -1296,15 +1011,6 @@ public:
}
};
-void SystemDictionaryShared::write_lambda_proxy_class_dictionary(LambdaProxyClassDictionary *dictionary) {
- CompactHashtableStats stats;
- dictionary->reset();
- CompactHashtableWriter writer(_dumptime_lambda_proxy_class_dictionary->_count, &stats);
- CopyLambdaProxyClassInfoToArchive copy(&writer);
- _dumptime_lambda_proxy_class_dictionary->iterate(©);
- writer.dump(dictionary, "lambda proxy class dictionary");
-}
-
void SystemDictionaryShared::write_dictionary(RunTimeSharedDictionary* dictionary,
bool is_builtin) {
CompactHashtableStats stats;
@@ -1321,13 +1027,11 @@ void SystemDictionaryShared::write_to_archive(bool is_static_archive) {
write_dictionary(&archive->_builtin_dictionary, true);
write_dictionary(&archive->_unregistered_dictionary, false);
-
- write_lambda_proxy_class_dictionary(&archive->_lambda_proxy_class_dictionary);
-}
-
-void SystemDictionaryShared::adjust_lambda_proxy_class_dictionary() {
- AdjustLambdaProxyClassInfo adjuster;
- _dumptime_lambda_proxy_class_dictionary->iterate(&adjuster);
+ if (CDSConfig::is_dumping_lambdas_in_legacy_mode()) {
+ LambdaProxyClassDictionary::write_dictionary(is_static_archive);
+ } else {
+ LambdaProxyClassDictionary::reset_dictionary(is_static_archive);
+ }
}
void SystemDictionaryShared::serialize_dictionary_headers(SerializeClosure* soc,
@@ -1336,7 +1040,7 @@ void SystemDictionaryShared::serialize_dictionary_headers(SerializeClosure* soc,
archive->_builtin_dictionary.serialize_header(soc);
archive->_unregistered_dictionary.serialize_header(soc);
- archive->_lambda_proxy_class_dictionary.serialize_header(soc);
+ LambdaProxyClassDictionary::serialize(soc, is_static_archive);
}
void SystemDictionaryShared::serialize_vm_classes(SerializeClosure* soc) {
@@ -1357,10 +1061,7 @@ SystemDictionaryShared::find_record(RunTimeSharedDictionary* static_dict, RunTim
if (DynamicArchive::is_mapped()) {
// Use the regenerated holder classes in the dynamic archive as they
// have more methods than those in the base archive.
- if (name == vmSymbols::java_lang_invoke_Invokers_Holder() ||
- name == vmSymbols::java_lang_invoke_DirectMethodHandle_Holder() ||
- name == vmSymbols::java_lang_invoke_LambdaForm_Holder() ||
- name == vmSymbols::java_lang_invoke_DelegatingMethodHandle_Holder()) {
+ if (LambdaFormInvokers::may_be_regenerated_class(name)) {
record = dynamic_dict->lookup(name, hash, 0);
if (record != nullptr) {
return record;
@@ -1405,7 +1106,7 @@ void SystemDictionaryShared::update_shared_entry(InstanceKlass* k, int id) {
info->_id = id;
}
-static const char* class_loader_name_for_shared(Klass* k) {
+const char* SystemDictionaryShared::loader_type_for_shared_class(Klass* k) {
assert(k != nullptr, "Sanity");
assert(k->is_shared(), "Must be");
assert(k->is_instance_klass(), "Must be");
@@ -1432,7 +1133,7 @@ public:
void do_value(const RunTimeClassInfo* record) {
ResourceMark rm;
_st->print_cr("%4d: %s %s", _index++, record->klass()->external_name(),
- class_loader_name_for_shared(record->klass()));
+ SystemDictionaryShared::loader_type_for_shared_class(record->klass()));
if (record->klass()->array_klasses() != nullptr) {
record->klass()->array_klasses()->cds_print_value_on(_st);
_st->cr();
@@ -1441,55 +1142,34 @@ public:
int index() const { return _index; }
};
-class SharedLambdaDictionaryPrinter : StackObj {
- outputStream* _st;
- int _index;
-public:
- SharedLambdaDictionaryPrinter(outputStream* st, int idx) : _st(st), _index(idx) {}
-
- void do_value(const RunTimeLambdaProxyClassInfo* record) {
- if (record->proxy_klass_head()->lambda_proxy_is_available()) {
- ResourceMark rm;
- Klass* k = record->proxy_klass_head();
- while (k != nullptr) {
- _st->print_cr("%4d: %s %s", _index++, k->external_name(),
- class_loader_name_for_shared(k));
- k = k->next_link();
- }
- }
- }
-};
-
void SystemDictionaryShared::ArchiveInfo::print_on(const char* prefix,
- outputStream* st) {
+ outputStream* st,
+ bool is_static_archive) {
st->print_cr("%sShared Dictionary", prefix);
SharedDictionaryPrinter p(st);
st->print_cr("%sShared Builtin Dictionary", prefix);
_builtin_dictionary.iterate(&p);
st->print_cr("%sShared Unregistered Dictionary", prefix);
_unregistered_dictionary.iterate(&p);
- if (!_lambda_proxy_class_dictionary.empty()) {
- st->print_cr("%sShared Lambda Dictionary", prefix);
- SharedLambdaDictionaryPrinter ldp(st, p.index());
- _lambda_proxy_class_dictionary.iterate(&ldp);
- }
+ LambdaProxyClassDictionary::print_on(prefix, st, p.index(), is_static_archive);
}
void SystemDictionaryShared::ArchiveInfo::print_table_statistics(const char* prefix,
- outputStream* st) {
+ outputStream* st,
+ bool is_static_archive) {
st->print_cr("%sArchve Statistics", prefix);
_builtin_dictionary.print_table_statistics(st, "Builtin Shared Dictionary");
_unregistered_dictionary.print_table_statistics(st, "Unregistered Shared Dictionary");
- _lambda_proxy_class_dictionary.print_table_statistics(st, "Lambda Shared Dictionary");
+ LambdaProxyClassDictionary::print_statistics(st, is_static_archive);
}
void SystemDictionaryShared::print_shared_archive(outputStream* st, bool is_static) {
if (CDSConfig::is_using_archive()) {
if (is_static) {
- _static_archive.print_on("", st);
+ _static_archive.print_on("", st, true);
} else {
if (DynamicArchive::is_mapped()) {
- _dynamic_archive.print_on("Dynamic ", st);
+ _dynamic_archive.print_on("Dynamic ", st, false);
}
}
}
@@ -1502,9 +1182,9 @@ void SystemDictionaryShared::print_on(outputStream* st) {
void SystemDictionaryShared::print_table_statistics(outputStream* st) {
if (CDSConfig::is_using_archive()) {
- _static_archive.print_table_statistics("Static ", st);
+ _static_archive.print_table_statistics("Static ", st, true);
if (DynamicArchive::is_mapped()) {
- _dynamic_archive.print_table_statistics("Dynamic ", st);
+ _dynamic_archive.print_table_statistics("Dynamic ", st, false);
}
}
}
@@ -1517,32 +1197,3 @@ bool SystemDictionaryShared::is_dumptime_table_empty() {
}
return false;
}
-
-class CleanupDumpTimeLambdaProxyClassTable: StackObj {
- public:
- bool do_entry(LambdaProxyClassKey& key, DumpTimeLambdaProxyClassInfo& info) {
- assert_lock_strong(DumpTimeTable_lock);
- InstanceKlass* caller_ik = key.caller_ik();
- InstanceKlass* nest_host = caller_ik->nest_host_not_null();
-
- // If the caller class and/or nest_host are excluded, the associated lambda proxy
- // must also be excluded.
- bool always_exclude = SystemDictionaryShared::check_for_exclusion(caller_ik, nullptr) ||
- SystemDictionaryShared::check_for_exclusion(nest_host, nullptr);
-
- for (int i = info._proxy_klasses->length() - 1; i >= 0; i--) {
- InstanceKlass* ik = info._proxy_klasses->at(i);
- if (always_exclude || SystemDictionaryShared::check_for_exclusion(ik, nullptr)) {
- SystemDictionaryShared::reset_registered_lambda_proxy_class(ik);
- info._proxy_klasses->remove_at(i);
- }
- }
- return info._proxy_klasses->length() == 0 ? true /* delete the node*/ : false;
- }
-};
-
-void SystemDictionaryShared::cleanup_lambda_proxy_class_dictionary() {
- assert_lock_strong(DumpTimeTable_lock);
- CleanupDumpTimeLambdaProxyClassTable cleanup_proxy_classes;
- _dumptime_lambda_proxy_class_dictionary->unlink(&cleanup_proxy_classes);
-}
diff --git a/src/hotspot/share/classfile/systemDictionaryShared.hpp b/src/hotspot/share/classfile/systemDictionaryShared.hpp
index 41e3d9c9716..e910bfb5d47 100644
--- a/src/hotspot/share/classfile/systemDictionaryShared.hpp
+++ b/src/hotspot/share/classfile/systemDictionaryShared.hpp
@@ -28,7 +28,6 @@
#include "cds/cds_globals.hpp"
#include "cds/filemap.hpp"
#include "cds/dumpTimeClassInfo.hpp"
-#include "cds/lambdaProxyClassDictionary.hpp"
#include "cds/runTimeClassInfo.hpp"
#include "classfile/classLoaderData.hpp"
#include "classfile/packageEntry.hpp"
@@ -113,11 +112,8 @@ class ConstantPoolCache;
class Dictionary;
class DumpTimeClassInfo;
class DumpTimeSharedClassTable;
-class LambdaProxyClassDictionary;
class RunTimeClassInfo;
class RunTimeSharedDictionary;
-class DumpTimeLambdaProxyClassDictionary;
-class LambdaProxyClassKey;
class SharedClassLoadingMark {
private:
@@ -137,32 +133,19 @@ class SharedClassLoadingMark {
};
class SystemDictionaryShared: public SystemDictionary {
- friend class CleanupDumpTimeLambdaProxyClassTable;
+ friend class LambdaProxyClassDictionary;
struct ArchiveInfo {
RunTimeSharedDictionary _builtin_dictionary;
RunTimeSharedDictionary _unregistered_dictionary;
- LambdaProxyClassDictionary _lambda_proxy_class_dictionary;
- const RunTimeLambdaProxyClassInfo* lookup_lambda_proxy_class(RunTimeLambdaProxyClassKey* key) {
- return _lambda_proxy_class_dictionary.lookup(key, key->hash(), 0);
- }
-
- void print_on(const char* prefix, outputStream* st);
- void print_table_statistics(const char* prefix, outputStream* st);
- };
-
-public:
- enum : char {
- FROM_FIELD_IS_PROTECTED = 1 << 0,
- FROM_IS_ARRAY = 1 << 1,
- FROM_IS_OBJECT = 1 << 2
+ void print_on(const char* prefix, outputStream* st, bool is_static_archive);
+ void print_table_statistics(const char* prefix, outputStream* st, bool is_static_archive);
};
private:
static DumpTimeSharedClassTable* _dumptime_table;
- static DumpTimeLambdaProxyClassDictionary* _dumptime_lambda_proxy_class_dictionary;
static ArchiveInfo _static_archive;
static ArchiveInfo _dynamic_archive;
@@ -189,19 +172,16 @@ private:
static void write_dictionary(RunTimeSharedDictionary* dictionary,
bool is_builtin);
- static void write_lambda_proxy_class_dictionary(LambdaProxyClassDictionary* dictionary);
- static void cleanup_lambda_proxy_class_dictionary();
- static void reset_registered_lambda_proxy_class(InstanceKlass* ik);
static bool is_jfr_event_class(InstanceKlass *k);
static bool check_for_exclusion_impl(InstanceKlass* k);
static void remove_dumptime_info(InstanceKlass* k) NOT_CDS_RETURN;
static bool has_been_redefined(InstanceKlass* k);
- static InstanceKlass* retrieve_lambda_proxy_class(const RunTimeLambdaProxyClassInfo* info) NOT_CDS_RETURN_(nullptr);
DEBUG_ONLY(static bool _class_loading_may_happen;)
+ static void copy_verification_constraints_from_preimage(InstanceKlass* klass);
+ static void copy_linking_constraints_from_preimage(InstanceKlass* klass);
+
public:
- static bool is_registered_lambda_proxy_class(InstanceKlass* ik);
- static bool is_hidden_lambda_proxy(InstanceKlass* ik);
static bool is_early_klass(InstanceKlass* k); // Was k loaded while JvmtiExport::is_early_phase()==true
static bool has_archived_enum_objs(InstanceKlass* ik);
static void set_has_archived_enum_objs(InstanceKlass* ik);
@@ -229,6 +209,7 @@ public:
static void initialize() NOT_CDS_RETURN;
static void init_dumptime_info(InstanceKlass* k) NOT_CDS_RETURN;
+ static void init_dumptime_info_from_preimage(InstanceKlass* k) NOT_CDS_RETURN;
static void handle_class_unloading(InstanceKlass* k) NOT_CDS_RETURN;
static Dictionary* boot_loader_dictionary() {
@@ -260,28 +241,10 @@ public:
static void add_enum_klass_static_field(InstanceKlass* ik, int root_index);
static void set_class_has_failed_verification(InstanceKlass* ik) NOT_CDS_RETURN;
static bool has_class_failed_verification(InstanceKlass* ik) NOT_CDS_RETURN_(false);
- static void add_lambda_proxy_class(InstanceKlass* caller_ik,
- InstanceKlass* lambda_ik,
- Symbol* invoked_name,
- Symbol* invoked_type,
- Symbol* method_type,
- Method* member_method,
- Symbol* instantiated_method_type, TRAPS) NOT_CDS_RETURN;
- static void add_to_dump_time_lambda_proxy_class_dictionary(LambdaProxyClassKey& key,
- InstanceKlass* proxy_klass) NOT_CDS_RETURN;
- static InstanceKlass* get_shared_lambda_proxy_class(InstanceKlass* caller_ik,
- Symbol* invoked_name,
- Symbol* invoked_type,
- Symbol* method_type,
- Method* member_method,
- Symbol* instantiated_method_type) NOT_CDS_RETURN_(nullptr);
- static InstanceKlass* get_shared_nest_host(InstanceKlass* lambda_ik) NOT_CDS_RETURN_(nullptr);
- static InstanceKlass* prepare_shared_lambda_proxy_class(InstanceKlass* lambda_ik,
- InstanceKlass* caller_ik, TRAPS) NOT_CDS_RETURN_(nullptr);
static bool check_linking_constraints(Thread* current, InstanceKlass* klass) NOT_CDS_RETURN_(false);
static void record_linking_constraint(Symbol* name, InstanceKlass* klass,
Handle loader1, Handle loader2) NOT_CDS_RETURN;
- static bool is_builtin(InstanceKlass* k) {
+ static bool is_builtin(const InstanceKlass* k) {
return (k->shared_classpath_index() != UNREGISTERED_INDEX);
}
static bool add_unregistered_class(Thread* current, InstanceKlass* k);
@@ -297,18 +260,16 @@ public:
static void set_excluded_locked(InstanceKlass* k);
static bool warn_excluded(InstanceKlass* k, const char* reason);
static void dumptime_classes_do(class MetaspaceClosure* it);
- static size_t estimate_size_for_archive();
static void write_to_archive(bool is_static_archive = true);
- static void adjust_lambda_proxy_class_dictionary();
static void serialize_dictionary_headers(class SerializeClosure* soc,
bool is_static_archive = true);
static void serialize_vm_classes(class SerializeClosure* soc);
+ static const char* loader_type_for_shared_class(Klass* k);
static void print() { return print_on(tty); }
static void print_on(outputStream* st) NOT_CDS_RETURN;
static void print_shared_archive(outputStream* st, bool is_static = true) NOT_CDS_RETURN;
static void print_table_statistics(outputStream* st) NOT_CDS_RETURN;
static bool is_dumptime_table_empty() NOT_CDS_RETURN_(true);
- static bool is_supported_invokedynamic(BootstrapInfo* bsi) NOT_CDS_RETURN_(false);
DEBUG_ONLY(static bool class_loading_may_happen() {return _class_loading_may_happen;})
#ifdef ASSERT
diff --git a/src/hotspot/share/classfile/verifier.cpp b/src/hotspot/share/classfile/verifier.cpp
index 4434b06c0b8..d8f9b8589ef 100644
--- a/src/hotspot/share/classfile/verifier.cpp
+++ b/src/hotspot/share/classfile/verifier.cpp
@@ -745,7 +745,6 @@ void ClassVerifier::verify_method(const methodHandle& m, TRAPS) {
LogTarget(Debug, verification) lt;
if (lt.is_enabled()) {
- ResourceMark rm(THREAD);
LogStream ls(lt);
stackmap_table.print_on(&ls);
}
@@ -788,7 +787,6 @@ void ClassVerifier::verify_method(const methodHandle& m, TRAPS) {
LogTarget(Debug, verification) lt;
if (lt.is_enabled()) {
- ResourceMark rm(THREAD);
LogStream ls(lt);
current_frame.print_on(&ls);
lt.print("offset = %d, opcode = %s", bci,
diff --git a/src/hotspot/share/classfile/vmIntrinsics.cpp b/src/hotspot/share/classfile/vmIntrinsics.cpp
index 2943f9d4af3..8011b059697 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.cpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.cpp
@@ -475,6 +475,7 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
case vmIntrinsics::_sha5_implCompress:
if (!UseSHA512Intrinsics) return true;
break;
+ case vmIntrinsics::_double_keccak:
case vmIntrinsics::_sha3_implCompress:
if (!UseSHA3Intrinsics) return true;
break;
@@ -487,6 +488,13 @@ bool vmIntrinsics::disabled_by_jvm_flags(vmIntrinsics::ID id) {
case vmIntrinsics::_chacha20Block:
if (!UseChaCha20Intrinsics) return true;
break;
+ case vmIntrinsics::_dilithiumAlmostNtt:
+ case vmIntrinsics::_dilithiumAlmostInverseNtt:
+ case vmIntrinsics::_dilithiumNttMult:
+ case vmIntrinsics::_dilithiumMontMulByConstant:
+ case vmIntrinsics::_dilithiumDecomposePoly:
+ if (!UseDilithiumIntrinsics) return true;
+ break;
case vmIntrinsics::_base64_encodeBlock:
case vmIntrinsics::_base64_decodeBlock:
if (!UseBASE64Intrinsics) return true;
diff --git a/src/hotspot/share/classfile/vmIntrinsics.hpp b/src/hotspot/share/classfile/vmIntrinsics.hpp
index 0c95f6ab410..93b67301b4b 100644
--- a/src/hotspot/share/classfile/vmIntrinsics.hpp
+++ b/src/hotspot/share/classfile/vmIntrinsics.hpp
@@ -189,6 +189,8 @@ class methodHandle;
do_intrinsic(_minF, java_lang_Math, min_name, float2_float_signature, F_S) \
do_intrinsic(_maxD, java_lang_Math, max_name, double2_double_signature, F_S) \
do_intrinsic(_minD, java_lang_Math, min_name, double2_double_signature, F_S) \
+ do_intrinsic(_maxL, java_lang_Math, max_name, long2_long_signature, F_S) \
+ do_intrinsic(_minL, java_lang_Math, min_name, long2_long_signature, F_S) \
do_intrinsic(_roundD, java_lang_Math, round_name, double_long_signature, F_S) \
do_intrinsic(_roundF, java_lang_Math, round_name, float_int_signature, F_S) \
do_intrinsic(_dcopySign, java_lang_Math, copySign_name, double2_double_signature, F_S) \
@@ -516,6 +518,12 @@ class methodHandle;
do_class(sun_security_provider_sha3, "sun/security/provider/SHA3") \
do_intrinsic(_sha3_implCompress, sun_security_provider_sha3, implCompress_name, implCompress_signature, F_R) \
\
+ /* support for sun.security.provider.SHAKE128Parallel */ \
+ do_class(sun_security_provider_sha3_parallel, "sun/security/provider/SHA3Parallel") \
+ do_intrinsic(_double_keccak, sun_security_provider_sha3_parallel, double_keccak_name, double_keccak_signature, F_S) \
+ do_name( double_keccak_name, "doubleKeccak") \
+ do_signature(double_keccak_signature, "([J[J)I") \
+ \
/* support for sun.security.provider.DigestBase */ \
do_class(sun_security_provider_digestbase, "sun/security/provider/DigestBase") \
do_intrinsic(_digestBase_implCompressMB, sun_security_provider_digestbase, implCompressMB_name, countPositives_signature, F_R) \
@@ -524,7 +532,7 @@ class methodHandle;
/* support for sun.security.util.math.intpoly.MontgomeryIntegerPolynomialP256 */ \
do_class(sun_security_util_math_intpoly_MontgomeryIntegerPolynomialP256, "sun/security/util/math/intpoly/MontgomeryIntegerPolynomialP256") \
do_intrinsic(_intpoly_montgomeryMult_P256, sun_security_util_math_intpoly_MontgomeryIntegerPolynomialP256, intPolyMult_name, intPolyMult_signature, F_R) \
- do_name(intPolyMult_name, "multImpl") \
+ do_name(intPolyMult_name, "mult") \
do_signature(intPolyMult_signature, "([J[J[J)V") \
\
do_class(sun_security_util_math_intpoly_IntegerPolynomial, "sun/security/util/math/intpoly/IntegerPolynomial") \
@@ -561,6 +569,26 @@ class methodHandle;
do_name(chacha20Block_name, "implChaCha20Block") \
do_signature(chacha20Block_signature, "([I[B)I") \
\
+ /* support for sun.security.provider.ML_DSA */ \
+ do_class(sun_security_provider_ML_DSA, "sun/security/provider/ML_DSA") \
+ do_signature(IaII_signature, "([II)I") \
+ do_signature(IaIaI_signature, "([I[I)I") \
+ do_signature(IaIaIaI_signature, "([I[I[I)I") \
+ do_signature(IaIaIaIII_signature, "([I[I[III)I") \
+ do_intrinsic(_dilithiumAlmostNtt, sun_security_provider_ML_DSA, dilithiumAlmostNtt_name, IaIaI_signature, F_S) \
+ do_name(dilithiumAlmostNtt_name, "implDilithiumAlmostNtt") \
+ do_intrinsic(_dilithiumAlmostInverseNtt, sun_security_provider_ML_DSA, \
+ dilithiumAlmostInverseNtt_name, IaIaI_signature, F_S) \
+ do_name(dilithiumAlmostInverseNtt_name, "implDilithiumAlmostInverseNtt") \
+ do_intrinsic(_dilithiumNttMult, sun_security_provider_ML_DSA, dilithiumNttMult_name, IaIaIaI_signature, F_S) \
+ do_name(dilithiumNttMult_name, "implDilithiumNttMult") \
+ do_intrinsic(_dilithiumMontMulByConstant, sun_security_provider_ML_DSA, \
+ dilithiumMontMulByConstant_name, IaII_signature, F_S) \
+ do_name(dilithiumMontMulByConstant_name, "implDilithiumMontMulByConstant") \
+ do_intrinsic(_dilithiumDecomposePoly, sun_security_provider_ML_DSA, \
+ dilithiumDecomposePoly_name, IaIaIaIII_signature, F_S) \
+ do_name(dilithiumDecomposePoly_name, "implDilithiumDecomposePoly") \
+ \
/* support for java.util.zip */ \
do_class(java_util_zip_CRC32, "java/util/zip/CRC32") \
do_intrinsic(_updateCRC32, java_util_zip_CRC32, update_name, int2_int_signature, F_SN) \
diff --git a/src/hotspot/share/code/codeBlob.cpp b/src/hotspot/share/code/codeBlob.cpp
index 9b61d9e6650..4ae4ccbc858 100644
--- a/src/hotspot/share/code/codeBlob.cpp
+++ b/src/hotspot/share/code/codeBlob.cpp
@@ -109,26 +109,26 @@ unsigned int CodeBlob::align_code_offset(int offset) {
// This must be consistent with the CodeBlob constructor's layout actions.
unsigned int CodeBlob::allocation_size(CodeBuffer* cb, int header_size) {
- unsigned int size = header_size;
- size += align_up(cb->total_relocation_size(), oopSize);
// align the size to CodeEntryAlignment
- size = align_code_offset(size);
+ unsigned int size = align_code_offset(header_size);
size += align_up(cb->total_content_size(), oopSize);
size += align_up(cb->total_oop_size(), oopSize);
- size += align_up(cb->total_metadata_size(), oopSize);
return size;
}
CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, CodeBuffer* cb, int size, uint16_t header_size,
- int16_t frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments) :
+ int16_t frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments,
+ int mutable_data_size) :
_oop_maps(nullptr), // will be set by set_oop_maps() call
_name(name),
+ _mutable_data(header_begin() + size), // default value is blob_end()
_size(size),
_relocation_size(align_up(cb->total_relocation_size(), oopSize)),
- _content_offset(CodeBlob::align_code_offset(header_size + _relocation_size)),
+ _content_offset(CodeBlob::align_code_offset(header_size)),
_code_offset(_content_offset + cb->total_offset_of(cb->insts())),
_data_offset(_content_offset + align_up(cb->total_content_size(), oopSize)),
_frame_size(frame_size),
+ _mutable_data_size(mutable_data_size),
S390_ONLY(_ctable_offset(0) COMMA)
_header_size(header_size),
_frame_complete_offset(frame_complete_offset),
@@ -139,12 +139,23 @@ CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, CodeBuffer* cb, int size
assert(is_aligned(header_size, oopSize), "unaligned size");
assert(is_aligned(_relocation_size, oopSize), "unaligned size");
assert(_data_offset <= _size, "codeBlob is too small: %d > %d", _data_offset, _size);
+ assert(is_nmethod() || (cb->total_oop_size() + cb->total_metadata_size() == 0), "must be nmethod");
assert(code_end() == content_end(), "must be the same - see code_end()");
#ifdef COMPILER1
// probably wrong for tiered
assert(_frame_size >= -1, "must use frame size or -1 for runtime stubs");
#endif // COMPILER1
+ if (_mutable_data_size > 0) {
+ _mutable_data = (address)os::malloc(_mutable_data_size, mtCode);
+ if (_mutable_data == nullptr) {
+ vm_exit_out_of_memory(_mutable_data_size, OOM_MALLOC_ERROR, "codebuffer: no space for mutable data");
+ }
+ } else {
+ // We need unique and valid not null address
+ assert(_mutable_data = blob_end(), "sanity");
+ }
+
set_oop_maps(oop_maps);
}
@@ -152,6 +163,7 @@ CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, CodeBuffer* cb, int size
CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, int size, uint16_t header_size) :
_oop_maps(nullptr),
_name(name),
+ _mutable_data(header_begin() + size), // default value is blob_end()
_size(size),
_relocation_size(0),
_content_offset(CodeBlob::align_code_offset(header_size)),
@@ -166,9 +178,15 @@ CodeBlob::CodeBlob(const char* name, CodeBlobKind kind, int size, uint16_t heade
{
assert(is_aligned(size, oopSize), "unaligned size");
assert(is_aligned(header_size, oopSize), "unaligned size");
+ assert(_mutable_data = blob_end(), "sanity");
}
void CodeBlob::purge() {
+ assert(_mutable_data != nullptr, "should never be null");
+ if (_mutable_data != blob_end()) {
+ os::free(_mutable_data);
+ _mutable_data = blob_end(); // Valid not null address
+ }
if (_oop_maps != nullptr) {
delete _oop_maps;
_oop_maps = nullptr;
@@ -210,7 +228,8 @@ RuntimeBlob::RuntimeBlob(
int frame_size,
OopMapSet* oop_maps,
bool caller_must_gc_arguments)
- : CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments)
+ : CodeBlob(name, kind, cb, size, header_size, frame_complete, frame_size, oop_maps, caller_must_gc_arguments,
+ align_up(cb->total_relocation_size(), oopSize))
{
cb->copy_code_and_locs_to(this);
}
@@ -471,9 +490,9 @@ void* RuntimeStub::operator new(size_t s, unsigned size) throw() {
}
// operator new shared by all singletons:
-void* SingletonBlob::operator new(size_t s, unsigned size) throw() {
+void* SingletonBlob::operator new(size_t s, unsigned size, bool alloc_fail_is_fatal) throw() {
void* p = CodeCache::allocate(size, CodeBlobType::NonNMethod);
- if (!p) fatal("Initial size of CodeCache is too small");
+ if (alloc_fail_is_fatal && !p) fatal("Initial size of CodeCache is too small");
return p;
}
@@ -555,7 +574,7 @@ UncommonTrapBlob* UncommonTrapBlob::create(
ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock
{
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
- blob = new (size) UncommonTrapBlob(cb, size, oop_maps, frame_size);
+ blob = new (size, false) UncommonTrapBlob(cb, size, oop_maps, frame_size);
}
trace_new_stub(blob, "UncommonTrapBlob");
@@ -587,7 +606,7 @@ ExceptionBlob* ExceptionBlob::create(
ThreadInVMfromUnknown __tiv; // get to VM state in case we block on CodeCache_lock
{
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
- blob = new (size) ExceptionBlob(cb, size, oop_maps, frame_size);
+ blob = new (size, false) ExceptionBlob(cb, size, oop_maps, frame_size);
}
trace_new_stub(blob, "ExceptionBlob");
diff --git a/src/hotspot/share/code/codeBlob.hpp b/src/hotspot/share/code/codeBlob.hpp
index d0ac2bf1b5f..1daa35c16b5 100644
--- a/src/hotspot/share/code/codeBlob.hpp
+++ b/src/hotspot/share/code/codeBlob.hpp
@@ -67,12 +67,14 @@ enum class CodeBlobType {
// UpcallStub : Used for upcalls from native code
//
//
-// Layout : continuous in the CodeCache
+// Layout in the CodeCache:
// - header
-// - relocation
// - content space
// - instruction space
-// - data space
+// Outside of the CodeCache:
+// - mutable_data
+// - relocation info
+// - additional data for subclasses
enum class CodeBlobKind : u1 {
None,
@@ -104,14 +106,15 @@ protected:
// order fields from large to small to minimize padding between fields
ImmutableOopMapSet* _oop_maps; // OopMap for this CodeBlob
const char* _name;
+ address _mutable_data;
int _size; // total size of CodeBlob in bytes
int _relocation_size; // size of relocation (could be bigger than 64Kb)
int _content_offset; // offset to where content region begins (this includes consts, insts, stubs)
int _code_offset; // offset to where instructions region begins (this includes insts, stubs)
-
int _data_offset; // offset to where data region begins
int _frame_size; // size of stack frame in words (NOT slots. On x64 these are 64bit words)
+ int _mutable_data_size;
S390_ONLY(int _ctable_offset;)
@@ -142,7 +145,8 @@ protected:
const Vptr* vptr() const;
CodeBlob(const char* name, CodeBlobKind kind, CodeBuffer* cb, int size, uint16_t header_size,
- int16_t frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments);
+ int16_t frame_complete_offset, int frame_size, OopMapSet* oop_maps, bool caller_must_gc_arguments,
+ int mutable_data_size);
// Simple CodeBlob used for simple BufferBlob.
CodeBlob(const char* name, CodeBlobKind kind, int size, uint16_t header_size);
@@ -190,20 +194,25 @@ public:
// Boundaries
address header_begin() const { return (address) this; }
address header_end() const { return ((address) this) + _header_size; }
- relocInfo* relocation_begin() const { return (relocInfo*) header_end(); }
- relocInfo* relocation_end() const { return (relocInfo*)(header_end() + _relocation_size); }
address content_begin() const { return (address) header_begin() + _content_offset; }
address content_end() const { return (address) header_begin() + _data_offset; }
address code_begin() const { return (address) header_begin() + _code_offset; }
- // code_end == content_end is true for all types of blobs for now, it is also checked in the constructor
address code_end() const { return (address) header_begin() + _data_offset; }
address data_begin() const { return (address) header_begin() + _data_offset; }
address data_end() const { return (address) header_begin() + _size; }
+ address blob_end() const { return (address) header_begin() + _size; }
+ // code_end == content_end is true for all types of blobs for now, it is also checked in the constructor
+
+ int mutable_data_size() const { return _mutable_data_size; }
+ address mutable_data_begin() const { return _mutable_data; }
+ address mutable_data_end() const { return _mutable_data + _mutable_data_size; }
+
+ relocInfo* relocation_begin() const { return (relocInfo*)_mutable_data; }
+ relocInfo* relocation_end() const { return (relocInfo*)((address)relocation_begin() + _relocation_size); }
// Offsets
int content_offset() const { return _content_offset; }
int code_offset() const { return _code_offset; }
- int data_offset() const { return _data_offset; }
// This field holds the beginning of the const section in the old code buffer.
// It is needed to fix relocations of pc-relative loads when resizing the
@@ -221,11 +230,11 @@ public:
// Only used from CodeCache::free_unused_tail() after the Interpreter blob was trimmed
void adjust_size(size_t used) {
_size = (int)used;
- _data_offset = (int)used;
+ _data_offset = _size;
}
// Containment
- bool blob_contains(address addr) const { return header_begin() <= addr && addr < data_end(); }
+ bool blob_contains(address addr) const { return header_begin() <= addr && addr < blob_end(); }
bool code_contains(address addr) const { return code_begin() <= addr && addr < code_end(); }
bool contains(address addr) const { return content_begin() <= addr && addr < content_end(); }
bool is_frame_complete_at(address addr) const { return _frame_complete_offset != CodeOffsets::frame_never_safe &&
@@ -445,7 +454,7 @@ class SingletonBlob: public RuntimeBlob {
friend class VMStructs;
protected:
- void* operator new(size_t s, unsigned size) throw();
+ void* operator new(size_t s, unsigned size, bool alloc_fail_is_fatal=true) throw();
public:
SingletonBlob(
diff --git a/src/hotspot/share/code/codeCache.cpp b/src/hotspot/share/code/codeCache.cpp
index 3b237e9643e..902d4345622 100644
--- a/src/hotspot/share/code/codeCache.cpp
+++ b/src/hotspot/share/code/codeCache.cpp
@@ -868,10 +868,7 @@ void CodeCache::on_gc_marking_cycle_finish() {
}
void CodeCache::arm_all_nmethods() {
- BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
- if (bs_nm != nullptr) {
- bs_nm->arm_all_nmethods();
- }
+ BarrierSet::barrier_set()->barrier_set_nmethod()->arm_all_nmethods();
}
// Mark nmethods for unloading if they contain otherwise unreachable oops.
@@ -1364,7 +1361,7 @@ void CodeCache::make_marked_nmethods_deoptimized() {
while(iter.next()) {
nmethod* nm = iter.method();
if (nm->is_marked_for_deoptimization() && !nm->has_been_deoptimized() && nm->can_be_deoptimized()) {
- nm->make_not_entrant();
+ nm->make_not_entrant("marked for deoptimization");
nm->make_deoptimized();
}
}
diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp
index 4f72c193d7f..01ace66f4de 100644
--- a/src/hotspot/share/code/nmethod.cpp
+++ b/src/hotspot/share/code/nmethod.cpp
@@ -127,6 +127,7 @@ struct java_nmethod_stats_struct {
uint nmethod_count;
uint total_nm_size;
uint total_immut_size;
+ uint total_mut_size;
uint relocation_size;
uint consts_size;
uint insts_size;
@@ -147,6 +148,7 @@ struct java_nmethod_stats_struct {
nmethod_count += 1;
total_nm_size += nm->size();
total_immut_size += nm->immutable_data_size();
+ total_mut_size += nm->mutable_data_size();
relocation_size += nm->relocation_size();
consts_size += nm->consts_size();
insts_size += nm->insts_size();
@@ -166,7 +168,7 @@ struct java_nmethod_stats_struct {
void print_nmethod_stats(const char* name) {
if (nmethod_count == 0) return;
tty->print_cr("Statistics for %u bytecoded nmethods for %s:", nmethod_count, name);
- uint total_size = total_nm_size + total_immut_size;
+ uint total_size = total_nm_size + total_immut_size + total_mut_size;
if (total_nm_size != 0) {
tty->print_cr(" total size = %u (100%%)", total_size);
tty->print_cr(" in CodeCache = %u (%f%%)", total_nm_size, (total_nm_size * 100.0f)/total_size);
@@ -175,9 +177,6 @@ struct java_nmethod_stats_struct {
if (nmethod_count != 0) {
tty->print_cr(" header = %u (%f%%)", header_size, (header_size * 100.0f)/total_nm_size);
}
- if (relocation_size != 0) {
- tty->print_cr(" relocation = %u (%f%%)", relocation_size, (relocation_size * 100.0f)/total_nm_size);
- }
if (consts_size != 0) {
tty->print_cr(" constants = %u (%f%%)", consts_size, (consts_size * 100.0f)/total_nm_size);
}
@@ -190,12 +189,18 @@ struct java_nmethod_stats_struct {
if (oops_size != 0) {
tty->print_cr(" oops = %u (%f%%)", oops_size, (oops_size * 100.0f)/total_nm_size);
}
+ if (total_mut_size != 0) {
+ tty->print_cr(" mutable data = %u (%f%%)", total_mut_size, (total_mut_size * 100.0f)/total_size);
+ }
+ if (relocation_size != 0) {
+ tty->print_cr(" relocation = %u (%f%%)", relocation_size, (relocation_size * 100.0f)/total_mut_size);
+ }
if (metadata_size != 0) {
- tty->print_cr(" metadata = %u (%f%%)", metadata_size, (metadata_size * 100.0f)/total_nm_size);
+ tty->print_cr(" metadata = %u (%f%%)", metadata_size, (metadata_size * 100.0f)/total_mut_size);
}
#if INCLUDE_JVMCI
if (jvmci_data_size != 0) {
- tty->print_cr(" JVMCI data = %u (%f%%)", jvmci_data_size, (jvmci_data_size * 100.0f)/total_nm_size);
+ tty->print_cr(" JVMCI data = %u (%f%%)", jvmci_data_size, (jvmci_data_size * 100.0f)/total_mut_size);
}
#endif
if (total_immut_size != 0) {
@@ -683,10 +688,6 @@ address nmethod::oops_reloc_begin() const {
return code_begin() + frame_complete_offset();
}
- // It is not safe to read oops concurrently using entry barriers, if their
- // location depend on whether the nmethod is entrant or not.
- // assert(BarrierSet::barrier_set()->barrier_set_nmethod() == nullptr, "Not safe oop scan");
-
address low_boundary = verified_entry_point();
if (!is_in_use()) {
low_boundary += NativeJump::instruction_size;
@@ -1072,6 +1073,13 @@ static void assert_no_oops_or_metadata(nmethod* nm) {
}
#endif
+static int required_mutable_data_size(CodeBuffer* code_buffer,
+ int jvmci_data_size = 0) {
+ return align_up(code_buffer->total_relocation_size(), oopSize) +
+ align_up(code_buffer->total_metadata_size(), oopSize) +
+ align_up(jvmci_data_size, oopSize);
+}
+
nmethod* nmethod::new_native_nmethod(const methodHandle& method,
int compile_id,
CodeBuffer *code_buffer,
@@ -1096,6 +1104,8 @@ nmethod* nmethod::new_native_nmethod(const methodHandle& method,
offsets.set_value(CodeOffsets::Exceptions, exception_handler);
}
+ int mutable_data_size = required_mutable_data_size(code_buffer);
+
// MH intrinsics are dispatch stubs which are compatible with NonNMethod space.
// IsUnloadingBehaviour::is_unloading needs to handle them separately.
bool allow_NonNMethod_space = method->can_be_allocated_in_NonNMethod_space();
@@ -1105,7 +1115,7 @@ nmethod* nmethod::new_native_nmethod(const methodHandle& method,
code_buffer, frame_size,
basic_lock_owner_sp_offset,
basic_lock_sp_offset,
- oop_maps);
+ oop_maps, mutable_data_size);
DEBUG_ONLY( if (allow_NonNMethod_space) assert_no_oops_or_metadata(nm); )
NOT_PRODUCT(if (nm != nullptr) native_nmethod_stats.note_native_nmethod(nm));
}
@@ -1144,11 +1154,6 @@ nmethod* nmethod::new_nmethod(const methodHandle& method,
// create nmethod
nmethod* nm = nullptr;
int nmethod_size = CodeBlob::allocation_size(code_buffer, sizeof(nmethod));
-#if INCLUDE_JVMCI
- if (compiler->is_jvmci()) {
- nmethod_size += align_up(jvmci_data->size(), oopSize);
- }
-#endif
int immutable_data_size =
adjust_pcs_size(debug_info->pcs_size())
@@ -1169,11 +1174,15 @@ nmethod* nmethod::new_nmethod(const methodHandle& method,
return nullptr;
}
}
+
+ int mutable_data_size = required_mutable_data_size(code_buffer
+ JVMCI_ONLY(COMMA (compiler->is_jvmci() ? jvmci_data->size() : 0)));
+
{
MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
nm = new (nmethod_size, comp_level)
- nmethod(method(), compiler->type(), nmethod_size, immutable_data_size,
+ nmethod(method(), compiler->type(), nmethod_size, immutable_data_size, mutable_data_size,
compile_id, entry_bci, immutable_data, offsets, orig_pc_offset,
debug_info, dependencies, code_buffer, frame_size, oop_maps,
handler_table, nul_chk_table, compiler, comp_level
@@ -1276,9 +1285,10 @@ nmethod::nmethod(
int frame_size,
ByteSize basic_lock_owner_sp_offset,
ByteSize basic_lock_sp_offset,
- OopMapSet* oop_maps )
+ OopMapSet* oop_maps,
+ int mutable_data_size)
: CodeBlob("native nmethod", CodeBlobKind::Nmethod, code_buffer, nmethod_size, sizeof(nmethod),
- offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false),
+ offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, mutable_data_size),
_deoptimization_generation(0),
_gc_epoch(CodeCache::gc_epoch()),
_method(method),
@@ -1312,17 +1322,15 @@ nmethod::nmethod(
_deopt_mh_handler_offset = 0;
_unwind_handler_offset = 0;
- CHECKED_CAST(_metadata_offset, uint16_t, (align_up(code_buffer->total_oop_size(), oopSize)));
- int data_end_offset = _metadata_offset + align_up(code_buffer->total_metadata_size(), wordSize);
-#if INCLUDE_JVMCI
- // jvmci_data_size is 0 in native wrapper but we need to set offset
- // to correctly calculate metadata_end address
- CHECKED_CAST(_jvmci_data_offset, uint16_t, data_end_offset);
-#endif
- assert((data_offset() + data_end_offset) <= nmethod_size, "wrong nmethod's size: %d < %d", nmethod_size, (data_offset() + data_end_offset));
+ CHECKED_CAST(_oops_size, uint16_t, align_up(code_buffer->total_oop_size(), oopSize));
+ int metadata_size = align_up(code_buffer->total_metadata_size(), wordSize);
+ JVMCI_ONLY( _jvmci_data_size = 0; )
+ assert(_mutable_data_size == _relocation_size + metadata_size,
+ "wrong mutable data size: %d != %d + %d",
+ _mutable_data_size, _relocation_size, metadata_size);
// native wrapper does not have read-only data but we need unique not null address
- _immutable_data = data_end();
+ _immutable_data = blob_end();
_immutable_data_size = 0;
_nul_chk_table_offset = 0;
_handler_table_offset = 0;
@@ -1399,6 +1407,7 @@ nmethod::nmethod(
CompilerType type,
int nmethod_size,
int immutable_data_size,
+ int mutable_data_size,
int compile_id,
int entry_bci,
address immutable_data,
@@ -1420,7 +1429,7 @@ nmethod::nmethod(
#endif
)
: CodeBlob("nmethod", CodeBlobKind::Nmethod, code_buffer, nmethod_size, sizeof(nmethod),
- offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false),
+ offsets->value(CodeOffsets::Frame_Complete), frame_size, oop_maps, false, mutable_data_size),
_deoptimization_generation(0),
_gc_epoch(CodeCache::gc_epoch()),
_method(method),
@@ -1486,18 +1495,16 @@ nmethod::nmethod(
} else {
_unwind_handler_offset = -1;
}
- CHECKED_CAST(_metadata_offset, uint16_t, (align_up(code_buffer->total_oop_size(), oopSize)));
- int metadata_end_offset = _metadata_offset + align_up(code_buffer->total_metadata_size(), wordSize);
-#if INCLUDE_JVMCI
- CHECKED_CAST(_jvmci_data_offset, uint16_t, metadata_end_offset);
- int jvmci_data_size = compiler->is_jvmci() ? jvmci_data->size() : 0;
- DEBUG_ONLY( int data_end_offset = _jvmci_data_offset + align_up(jvmci_data_size, oopSize); )
-#else
- DEBUG_ONLY( int data_end_offset = metadata_end_offset; )
-#endif
- assert((data_offset() + data_end_offset) <= nmethod_size, "wrong nmethod's size: %d > %d",
- (data_offset() + data_end_offset), nmethod_size);
+ CHECKED_CAST(_oops_size, uint16_t, align_up(code_buffer->total_oop_size(), oopSize));
+ uint16_t metadata_size = (uint16_t)align_up(code_buffer->total_metadata_size(), wordSize);
+ JVMCI_ONLY(CHECKED_CAST(_jvmci_data_size, uint16_t, align_up(compiler->is_jvmci() ? jvmci_data->size() : 0, oopSize)));
+ int jvmci_data_size = 0 JVMCI_ONLY(+ _jvmci_data_size);
+ assert(_mutable_data_size == _relocation_size + metadata_size + jvmci_data_size,
+ "wrong mutable data size: %d != %d + %d + %d",
+ _mutable_data_size, _relocation_size, metadata_size, jvmci_data_size);
+ assert(nmethod_size == data_end() - header_begin(), "wrong nmethod size: %d != %d",
+ nmethod_size, (int)(code_end() - header_begin()));
_immutable_data_size = immutable_data_size;
if (immutable_data_size > 0) {
@@ -1505,7 +1512,7 @@ nmethod::nmethod(
_immutable_data = immutable_data;
} else {
// We need unique not null address
- _immutable_data = data_end();
+ _immutable_data = blob_end();
}
CHECKED_CAST(_nul_chk_table_offset, uint16_t, (align_up((int)dependencies->size_in_bytes(), oopSize)));
CHECKED_CAST(_handler_table_offset, uint16_t, (_nul_chk_table_offset + align_up(nul_chk_table->size_in_bytes(), oopSize)));
@@ -1957,21 +1964,27 @@ void nmethod::invalidate_osr_method() {
}
}
-void nmethod::log_state_change() const {
+void nmethod::log_state_change(const char* reason) const {
+ assert(reason != nullptr, "Must provide a reason");
+
if (LogCompilation) {
if (xtty != nullptr) {
ttyLocker ttyl; // keep the following output all in one block
- xtty->begin_elem("make_not_entrant thread='%zu'",
- os::current_thread_id());
+ xtty->begin_elem("make_not_entrant thread='%zu' reason='%s'",
+ os::current_thread_id(), reason);
log_identity(xtty);
xtty->stamp();
xtty->end_elem();
}
}
- CompileTask::print_ul(this, "made not entrant");
+ ResourceMark rm;
+ stringStream ss(NEW_RESOURCE_ARRAY(char, 256), 256);
+ ss.print("made not entrant: %s", reason);
+
+ CompileTask::print_ul(this, ss.freeze());
if (PrintCompilation) {
- print_on_with_msg(tty, "made not entrant");
+ print_on_with_msg(tty, ss.freeze());
}
}
@@ -1982,7 +1995,9 @@ void nmethod::unlink_from_method() {
}
// Invalidate code
-bool nmethod::make_not_entrant() {
+bool nmethod::make_not_entrant(const char* reason) {
+ assert(reason != nullptr, "Must provide a reason");
+
// This can be called while the system is already at a safepoint which is ok
NoSafepointVerifier nsv;
@@ -2040,7 +2055,7 @@ bool nmethod::make_not_entrant() {
assert(success, "Transition can't fail");
// Log the transition once
- log_state_change();
+ log_state_change(reason);
// Remove nmethod from method.
unlink_from_method();
@@ -2127,9 +2142,9 @@ void nmethod::purge(bool unregister_nmethod) {
}
delete[] _compiled_ic_data;
- if (_immutable_data != data_end()) {
+ if (_immutable_data != blob_end()) {
os::free(_immutable_data);
- _immutable_data = data_end(); // Valid not null address
+ _immutable_data = blob_end(); // Valid not null address
}
if (unregister_nmethod) {
Universe::heap()->unregister_nmethod(this);
@@ -3061,10 +3076,6 @@ void nmethod::print_on_impl(outputStream* st) const {
p2i(this),
p2i(this) + size(),
size());
- if (relocation_size () > 0) st->print_cr(" relocation [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
- p2i(relocation_begin()),
- p2i(relocation_end()),
- relocation_size());
if (consts_size () > 0) st->print_cr(" constants [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
p2i(consts_begin()),
p2i(consts_end()),
@@ -3081,6 +3092,14 @@ void nmethod::print_on_impl(outputStream* st) const {
p2i(oops_begin()),
p2i(oops_end()),
oops_size());
+ if (mutable_data_size() > 0) st->print_cr(" mutable data [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
+ p2i(mutable_data_begin()),
+ p2i(mutable_data_end()),
+ mutable_data_size());
+ if (relocation_size() > 0) st->print_cr(" relocation [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
+ p2i(relocation_begin()),
+ p2i(relocation_end()),
+ relocation_size());
if (metadata_size () > 0) st->print_cr(" metadata [" INTPTR_FORMAT "," INTPTR_FORMAT "] = %d",
p2i(metadata_begin()),
p2i(metadata_end()),
@@ -3550,7 +3569,10 @@ const char* nmethod::reloc_string_for(u_char* begin, u_char* end) {
while (iter.next()) {
have_one = true;
switch (iter.type()) {
- case relocInfo::none: return "no_reloc";
+ case relocInfo::none: {
+ // Skip it and check next
+ break;
+ }
case relocInfo::oop_type: {
// Get a non-resizable resource-allocated stringStream.
// Our callees make use of (nested) ResourceMarks.
@@ -3579,6 +3601,16 @@ const char* nmethod::reloc_string_for(u_char* begin, u_char* end) {
st.print("runtime_call");
CallRelocation* r = (CallRelocation*)iter.reloc();
address dest = r->destination();
+ if (StubRoutines::contains(dest)) {
+ StubCodeDesc* desc = StubCodeDesc::desc_for(dest);
+ if (desc == nullptr) {
+ desc = StubCodeDesc::desc_for(dest + frame::pc_return_offset);
+ }
+ if (desc != nullptr) {
+ st.print(" Stub::%s", desc->name());
+ return st.as_string();
+ }
+ }
CodeBlob* cb = CodeCache::find_blob(dest);
if (cb != nullptr) {
st.print(" %s", cb->name());
diff --git a/src/hotspot/share/code/nmethod.hpp b/src/hotspot/share/code/nmethod.hpp
index ac9b1a7098f..7b19cf75a76 100644
--- a/src/hotspot/share/code/nmethod.hpp
+++ b/src/hotspot/share/code/nmethod.hpp
@@ -134,27 +134,27 @@ public:
// nmethods (native methods) are the compiled code versions of Java methods.
//
// An nmethod contains:
-// - header (the nmethod structure)
-// [Relocation]
-// - relocation information
-// - constant part (doubles, longs and floats used in nmethod)
-// - oop table
-// [Code]
-// - code body
-// - exception handler
-// - stub code
-// [Debugging information]
-// - oop array
-// - data array
-// - pcs
-// [Exception handler table]
-// - handler entry point array
-// [Implicit Null Pointer exception table]
-// - implicit null table array
-// [Speculations]
-// - encoded speculations array
-// [JVMCINMethodData]
-// - meta data for JVMCI compiled nmethod
+// - Header (the nmethod structure)
+// - Constant part (doubles, longs and floats used in nmethod)
+// - Code part:
+// - Code body
+// - Exception handler
+// - Stub code
+// - OOP table
+//
+// As a CodeBlob, an nmethod references [mutable data] allocated on the C heap:
+// - CodeBlob relocation data
+// - Metainfo
+// - JVMCI data
+//
+// An nmethod references [immutable data] allocated on C heap:
+// - Dependency assertions data
+// - Implicit null table array
+// - Handler entry point array
+// - Debugging information:
+// - Scopes data array
+// - Scopes pcs array
+// - JVMCI speculations array
#if INCLUDE_JVMCI
class FailedSpeculation;
@@ -235,11 +235,9 @@ class nmethod : public CodeBlob {
// Number of arguments passed on the stack
uint16_t _num_stack_arg_slots;
- // Offsets in mutable data section
- // _oops_offset == _data_offset, offset where embedded oop table begins (inside data)
- uint16_t _metadata_offset; // embedded meta data table
+ uint16_t _oops_size;
#if INCLUDE_JVMCI
- uint16_t _jvmci_data_offset;
+ uint16_t _jvmci_data_size;
#endif
// Offset in immutable data section
@@ -305,13 +303,15 @@ class nmethod : public CodeBlob {
int frame_size,
ByteSize basic_lock_owner_sp_offset, /* synchronized natives only */
ByteSize basic_lock_sp_offset, /* synchronized natives only */
- OopMapSet* oop_maps);
+ OopMapSet* oop_maps,
+ int mutable_data_size);
// For normal JIT compiled code
nmethod(Method* method,
CompilerType type,
int nmethod_size,
int immutable_data_size,
+ int mutable_data_size,
int compile_id,
int entry_bci,
address immutable_data,
@@ -526,22 +526,22 @@ public:
address insts_begin () const { return code_begin() ; }
address insts_end () const { return header_begin() + _stub_offset ; }
address stub_begin () const { return header_begin() + _stub_offset ; }
- address stub_end () const { return data_begin() ; }
+ address stub_end () const { return code_end() ; }
address exception_begin () const { return header_begin() + _exception_offset ; }
address deopt_handler_begin () const { return header_begin() + _deopt_handler_offset ; }
address deopt_mh_handler_begin() const { return header_begin() + _deopt_mh_handler_offset ; }
address unwind_handler_begin () const { return _unwind_handler_offset != -1 ? (insts_end() - _unwind_handler_offset) : nullptr; }
+ oop* oops_begin () const { return (oop*) data_begin(); }
+ oop* oops_end () const { return (oop*) data_end(); }
// mutable data
- oop* oops_begin () const { return (oop*) data_begin(); }
- oop* oops_end () const { return (oop*) (data_begin() + _metadata_offset) ; }
- Metadata** metadata_begin () const { return (Metadata**) (data_begin() + _metadata_offset) ; }
+ Metadata** metadata_begin () const { return (Metadata**) (mutable_data_begin() + _relocation_size); }
#if INCLUDE_JVMCI
- Metadata** metadata_end () const { return (Metadata**) (data_begin() + _jvmci_data_offset) ; }
- address jvmci_data_begin () const { return data_begin() + _jvmci_data_offset ; }
- address jvmci_data_end () const { return data_end(); }
+ Metadata** metadata_end () const { return (Metadata**) (mutable_data_end() - _jvmci_data_size); }
+ address jvmci_data_begin () const { return mutable_data_end() - _jvmci_data_size; }
+ address jvmci_data_end () const { return mutable_data_end(); }
#else
- Metadata** metadata_end () const { return (Metadata**) data_end(); }
+ Metadata** metadata_end () const { return (Metadata**) mutable_data_end(); }
#endif
// immutable data
@@ -631,8 +631,8 @@ public:
// alive. It is used when an uncommon trap happens. Returns true
// if this thread changed the state of the nmethod or false if
// another thread performed the transition.
- bool make_not_entrant();
- bool make_not_used() { return make_not_entrant(); }
+ bool make_not_entrant(const char* reason);
+ bool make_not_used() { return make_not_entrant("not used"); }
bool is_marked_for_deoptimization() const { return deoptimization_status() != not_marked; }
bool has_been_deoptimized() const { return deoptimization_status() == deoptimize_done; }
@@ -945,7 +945,7 @@ public:
// Logging
void log_identity(xmlStream* log) const;
void log_new_nmethod() const;
- void log_state_change() const;
+ void log_state_change(const char* reason) const;
// Prints block-level comments, including nmethod specific block labels:
void print_nmethod_labels(outputStream* stream, address block_begin, bool print_section_labels=true) const;
diff --git a/src/hotspot/share/code/relocInfo.cpp b/src/hotspot/share/code/relocInfo.cpp
index 9dc4eced966..ad194c71bb2 100644
--- a/src/hotspot/share/code/relocInfo.cpp
+++ b/src/hotspot/share/code/relocInfo.cpp
@@ -116,6 +116,9 @@ void relocInfo::change_reloc_info_for_address(RelocIterator *itr, address pc, re
// ----------------------------------------------------------------------------------------------------
// Implementation of RelocIterator
+// A static dummy to serve as a safe pointer when there is no relocation info.
+static relocInfo dummy_relocInfo = relocInfo(relocInfo::none, 0);
+
void RelocIterator::initialize(nmethod* nm, address begin, address limit) {
initialize_misc();
@@ -127,8 +130,14 @@ void RelocIterator::initialize(nmethod* nm, address begin, address limit) {
guarantee(nm != nullptr, "must be able to deduce nmethod from other arguments");
_code = nm;
- _current = nm->relocation_begin() - 1;
- _end = nm->relocation_end();
+ if (nm->relocation_size() == 0) {
+ _current = &dummy_relocInfo - 1;
+ _end = &dummy_relocInfo;
+ } else {
+ assert(((nm->relocation_begin() != nullptr) && (nm->relocation_end() != nullptr)), "valid start and end pointer");
+ _current = nm->relocation_begin() - 1;
+ _end = nm->relocation_end();
+ }
_addr = nm->content_begin();
// Initialize code sections.
@@ -150,7 +159,7 @@ void RelocIterator::initialize(nmethod* nm, address begin, address limit) {
RelocIterator::RelocIterator(CodeSection* cs, address begin, address limit) {
initialize_misc();
assert(((cs->locs_start() != nullptr) && (cs->locs_end() != nullptr)), "valid start and end pointer");
- _current = cs->locs_start()-1;
+ _current = cs->locs_start() - 1;
_end = cs->locs_end();
_addr = cs->start();
_code = nullptr; // Not cb->blob();
@@ -869,13 +878,43 @@ void RelocIterator::print_current() {
static_call_Relocation* r = (static_call_Relocation*) reloc();
tty->print(" | [destination=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]",
p2i(r->destination()), p2i(r->method_value()));
+ CodeBlob* cb = CodeCache::find_blob(r->destination());
+ if (cb != nullptr) {
+ tty->print(" Blob::%s", cb->name());
+ }
break;
}
case relocInfo::runtime_call_type:
case relocInfo::runtime_call_w_cp_type:
{
CallRelocation* r = (CallRelocation*) reloc();
- tty->print(" | [destination=" INTPTR_FORMAT "]", p2i(r->destination()));
+ address dest = r->destination();
+ tty->print(" | [destination=" INTPTR_FORMAT "]", p2i(dest));
+ if (StubRoutines::contains(dest)) {
+ StubCodeDesc* desc = StubCodeDesc::desc_for(dest);
+ if (desc == nullptr) {
+ desc = StubCodeDesc::desc_for(dest + frame::pc_return_offset);
+ }
+ if (desc != nullptr) {
+ tty->print(" Stub::%s", desc->name());
+ }
+ } else {
+ CodeBlob* cb = CodeCache::find_blob(dest);
+ if (cb != nullptr) {
+ tty->print(" %s", cb->name());
+ } else {
+ ResourceMark rm;
+ const int buflen = 1024;
+ char* buf = NEW_RESOURCE_ARRAY(char, buflen);
+ int offset;
+ if (os::dll_address_to_function_name(dest, buf, buflen, &offset)) {
+ tty->print(" %s", buf);
+ if (offset != 0) {
+ tty->print("+%d", offset);
+ }
+ }
+ }
+ }
break;
}
case relocInfo::virtual_call_type:
@@ -883,6 +922,10 @@ void RelocIterator::print_current() {
virtual_call_Relocation* r = (virtual_call_Relocation*) reloc();
tty->print(" | [destination=" INTPTR_FORMAT " cached_value=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]",
p2i(r->destination()), p2i(r->cached_value()), p2i(r->method_value()));
+ CodeBlob* cb = CodeCache::find_blob(r->destination());
+ if (cb != nullptr) {
+ tty->print(" Blob::%s", cb->name());
+ }
break;
}
case relocInfo::static_stub_type:
@@ -902,6 +945,10 @@ void RelocIterator::print_current() {
opt_virtual_call_Relocation* r = (opt_virtual_call_Relocation*) reloc();
tty->print(" | [destination=" INTPTR_FORMAT " metadata=" INTPTR_FORMAT "]",
p2i(r->destination()), p2i(r->method_value()));
+ CodeBlob* cb = CodeCache::find_blob(r->destination());
+ if (cb != nullptr) {
+ tty->print(" Blob::%s", cb->name());
+ }
break;
}
default:
diff --git a/src/hotspot/share/compiler/compilationFailureInfo.cpp b/src/hotspot/share/compiler/compilationFailureInfo.cpp
index aaa9cb79d12..3940b4849f6 100644
--- a/src/hotspot/share/compiler/compilationFailureInfo.cpp
+++ b/src/hotspot/share/compiler/compilationFailureInfo.cpp
@@ -33,12 +33,12 @@
#include "compiler/compilationFailureInfo.hpp"
#include "compiler/compileTask.hpp"
#ifdef COMPILER2
-#include "opto/node.hpp"
#include "opto/compile.hpp"
+#include "opto/node.hpp"
#endif
#include "runtime/os.hpp"
-#include "utilities/ostream.hpp"
#include "utilities/nativeCallStack.hpp"
+#include "utilities/ostream.hpp"
int CompilationFailureInfo::current_compile_id_or_0() {
ciEnv* env = ciEnv::current();
diff --git a/src/hotspot/share/compiler/compilationMemStatInternals.hpp b/src/hotspot/share/compiler/compilationMemStatInternals.hpp
new file mode 100644
index 00000000000..a6050823771
--- /dev/null
+++ b/src/hotspot/share/compiler/compilationMemStatInternals.hpp
@@ -0,0 +1,287 @@
+/*
+ * Copyright (c) 2024, 2025, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_COMPILER_COMPILATIONMEMSTATINTERNALS_HPP
+#define SHARE_COMPILER_COMPILATIONMEMSTATINTERNALS_HPP
+
+#include "compiler/compilationMemoryStatistic.hpp"
+#include "compiler/compilerDefinitions.hpp"
+#include "memory/allocation.hpp"
+#include "memory/arena.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+#ifdef COMPILER2
+#include "opto/phase.hpp"
+#endif
+
+class CompileTask;
+class Method;
+class Symbol;
+class outputStream;
+
+#ifdef COMPILER2
+constexpr int phase_trc_id_max = (int)Phase::PhaseTraceId::max_phase_timers;
+constexpr int phase_trc_id_none = (int)Phase::PhaseTraceId::_t_none;
+#else
+// In minimal builds, the ArenaCounterTable is just a single-dimension vector of arena tags (see below)
+constexpr int phase_trc_id_max = 1;
+constexpr int phase_trc_id_none = 0;
+#endif
+inline void check_phase_trace_id(int v) { assert(v >= 0 && v < phase_trc_id_max, "OOB (%d)", v); }
+
+constexpr int arena_tag_max = (int)Arena::Tag::tag_count;
+inline void check_arena_tag(int v) { assert(v >= 0 && v < arena_tag_max, "OOB (%d)", v); }
+
+// A two-dimensional table, containing byte counters per arena type and
+// per compilation phase.
+class ArenaCounterTable {
+ size_t _v[phase_trc_id_max][arena_tag_max];
+public:
+ ArenaCounterTable();
+ void copy_from(const ArenaCounterTable& other);
+ inline size_t at(int phase_trc_id, int arena_tag) const;
+ inline void add(size_t size, int phase_trc_id, int arena_tag);
+ inline void sub(size_t size, int phase_trc_id, int arena_tag);
+ void print_on(outputStream* ss) const;
+ void summarize(size_t out[arena_tag_max]) const;
+};
+
+struct PhaseInfo {
+ int id, num;
+ const char* text;
+};
+
+// A stack keeping track of the current compilation phase. Fixed-width for simplicity
+// (we should never go beyond 5 or so in depth).
+class PhaseInfoStack {
+ static constexpr int max_depth = 16;
+ int _depth;
+ PhaseInfo _stack[max_depth];
+public:
+ inline PhaseInfoStack();
+ inline bool empty() const { return _depth == 0; }
+ inline void push(PhaseInfo info);
+ inline void pop();
+ inline const PhaseInfo& top() const;
+ inline int depth() const { return _depth; }
+};
+
+// A very simple fixed-width FIFO buffer, used for the phase timeline
+template
+class SimpleFifo {
+ STATIC_ASSERT((size * 2) < INT_MAX);
+ T _v[size];
+ int _pos;
+ int _oldest;
+ uint64_t _lost;
+
+ int current_pos() const { return _pos; }
+ static int pos_to_index(int pos) { return pos % size; }
+ T& at(int pos) { return *(_v + pos_to_index(pos)); }
+
+public:
+ SimpleFifo() : _pos(0), _oldest(0), _lost(0UL) {}
+ T& current() { return at(current_pos()); }
+ T& last() { assert(!empty(), "sanity"); return at(current_pos() - 1); }
+ bool empty() const { return _pos == _oldest; }
+ uint64_t lost() const { return _lost; }
+
+ void advance() {
+ _pos ++;
+ if (_pos >= size) {
+ _oldest ++;
+ _lost ++;
+ }
+ if (_pos == INT_MAX) {
+ _pos -= size;
+ _oldest -= size;
+ }
+ }
+
+ void revert() {
+ assert(!empty(), "sanity");
+ _pos--;
+ }
+
+ template
+ void iterate_all(F f) const {
+ for (int i = _oldest; i < _pos; i++) {
+ const int index = pos_to_index(i);
+ f(_v[index]);
+ }
+ }
+
+ void copy_from(const SimpleFifo& other) {
+ memcpy(_v, other._v, sizeof(_v));
+ _pos = other._pos;
+ _lost = other._lost;
+ _oldest = other._oldest;
+ }
+};
+
+// Holds a table of n entries; each entry keeping start->end footprints when
+// a phase started and ended; each entry also keeping the phase-local peak (if
+// a phase caused a temporary spike in footprint that vanished before the phase
+// ended).
+// Handling nested phases: for this structure, there is always a phase active;
+// if a phase ends, we "restart" the parent phase (which often is the
+// "outside any phase" phase).
+class FootprintTimeline {
+public:
+ static constexpr unsigned max_num_phases = 256; // beyond that we wrap, keeping just the last n phases
+private:
+ template
+ struct C {
+ T start, peak, cur;
+ void init(T v) { start = cur = peak = v; }
+ void update(T v) { cur = v; if (v > peak) peak = v; }
+ dT end_delta() const { return (dT)cur - (dT)start; }
+ // Returns the peak size during this phase: how high usage rose above either
+ // start or end of phase. The background is that we want to know the max. memory
+ // consumption during this phase, but that may not be reflected by the start or the
+ // end counters if an Arena was created during the phase and only lived temporarily.
+ size_t temporary_peak_size() const { return MIN2(peak - cur, peak - start); }
+ };
+ struct Entry {
+ PhaseInfo info;
+ int level;
+ C _bytes;
+ C _live_nodes;
+ };
+ SimpleFifo _fifo;
+ DEBUG_ONLY(bool _inbetween_phases;)
+public:
+ FootprintTimeline();
+ void copy_from(const FootprintTimeline& other);
+ inline void on_footprint_change(size_t cur_abs, unsigned cur_nodes);
+ void on_phase_end(size_t cur_abs, unsigned cur_nodes);
+ void on_phase_start(PhaseInfo info, size_t cur_abs, unsigned cur_nodes, int level);
+ void print_on(outputStream* st) const;
+};
+
+// We keep the name of the involved symbols in Symbol (made permanent) instead of resolving them to string and
+// storing those. That significantly reduces footprint for the result store and delays resolving until printing
+// time, which may be never.
+class FullMethodName {
+ Symbol* _k;
+ Symbol* _m;
+ Symbol* _s;
+public:
+ FullMethodName();
+ FullMethodName(const Method* m);
+ FullMethodName(const FullMethodName& o);
+ FullMethodName& operator=(const FullMethodName& o);
+ void make_permanent();
+ void print_on(outputStream* st) const;
+ char* as_C_string(char* buf, size_t len) const;
+ bool operator== (const FullMethodName& b) const;
+ DEBUG_ONLY(bool is_test_class() const;)
+};
+
+// ArenaState is the central data structure holding all statistics and temp data during
+// a single compilation. It is created on demand (if memstat is active) and tied to the
+// CompilerThread.
+class ArenaStatCounter : public CHeapObj {
+
+ FullMethodName _fmn;
+
+ // from directives
+ const bool _should_print_memstat;
+ const bool _should_crash_on_memlimit;
+
+ // Bytes total now
+ size_t _current;
+ // Bytes total at last global peak
+ size_t _peak;
+ // Bytes per arena/phase, now
+ ArenaCounterTable _counters_current;
+ // Bytes per arena/phase when we last reached the global peak
+ ArenaCounterTable _counters_at_global_peak;
+
+ // Number of live nodes now (C2 only)
+ unsigned _live_nodes_current;
+ // Number of live nodes at global peak (C2 only)
+ unsigned _live_nodes_at_global_peak;
+
+ // MemLimit handling
+ const size_t _limit;
+ bool _hit_limit;
+ bool _limit_in_process;
+
+ // Keep track of current C2 phase
+ int _phase_counter;
+ PhaseInfoStack _phase_info_stack;
+
+ // Keep track of C2 phase allocations over time
+ FootprintTimeline _timeline;
+
+ const CompilerType _comp_type;
+ const int _comp_id;
+
+ DEBUG_ONLY(bool _is_test_class;)
+
+ int retrieve_live_node_count() const;
+
+public:
+ ArenaStatCounter(const CompileTask* task, size_t limit);
+
+ void on_phase_start(PhaseInfo info);
+ void on_phase_end();
+
+ // Account an arena allocation. Returns true if new peak reached.
+ bool on_arena_chunk_allocation(size_t size, int arena_tag, uint64_t* stamp);
+
+ // Account an arena deallocation.
+ void on_arena_chunk_deallocation(size_t size, uint64_t stamp);
+
+ void print_peak_state_on(outputStream* st) const;
+ void print_error_state_on(outputStream* st) const;
+
+ size_t limit() const { return _limit; }
+ bool hit_limit() const { return _hit_limit; }
+ bool limit_in_process() const { return _limit_in_process; }
+ void set_limit_in_process(bool v) { _limit_in_process = v; }
+
+ const FullMethodName& fmn() const { return _fmn; }
+ bool should_print_memstat() { return _should_print_memstat; };
+ bool should_crash_on_memlimit() const { return _should_crash_on_memlimit; };
+
+ CompilerType comp_type() const { return _comp_type; }
+ int comp_id() const { return _comp_id; }
+ DEBUG_ONLY(bool is_test_class() const { return _is_test_class; })
+
+ // Bytes total at last global peak
+ size_t peak() const { return _peak; }
+
+ // Bytes per arena/phase when we last reached the global peak
+ const ArenaCounterTable& counters_at_global_peak() const { return _counters_at_global_peak; }
+ const FootprintTimeline& timeline() const { return _timeline; }
+ // Number of live nodes at global peak (C2 only)
+ unsigned live_nodes_at_global_peak() const { return _live_nodes_at_global_peak; }
+
+ int advance_phase_counter() { return ++_phase_counter; }
+};
+
+#endif // SHARE_COMPILER_COMPILATIONMEMSTATINTERNALS_HPP
diff --git a/src/hotspot/share/compiler/compilationMemStatInternals.inline.hpp b/src/hotspot/share/compiler/compilationMemStatInternals.inline.hpp
new file mode 100644
index 00000000000..94be8606ac9
--- /dev/null
+++ b/src/hotspot/share/compiler/compilationMemStatInternals.inline.hpp
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2024, 2025, Red Hat, Inc. and/or its affiliates.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_COMPILATIONMEMSTATINTERNALS_INLINE_HPP
+#define SHARE_COMPILATIONMEMSTATINTERNALS_INLINE_HPP
+
+#include "compiler/compilationMemStatInternals.hpp"
+
+inline PhaseInfoStack::PhaseInfoStack() : _depth(0) {}
+
+inline void PhaseInfoStack::push(PhaseInfo info) {
+#ifdef ASSERT
+ check_phase_trace_id(info.id);
+ if (_depth == 0) {
+ assert(info.id == phase_trc_id_none, "first entry must be none");
+ } else {
+ assert(info.id != phase_trc_id_none, "subsequent entries must not be none");
+ }
+ assert(_depth < max_depth, "Sanity");
+#endif // ASSERT
+ _stack[_depth] = info;
+ if (_depth < max_depth) {
+ _depth++;
+ }
+}
+
+inline void PhaseInfoStack::pop() {
+#ifdef ASSERT
+ assert(!empty(), "Sanity ");
+ const PhaseInfo to_be_popped = top();
+ if (_depth == 1) {
+ assert(to_be_popped.id == phase_trc_id_none, "first entry must be none");
+ } else {
+ assert(to_be_popped.id != phase_trc_id_none, "subsequent entries must not be none");
+ }
+#endif // ASSERT
+ if (_depth > 0) {
+ _depth--;
+ }
+}
+
+inline const PhaseInfo& PhaseInfoStack::top() const {
+ assert(!empty(), "Sanity");
+ return _stack[_depth - 1];
+}
+
+inline size_t ArenaCounterTable::at(int phase_trc_id, int arena_tag) const {
+ check_phase_trace_id(phase_trc_id);
+ check_arena_tag(arena_tag);
+ return _v[phase_trc_id][arena_tag];
+}
+
+inline void ArenaCounterTable::add(size_t size, int phase_trc_id, int arena_tag) {
+ check_arena_tag(arena_tag);
+ const size_t old = at(phase_trc_id, arena_tag);
+ _v[phase_trc_id][arena_tag] += size;
+ assert(at(phase_trc_id, arena_tag) >= old, "Overflow");
+}
+
+inline void ArenaCounterTable::sub(size_t size, int phase_trc_id, int arena_tag) {
+ check_arena_tag(arena_tag);
+ assert(at(phase_trc_id, arena_tag) >= size, "Underflow (%zu %zu)", at(phase_trc_id, arena_tag), size);
+ _v[phase_trc_id][arena_tag] -= size;
+}
+
+inline void FootprintTimeline::on_footprint_change(size_t cur_abs, unsigned cur_nodes) {
+ assert(!_inbetween_phases, "no phase started?");
+ Entry& e = _fifo.current();
+ e._bytes.update(cur_abs);
+ e._live_nodes.update(cur_nodes);
+}
+
+#endif // SHARE_COMPILATIONMEMSTATINTERNALS_INLINE_HPP
diff --git a/src/hotspot/share/compiler/compilationMemoryStatistic.cpp b/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
index 89d0b60212e..0c2822c94d2 100644
--- a/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
+++ b/src/hotspot/share/compiler/compilationMemoryStatistic.cpp
@@ -1,6 +1,6 @@
/*
* Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, 2024, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2023, 2025, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -23,413 +23,778 @@
*
*/
+#include "code/nmethod.hpp"
+#include "compiler/abstractCompiler.hpp"
+#include "compiler/compilationMemStatInternals.inline.hpp"
+#include "compiler/compilerDefinitions.inline.hpp"
+#include "compiler/compilerDirectives.hpp"
+#include "compiler/compilerOracle.hpp"
+#include "compiler/compilerThread.hpp"
+#include "compiler/compileTask.hpp"
#include "logging/log.hpp"
#include "logging/logStream.hpp"
-#ifdef COMPILER1
-#include "c1/c1_Compilation.hpp"
-#endif
-#include "compiler/abstractCompiler.hpp"
-#include "compiler/compilationMemoryStatistic.hpp"
-#include "compiler/compilerDirectives.hpp"
-#include "compiler/compileTask.hpp"
-#include "compiler/compilerDefinitions.hpp"
-#include "compiler/compilerThread.hpp"
#include "memory/arena.hpp"
-#include "memory/resourceArea.hpp"
#include "nmt/nmtCommon.hpp"
+#include "oops/method.inline.hpp"
#include "oops/symbol.hpp"
-#ifdef COMPILER2
-#include "opto/node.hpp" // compile.hpp is not self-contained
-#include "opto/compile.hpp"
-#endif
+#include "runtime/atomic.hpp"
#include "runtime/mutexLocker.hpp"
#include "runtime/os.hpp"
+#include "utilities/checkedCast.hpp"
#include "utilities/debug.hpp"
#include "utilities/globalDefinitions.hpp"
#include "utilities/ostream.hpp"
-#include "utilities/quickSort.hpp"
-#include "utilities/resourceHash.hpp"
-ArenaStatCounter::ArenaStatCounter() {
- reset();
-}
+#ifdef COMPILER1
+#include "c1/c1_Compilation.hpp"
+#endif
-void ArenaStatCounter::reset() {
- _current = 0;
- _peak = 0;
- _current_by_tag.clear();
- _peak_by_tag.clear();
- _limit = 0;
- _hit_limit = false;
- _limit_in_process = false;
- _live_nodes_at_peak = 0;
- _active = false;
-}
-
-void ArenaStatCounter::start(size_t limit) {
- reset();
- _active = true;
- _limit = limit;
-}
-
-void ArenaStatCounter::end() {
- _limit = 0;
- _hit_limit = false;
- _active = false;
-}
-
-void ArenaStatCounter::update_c2_node_count() {
- assert(_active, "compilaton has not yet started");
#ifdef COMPILER2
- CompilerThread* const th = Thread::current()->as_Compiler_thread();
- const CompileTask* const task = th->task();
- if (task != nullptr &&
- th->task()->compiler() != nullptr &&
- th->task()->compiler()->type() == compiler_c2) {
- const Compile* const comp = Compile::current();
- if (comp != nullptr) {
- _live_nodes_at_peak = comp->live_nodes();
+#include "opto/compile.hpp"
+#include "opto/node.hpp" // compile.hpp is not self-contained
+#endif
+
+static const char* phase_trc_id_to_string(int phase_trc_id) {
+ return COMPILER2_PRESENT(Phase::get_phase_trace_id_text((Phase::PhaseTraceId)phase_trc_id))
+ NOT_COMPILER2("");
+}
+
+// If crash option on memlimit is enabled and an oom occurred, the stats for the
+// first offending compilation.
+static ArenaStatCounter* volatile _arenastat_oom_crash = nullptr;
+
+// Arena-chunk stamping
+union chunkstamp_t {
+ uint64_t raw;
+ struct {
+ uint32_t tracked;
+ uint16_t arena_tag;
+ uint16_t phase_id;
+ };
+};
+STATIC_ASSERT(sizeof(chunkstamp_t) == sizeof(chunkstamp_t::raw));
+
+ArenaCounterTable::ArenaCounterTable() {
+ memset(_v, 0, sizeof(_v));
+}
+
+void ArenaCounterTable::copy_from(const ArenaCounterTable& other) {
+ memcpy(_v, other._v, sizeof(_v));
+}
+
+void ArenaCounterTable::summarize(size_t out[arena_tag_max]) const {
+ memset(out, 0, arena_tag_max * sizeof(size_t));
+ for (int i = 0; i < phase_trc_id_max; i++) {
+ for (int j = 0; j < arena_tag_max; j++) {
+ out[j] += _v[i][j];
}
}
-#endif
}
-// Account an arena allocation or de-allocation.
-bool ArenaStatCounter::account(ssize_t delta, int tag) {
- assert(_active, "compilaton has not yet started");
- bool rc = false;
+void ArenaCounterTable::print_on(outputStream* st) const {
+ bool header_printed = false;
+ for (int phase_trc_id = 0; phase_trc_id < phase_trc_id_max; phase_trc_id++) {
+ size_t sum = 0;
+ for (int arena_tag = 0; arena_tag < arena_tag_max; arena_tag++) {
+ sum += at(phase_trc_id, arena_tag);
+ }
+ if (sum > 0) { // omit phases that did not contribute to allocation load
+ if (!header_printed) {
+ st->print("%-24s %10s", "Phase", "Total");
+ for (int arena_tag = 0; arena_tag < arena_tag_max; arena_tag++) {
+ st->print("%10s", Arena::tag_name[arena_tag]);
+ }
+ st->cr();
+ header_printed = true;
+ }
+ st->print("%-24s ", phase_trc_id_to_string(phase_trc_id));
+ st->print("%10zu", sum);
+ for (int arena_tag = 0; arena_tag < arena_tag_max; arena_tag++) {
+ const size_t v = at(phase_trc_id, arena_tag);
+ st->print("%10zu", v);
+ }
+ st->cr();
+ }
+ }
+}
+
+// When reporting phase footprint movements, if phase-local peak over start as well over end
+// was larger than this threshold, we report it.
+static constexpr size_t significant_peak_threshold = M;
+
+FootprintTimeline::FootprintTimeline() {
+ DEBUG_ONLY(_inbetween_phases = true;)
+}
+
+void FootprintTimeline::copy_from(const FootprintTimeline& other) {
+ _fifo.copy_from(other._fifo);
+ DEBUG_ONLY(_inbetween_phases = other._inbetween_phases;)
+}
+
+void FootprintTimeline::print_on(outputStream* st) const {
+ const int start_indent = st->indentation();
+ if (!_fifo.empty()) {
+ // .123456789.123456789.123456789.123456789.123456789.123456789.123456789.123456789.123456789.123456789
+ st->print_cr("Phase seq. number Bytes Nodes");
+ unsigned from = 0;
+ if (_fifo.lost() > 0) {
+ st->print_cr(" (" UINT64_FORMAT " older entries lost)", _fifo.lost());
+ }
+ int last_level = 0;
+ int last_num = 0; // see if we are regressing
+ auto printer = [&](const Entry& e) {
+ int col = start_indent;
+ check_phase_trace_id(e.info.id);
+ st->print("%*s", e.level,
+ ((e.level < last_level) ? "<" : ((e.level > last_level) ? ">" : " "))
+ );
+ last_level = e.level;
+ st->print("%d ", e.info.num);
+ if (e.info.num < last_num) {
+ st->print("(cont.) ");
+ }
+ last_num = e.info.num;
+ col += 15; st->fill_to(col);
+ st->print("%24s", e.info.text);
+ col += 25; st->fill_to(col);
+ char tmp[64];
+ os::snprintf(tmp, sizeof(tmp), "%9zu (%+zd)", e._bytes.cur, e._bytes.end_delta());
+ st->print("%s ", tmp); // end
+ col += 21; st->fill_to(col);
+ os::snprintf(tmp, sizeof(tmp), "%6u (%+d)", e._live_nodes.cur, e._live_nodes.end_delta());
+ st->print("%s ", tmp); // end
+ if (e._bytes.temporary_peak_size() > significant_peak_threshold) {
+ col += 20; st->fill_to(col);
+ st->print(" significant temporary peak: %zu (%+zd)", e._bytes.peak, (ssize_t)e._bytes.peak - e._bytes.start); // peak
+ }
+ st->cr();
+ };
+ _fifo.iterate_all(printer);
+ }
+}
+
+void FootprintTimeline::on_phase_end(size_t cur_abs, unsigned cur_nodes) {
+ const Entry& old = _fifo.current();
+
+ // One last counter update in old phase:
+ // We see all allocations, so cur_abs given should correspond to our topmost cur.
+ // But the same does not hold for nodes, since we only get updated when node allocation
+ // would cause a new arena chunk to be born. Node allocations that don't cause arena
+ // chunks (the vast majority) fly by us.
+ assert(old._bytes.cur == cur_abs, "miscount");
+ on_footprint_change(cur_abs, cur_nodes);
+
+ // Close old, open new entry
+ _fifo.advance();
+
+ DEBUG_ONLY(_inbetween_phases = true;)
+}
+
+void FootprintTimeline::on_phase_start(PhaseInfo info, size_t cur_abs, unsigned cur_nodes, int level) {
+ if (!_fifo.empty() && _fifo.last().info.id == info.id && _fifo.last().level == level) {
+ // Two phases with the same id are collapsed if they were not interleaved by another phase
+ _fifo.revert();
+ // We now just continue bookkeeping into the last entry
+ } else {
+ // seed current entry
+ Entry& e = _fifo.current();
+ e._bytes.init(cur_abs);
+ e._live_nodes.init(cur_nodes);
+ e.info = info;
+ e.level = level;
+ }
+ DEBUG_ONLY(_inbetween_phases = false;)
+}
+
+FullMethodName::FullMethodName() : _k(nullptr), _m(nullptr), _s(nullptr) {}
+
+FullMethodName::FullMethodName(const Method* m) :
+ _k(m->klass_name()), _m(m->name()), _s(m->signature()) {};
+
+FullMethodName::FullMethodName(const FullMethodName& o) : _k(o._k), _m(o._m), _s(o._s) {}
+
+FullMethodName& FullMethodName::operator=(const FullMethodName& o) {
+ _k = o._k; _m = o._m; _s = o._s;
+ return *this;
+}
+
+void FullMethodName::make_permanent() {
+ _k->make_permanent();
+ _m->make_permanent();
+ _s->make_permanent();
+}
+
+void FullMethodName::print_on(outputStream* st) const {
+ char tmp[1024];
+ st->print_raw(_k->as_C_string(tmp, sizeof(tmp)));
+ st->print_raw("::");
+ st->print_raw(_m->as_C_string(tmp, sizeof(tmp)));
+ st->put('(');
+ st->print_raw(_s->as_C_string(tmp, sizeof(tmp)));
+ st->put(')');
+}
+
+char* FullMethodName::as_C_string(char* buf, size_t len) const {
+ stringStream ss(buf, len);
+ print_on(&ss);
+ return buf;
+}
+
+bool FullMethodName::operator== (const FullMethodName& b) const {
+ return _k == b._k && _m == b._m && _s == b._s;
+}
+
#ifdef ASSERT
- // Note: if this fires, we free more arena memory under the scope of the
- // CompilationMemoryHistoryMark than we allocate. This cannot be since we
- // assume arena allocations in CompilerThread to be stack bound and symmetric.
- assert(delta >= 0 || ((ssize_t)_current + delta) >= 0,
- "Negative overflow (d=%zd %zu %zu)", delta, _current, _peak);
-#endif
- // Update totals
- _current += delta;
- _current_by_tag.add(tag, delta);
- // Did we reach a peak?
+bool FullMethodName::is_test_class() const {
+ char tmp[1024];
+ _k->as_C_string(tmp, sizeof(tmp));
+ return strstr(tmp, "CompileCommandPrintMemStat") != nullptr ||
+ strstr(tmp, "CompileCommandMemLimit") != nullptr;
+}
+#endif // ASSERT
+
+ArenaStatCounter::ArenaStatCounter(const CompileTask* task, size_t limit) :
+ _fmn(task->method()),
+ _should_print_memstat(task->directive()->should_print_memstat()),
+ _should_crash_on_memlimit(task->directive()->should_crash_at_mem_limit()),
+ _current(0), _peak(0), _live_nodes_current(0), _live_nodes_at_global_peak(0),
+ _limit(limit), _hit_limit(false), _limit_in_process(false),
+ _phase_counter(0), _comp_type(task->compiler()->type()), _comp_id(task->compile_id())
+ DEBUG_ONLY(COMMA _is_test_class(false))
+{
+ _fmn.make_permanent();
+#ifdef ASSERT
+ // If the class name matches the JTreg test class, we are in test mode and
+ // will do some test allocations to test the statistic
+ _is_test_class = _fmn.is_test_class();
+#endif // ASSERT
+}
+
+void ArenaStatCounter::on_phase_start(PhaseInfo info) {
+ // Update node counter
+ _live_nodes_current = retrieve_live_node_count();
+
+ // For the timeline, when nesting TracePhase happens, we maintain the illusion of a flat succession of
+ // separate phases. Thus, { TracePhase p1; { TracePhase p2; }} will be seen as:
+ // P1 starts -> P1 ends -> P2 starts -> P2 ends -> P1 starts -> P1 ends
+ // In other words, when a child phase interrupts a parent phase, it "ends" the parent phase, which will
+ // be "restarted" when the child phase ends.
+ // This is the only way to get a per-phase timeline that makes any sort of sense.
+ if (!_phase_info_stack.empty()) {
+ _timeline.on_phase_end(_current, _live_nodes_current);
+ }
+ _phase_info_stack.push(info);
+ _timeline.on_phase_start(info, _current, _live_nodes_current, _phase_info_stack.depth());
+}
+
+void ArenaStatCounter::on_phase_end() {
+ PhaseInfo top = _phase_info_stack.top();
+ _phase_info_stack.pop();
+ _live_nodes_current = retrieve_live_node_count();
+ _timeline.on_phase_end(_current, _live_nodes_current);
+ if (!_phase_info_stack.empty()) {
+ // "restart" parent phase in timeline
+ _timeline.on_phase_start(_phase_info_stack.top(), _current, _live_nodes_current, _phase_info_stack.depth());
+ }
+}
+
+int ArenaStatCounter::retrieve_live_node_count() const {
+ int result = 0;
+#ifdef COMPILER2
+ if (_comp_type == compiler_c2) {
+ // Update C2 node count
+ // Careful, Compile::current() may be null in a short time window when Compile itself
+ // is still being constructed.
+ if (Compile::current() != nullptr) {
+ result = Compile::current()->live_nodes();
+ }
+ }
+#endif // COMPILER2
+ return result;
+}
+
+// Account an arena allocation. Returns true if new peak reached.
+bool ArenaStatCounter::on_arena_chunk_allocation(size_t size, int arena_tag, uint64_t* stamp) {
+ bool rc = false;
+
+ const size_t old_current = _current;
+ _current += size;
+ assert(_current >= old_current, "Overflow");
+
+ const int phase_trc_id = _phase_info_stack.top().id;
+ _counters_current.add(size, phase_trc_id, arena_tag);
+ _live_nodes_current = retrieve_live_node_count();
+
+ _timeline.on_footprint_change(_current, _live_nodes_current);
+
+ // Did we reach a global peak?
if (_current > _peak) {
_peak = _current;
- assert(delta > 0, "Sanity (%zu %zu)", _current, _peak);
- update_c2_node_count();
- _peak_by_tag = _current_by_tag;
- rc = true;
+ // snapshot all current counters
+ _counters_at_global_peak.copy_from(_counters_current);
+ // snapshot live nodes
+ _live_nodes_at_global_peak = _live_nodes_current;
// Did we hit the memory limit?
if (!_hit_limit && _limit > 0 && _peak > _limit) {
_hit_limit = true;
}
+ // report peak back
+ rc = true;
}
+
+ // calculate arena chunk stamp
+ chunkstamp_t cs;
+ cs.tracked = 1;
+ cs.arena_tag = checked_cast(arena_tag);
+ cs.phase_id = checked_cast(_phase_info_stack.top().id);
+ *stamp = cs.raw;
+
return rc;
}
-void ArenaStatCounter::print_on(outputStream* st) const {
- st->print("%zu [", _peak);
- for (int tag = 0; tag < _peak_by_tag.element_count(); tag++) {
- if (_peak_by_tag.counter(tag) > 0) {
- st->print("%s %zu ", _peak_by_tag.tag_name(tag), _peak_by_tag.counter(tag));
- }
- }
- st->print("]");
-#ifdef ASSERT
- st->print(" (%zu->%zu)", _peak, _current);
-#endif
+void ArenaStatCounter::on_arena_chunk_deallocation(size_t size, uint64_t stamp) {
+ assert(_current >= size, "Underflow (%zu %zu)", size, _current);
+
+ // Extract tag and phase id from stamp
+ chunkstamp_t cs;
+ cs.raw = stamp;
+ assert(cs.tracked == 1, "Sanity");
+ const int arena_tag = cs.arena_tag;
+ assert(arena_tag >= 0 && arena_tag < arena_tag_max, "Arena Tag OOB (%d)", arena_tag_max);
+ const int phase_trc_id(cs.phase_id);
+ assert(phase_trc_id >= 0 && phase_trc_id < phase_trc_id_max, "Phase trace id OOB (%d)", phase_trc_id);
+
+ _current -= size;
+ _counters_current.sub(size, phase_trc_id, arena_tag);
+ _live_nodes_current = retrieve_live_node_count();
+ _timeline.on_footprint_change(_current, _live_nodes_current);
}
-//////////////////////////
-// Backend
-
-class FullMethodName {
- Symbol* const _k;
- Symbol* const _m;
- Symbol* const _s;
-
-public:
-
- FullMethodName(const Method* m) :
- _k(m->klass_name()), _m(m->name()), _s(m->signature()) {};
- FullMethodName(const FullMethodName& o) : _k(o._k), _m(o._m), _s(o._s) {}
-
- void make_permanent() {
- _k->make_permanent();
- _m->make_permanent();
- _s->make_permanent();
+// Used for logging, not for the report table generated with jcmd Compiler.memory
+void ArenaStatCounter::print_peak_state_on(outputStream* st) const {
+ st->print("Total Usage: %zu ", _peak);
+ if (_peak > 0) {
+#ifdef COMPILER2
+ // C1: print allocations broken down by arena types
+ if (_comp_type == CompilerType::compiler_c1) {
+ st->print("[");
+ size_t sums[arena_tag_max];
+ _counters_at_global_peak.summarize(sums);
+ bool print_comma = false;
+ for (int i = 0; i < arena_tag_max; i++) {
+ if (sums[i] > 0) {
+ if (print_comma) {
+ st->print_raw(", ");
+ }
+ st->print("%s %zu", Arena::tag_name[i], sums[i]);
+ print_comma = true;
+ }
+ }
+ st->print_cr("]");
+ }
+#endif // COMPILER1
+#ifdef COMPILER2
+ // C2: print counters and timeline on multiple lines, indented
+ if (_comp_type == CompilerType::compiler_c2) {
+ streamIndentor si(st, 4);
+ st->cr();
+ st->print_cr("--- Arena Usage by Arena Type and compilation phase, at arena usage peak of %zu ---", _peak);
+ {
+ streamIndentor si(st, 4);
+ _counters_at_global_peak.print_on(st);
+ }
+ st->print_cr("--- Allocation timelime by phase ---");
+ {
+ streamIndentor si(st, 4);
+ _timeline.print_on(st);
+ }
+ st->print_cr("---");
+ }
+#endif
+ } else {
+ st->cr();
}
+}
- static unsigned compute_hash(const FullMethodName& n) {
- return Symbol::compute_hash(n._k) ^
- Symbol::compute_hash(n._m) ^
- Symbol::compute_hash(n._s);
- }
+class MemStatEntry : public CHeapObj {
- char* as_C_string(char* buf, size_t len) const {
- stringStream ss(buf, len);
- ResourceMark rm;
- ss.print_raw(_k->as_C_string());
- ss.print_raw("::");
- ss.print_raw(_m->as_C_string());
- ss.put('(');
- ss.print_raw(_s->as_C_string());
- ss.put(')');
- return buf;
- }
- bool operator== (const FullMethodName& b) const {
- return _k == b._k && _m == b._m && _s == b._s;
- }
-};
-
-// Note: not mtCompiler since we don't want to change what we measure
-class MemStatEntry : public CHeapObj {
- const FullMethodName _method;
- CompilerType _comptype;
+ FullMethodName _fmn;
+ CompilerType _comp_type;
+ int _comp_id;
double _time;
- // How often this has been recompiled.
- int _num_recomp;
// Compiling thread. Only for diagnostic purposes. Thread may not be alive anymore.
const Thread* _thread;
// active limit for this compilation, if any
size_t _limit;
-
- // peak usage, bytes, over all arenas
- size_t _total;
- // usage per arena tag when total peaked
- ArenaCountersByTag _peak_by_tag;
- // number of nodes (c2 only) when total peaked
- unsigned _live_nodes_at_peak;
+ // true if the compilation hit the limit
+ bool _hit_limit;
+ // result as reported by compiler
const char* _result;
+ // Bytes total at global peak
+ size_t _peak;
+ // Bytes per arena tag.
+ size_t _peak_composition_per_arena_tag[arena_tag_max];
+ // Number of live nodes at global peak (C2 only)
+ unsigned _live_nodes_at_global_peak;
+
+ struct Details {
+ ArenaCounterTable counters_at_global_peak;
+ FootprintTimeline timeline;
+ };
+
+ Details* _detail_stats;
+
+ MemStatEntry(const MemStatEntry& e); // deny
+
public:
- MemStatEntry(FullMethodName method)
- : _method(method), _comptype(compiler_c1),
- _time(0), _num_recomp(0), _thread(nullptr), _limit(0),
- _total(0), _live_nodes_at_peak(0),
- _result(nullptr) {
- _peak_by_tag.clear();
+ MemStatEntry()
+ : _comp_type(compiler_none), _comp_id(-1),
+ _time(0), _thread(nullptr), _limit(0), _hit_limit(false),
+ _result(nullptr), _peak(0), _live_nodes_at_global_peak(0),
+ _detail_stats(nullptr) {
}
- void set_comptype(CompilerType comptype) { _comptype = comptype; }
+ ~MemStatEntry() {
+ clean_details();
+ }
+
+ void set_comp_id(int comp_id) { _comp_id = comp_id; }
+ void set_comptype(CompilerType comptype) { _comp_type = comptype; }
void set_current_time() { _time = os::elapsedTime(); }
void set_current_thread() { _thread = Thread::current(); }
void set_limit(size_t limit) { _limit = limit; }
- void inc_recompilation() { _num_recomp++; }
- void set_total(size_t n) { _total = n; }
- void set_peak_by_tag(ArenaCountersByTag peak_by_tag) { _peak_by_tag = peak_by_tag; }
- void set_live_nodes_at_peak(unsigned n) { _live_nodes_at_peak = n; }
+ void set_from_state(const ArenaStatCounter* state, bool store_details) {
+ _fmn = state->fmn();
+ _comp_type = state->comp_type();
+ _comp_id = state->comp_id();
+ _limit = state->limit();
+ _hit_limit = state->hit_limit();
+ _peak = state->peak();
+ _live_nodes_at_global_peak = state->live_nodes_at_global_peak();
+ state->counters_at_global_peak().summarize(_peak_composition_per_arena_tag);
+#ifdef COMPILER2
+ assert(_detail_stats == nullptr, "should have been cleaned");
+ if (store_details) {
+ _detail_stats = NEW_C_HEAP_OBJ(Details, mtCompiler);
+ _detail_stats->counters_at_global_peak.copy_from(state->counters_at_global_peak());
+ _detail_stats->timeline.copy_from(state->timeline());
+ }
+#endif // COMPILER2
+ }
- void set_result(const char* s) { _result = s; }
+ void clean_details() {
+ if (_detail_stats != nullptr) {
+ FREE_C_HEAP_ARRAY(Details, _detail_stats);
+ _detail_stats = nullptr;
+ }
+ }
- size_t total() const { return _total; }
+ void reset() {
+ clean_details();
+ _comp_type = CompilerType::compiler_none;
+ _comp_id = -1;
+ _limit = _peak = 0;
+ _live_nodes_at_global_peak = 0;
+ memset(_peak_composition_per_arena_tag, 0, sizeof(_peak_composition_per_arena_tag));
+ }
+
+ void set_result(const char* s) { _result = s; }
+
+ size_t peak() const { return _peak; }
+ bool is_c1() const { return _comp_type == CompilerType::compiler_c1; }
+ bool is_c2() const { return _comp_type == CompilerType::compiler_c2; }
static void print_legend(outputStream* st) {
#define LEGEND_KEY_FMT "%11s"
st->print_cr("Legend:");
- st->print_cr(" " LEGEND_KEY_FMT ": %s", "total", "memory allocated via arenas while compiling");
- for (int tag = 0; tag < Arena::tag_count(); tag++) {
+ st->print_cr(" " LEGEND_KEY_FMT ": %s", "ctype", "compiler type");
+ st->print_cr(" " LEGEND_KEY_FMT ": %s", "total", "peak memory allocated via arenas while compiling");
+ for (int tag = 0; tag < arena_tag_max; tag++) {
st->print_cr(" " LEGEND_KEY_FMT ": %s", Arena::tag_name[tag], Arena::tag_desc[tag]);
}
- st->print_cr(" " LEGEND_KEY_FMT ": %s", "result", "Result: 'ok' finished successfully, 'oom' hit memory limit, 'err' compilation failed");
st->print_cr(" " LEGEND_KEY_FMT ": %s", "#nodes", "...how many nodes (c2 only)");
- st->print_cr(" " LEGEND_KEY_FMT ": %s", "limit", "memory limit, if set");
- st->print_cr(" " LEGEND_KEY_FMT ": %s", "time", "time taken for last compilation (sec)");
- st->print_cr(" " LEGEND_KEY_FMT ": %s", "type", "compiler type");
- st->print_cr(" " LEGEND_KEY_FMT ": %s", "#rc", "how often recompiled");
+ st->print_cr(" " LEGEND_KEY_FMT ": %s", "result", "Result reported by compiler");
+ st->print_cr(" " LEGEND_KEY_FMT ": %s", "limit", "memory limit; followed by \"*\" if the limit was hit");
+ st->print_cr(" " LEGEND_KEY_FMT ": %s", "time", "timestamp");
+ st->print_cr(" " LEGEND_KEY_FMT ": %s", "id", "compile id");
st->print_cr(" " LEGEND_KEY_FMT ": %s", "thread", "compiler thread");
#undef LEGEND_KEY_FMT
}
static void print_header(outputStream* st) {
+ st->print("%-6s", "ctyp");
+
#define SIZE_FMT "%-10s"
st->print(SIZE_FMT, "total");
- for (int tag = 0; tag < Arena::tag_count(); tag++) {
+ for (int tag = 0; tag < arena_tag_max; tag++) {
st->print(SIZE_FMT, Arena::tag_name[tag]);
}
-#define HDR_FMT1 "%-8s%-8s%-8s%-8s"
-#define HDR_FMT2 "%-6s%-4s%-19s%s"
+#define HDR_FMT1 "%-8s%-8s%-10s%-8s"
+#define HDR_FMT2 "%-6s%-19s%s"
- st->print(HDR_FMT1, "result", "#nodes", "limit", "time");
- st->print(HDR_FMT2, "type", "#rc", "thread", "method");
- st->print_cr("");
+ st->print(HDR_FMT1, "#nodes", "result", "limit", "time");
+ st->print(HDR_FMT2, "id", "thread", "method");
}
- void print_on(outputStream* st, bool human_readable) const {
- int col = 0;
+ void print_brief_oneline(outputStream* st) const {
+ int col = st->indentation();
+
+ // Type
+ st->print("%2s ", compilertype2name(_comp_type));
+ col += 6; st->fill_to(col);
// Total
- if (human_readable) {
- st->print(PROPERFMT " ", PROPERFMTARGS(_total));
- } else {
- st->print("%zu ", _total);
- }
+ size_t v = _peak;
+ st->print("%zu ", v);
col += 10; st->fill_to(col);
- for (int tag = 0; tag < Arena::tag_count(); tag++) {
- if (human_readable) {
- st->print(PROPERFMT " ", PROPERFMTARGS(_peak_by_tag.counter(tag)));
- } else {
- st->print("%zu ", _peak_by_tag.counter(tag));
- }
+ for (int tag = 0; tag < arena_tag_max; tag++) {
+ v = _peak_composition_per_arena_tag[tag];
+ st->print("%zu ", v);
col += 10; st->fill_to(col);
}
+ // Number of Nodes when memory peaked
+ if (_live_nodes_at_global_peak > 0) {
+ st->print("%u ", _live_nodes_at_global_peak);
+ } else {
+ st->print("-");
+ }
+ col += 8; st->fill_to(col);
+
// result?
st->print("%s ", _result ? _result : "");
col += 8; st->fill_to(col);
- // Number of Nodes when memory peaked
- if (_live_nodes_at_peak > 0) {
- st->print("%u ", _live_nodes_at_peak);
- } else {
- st->print("-");
- }
- col += 8; st->fill_to(col);
-
// Limit
if (_limit > 0) {
- st->print(PROPERFMT " ", PROPERFMTARGS(_limit));
+ st->print("%zu%s ", _limit, _hit_limit ? "*" : "");
} else {
st->print("-");
}
- col += 8; st->fill_to(col);
+ col += 10; st->fill_to(col);
// TimeStamp
st->print("%.3f ", _time);
col += 8; st->fill_to(col);
- // Type
- st->print("%s ", compilertype2name(_comptype));
+ // Compile ID
+ st->print("%d ", _comp_id);
col += 6; st->fill_to(col);
- // Recomp
- st->print("%u ", _num_recomp);
- col += 4; st->fill_to(col);
-
// Thread
st->print(PTR_FORMAT " ", p2i(_thread));
// MethodName
char buf[1024];
- st->print("%s ", _method.as_C_string(buf, sizeof(buf)));
+ st->print("%s ", _fmn.as_C_string(buf, sizeof(buf)));
+
st->cr();
}
- int compare_by_size(const MemStatEntry* b) const {
- const size_t x1 = b->_total;
- const size_t x2 = _total;
- return x1 < x2 ? -1 : x1 == x2 ? 0 : 1;
- }
-};
+ void print_detailed(outputStream* st) const {
+ int col = 0;
-// The MemStatTable contains records of memory usage of all compilations. It is printed,
-// as memory summary, either with jcmd Compiler.memory, or - if the "print" suboption has
-// been given with the MemStat compile command - as summary printout at VM exit.
-// For any given compiled method, we only keep the memory statistics of the most recent
-// compilation, but on a per-compiler basis. If one needs statistics of prior compilations,
-// one needs to look into the log produced by the "print" suboption.
+ constexpr int indent1 = 40;
+ constexpr int indent2 = 50;
-class MemStatTableKey {
- const FullMethodName _fmn;
- const CompilerType _comptype;
-public:
- MemStatTableKey(FullMethodName fmn, CompilerType comptype) :
- _fmn(fmn), _comptype(comptype) {}
- MemStatTableKey(const MemStatTableKey& o) :
- _fmn(o._fmn), _comptype(o._comptype) {}
- bool operator== (const MemStatTableKey& other) const {
- return _fmn == other._fmn && _comptype == other._comptype;
- }
- static unsigned compute_hash(const MemStatTableKey& n) {
- return FullMethodName::compute_hash(n._fmn) + (unsigned)n._comptype;
- }
-};
-
-class MemStatTable :
- public ResourceHashtable
-{
-public:
-
- void add(const FullMethodName& fmn, CompilerType comptype,
- size_t total, ArenaCountersByTag peak_by_tag,
- unsigned live_nodes_at_peak, size_t limit, const char* result) {
- assert_lock_strong(NMTCompilationCostHistory_lock);
- MemStatTableKey key(fmn, comptype);
- MemStatEntry** pe = get(key);
- MemStatEntry* e = nullptr;
- if (pe == nullptr) {
- e = new MemStatEntry(fmn);
- put(key, e);
- } else {
- // Update existing entry
- e = *pe;
- assert(e != nullptr, "Sanity");
+ char buf[1024];
+ st->print_cr("Method : %s", _fmn.as_C_string(buf, sizeof(buf)));
+ st->print_cr("Compiler : %2s", compilertype2name(_comp_type));
+ st->print( "Arena Usage at peak : %zu", _peak);
+ if (_peak > M) {
+ st->print(" (%.2fM)", ((double)_peak/(double)M));
}
+ st->cr();
+ if (_comp_type == CompilerType::compiler_c2) {
+ st->print_cr("Nodes at peak : %u", _live_nodes_at_global_peak);
+ }
+ st->print_cr("Compile ID : %d", _comp_id);
+ st->print( "Result : %s", _result);
+ if (strcmp(_result, "oom") == 0) {
+ st->print(" (memory limit was: %zu)", _limit);
+ }
+ st->cr();
+ st->print_cr("Thread : " PTR_FORMAT, p2i(_thread));
+ st->print_cr("Timestamp : %.3f", _time);
+
+ if (_detail_stats != nullptr) {
+ st->cr();
+ st->print_cr("Arena Usage by Arena Type and compilation phase, at arena usage peak of %zu:", _peak);
+ _detail_stats->counters_at_global_peak.print_on(st);
+ st->cr();
+ st->print_cr("Allocation timelime by phase:");
+ _detail_stats->timeline.print_on(st);
+ } else {
+ st->cr();
+ st->print_cr("Arena Usage by Arena Type, at arena usage peak of %zu:", _peak);
+ for (int tag = 0; tag < arena_tag_max; tag++) {
+ const size_t v = _peak_composition_per_arena_tag[tag];
+ if (v > 0) {
+ st->print_cr("%-36s: %zu ", Arena::tag_desc[tag], v);
+ }
+ }
+ }
+ }
+};
+
+class MemStatStore : public CHeapObj {
+
+ // Total number of entries. Reaching this limit, we discard the least interesting (smallest allocation size) first.
+ static constexpr int max_entries = 64;
+
+ struct {
+ size_t s; MemStatEntry* e;
+ } _entries[max_entries];
+
+ struct iteration_result { unsigned num, num_c1, num_c2, num_filtered_out; };
+ template
+ void iterate_sorted_filtered(F f, size_t minsize, int max_num_printed, iteration_result& result) const {
+ assert_lock_strong(NMTCompilationCostHistory_lock);
+ const unsigned stop_after = max_num_printed == -1 ? UINT_MAX : (unsigned)max_num_printed;
+ result.num = result.num_c1 = result.num_c2 = result.num_filtered_out = 0;
+ for (int i = 0; i < max_entries && _entries[i].e != nullptr && result.num < stop_after; i++) {
+ if (_entries[i].s >= minsize) {
+ f(_entries[i].e);
+ result.num++;
+ result.num_c1 += _entries[i].e->is_c1() ? 1 : 0;
+ result.num_c2 += _entries[i].e->is_c2() ? 1 : 0;
+ } else {
+ result.num_filtered_out++;
+ }
+ }
+ }
+
+ void print_footer(outputStream* st, size_t minsize, const iteration_result& result) const {
+ if (result.num > 0) {
+ st->print_cr("Total: %u (C1: %u, C2: %u)", result.num, result.num_c1, result.num_c2);
+ } else {
+ st->print_cr("No entries.");
+ }
+ if (result.num_filtered_out > 0) {
+ st->print_cr(" (%d compilations smaller than %zu omitted)", result.num_filtered_out, minsize);
+ }
+ }
+
+public:
+
+ MemStatStore() {
+ memset(_entries, 0, sizeof(_entries));
+ }
+
+ void add(const ArenaStatCounter* state, const char* result) {
+
+ const size_t size = state->peak();
+
+ // search insert point
+ int i = 0;
+ while (i < max_entries && _entries[i].s > size) {
+ i++;
+ }
+ if (i == max_entries) {
+ return;
+ }
+ MemStatEntry* e = _entries[max_entries - 1].e; // recycle last one
+ if (e == nullptr) {
+ e = new MemStatEntry();
+ }
+ memmove(_entries + i + 1, _entries + i, sizeof(_entries[0]) * (max_entries - i - 1));
+
+ e->reset();
e->set_current_time();
e->set_current_thread();
- e->set_comptype(comptype);
- e->inc_recompilation();
- e->set_total(total);
- e->set_peak_by_tag(peak_by_tag);
- e->set_live_nodes_at_peak(live_nodes_at_peak);
- e->set_limit(limit);
e->set_result(result);
+
+ // Since we don't have phases in C1, for now we just avoid saving details for C1.
+ const bool save_details = state->comp_type() == CompilerType::compiler_c2;
+ e->set_from_state(state, save_details);
+
+ _entries[i].s = e->peak();
+ _entries[i].e = e;
}
- // Returns a C-heap-allocated SortMe array containing all entries from the table,
- // optionally filtered by entry size
- MemStatEntry** calc_flat_array(int& num, size_t min_size) {
+ void print_table(outputStream* st, bool legend, size_t minsize, int max_num_printed) const {
assert_lock_strong(NMTCompilationCostHistory_lock);
- const int num_all = number_of_entries();
- MemStatEntry** flat = NEW_C_HEAP_ARRAY(MemStatEntry*, num_all, mtInternal);
- int i = 0;
- auto do_f = [&] (const MemStatTableKey& ignored, MemStatEntry* e) {
- if (e->total() >= min_size) {
- flat[i] = e;
- assert(i < num_all, "Sanity");
- i ++;
- }
- };
- iterate_all(do_f);
- if (min_size == 0) {
- assert(i == num_all, "Sanity");
- } else {
- assert(i <= num_all, "Sanity");
+ if (legend) {
+ MemStatEntry::print_legend(st);
+ st->cr();
}
- num = i;
- return flat;
+
+ MemStatEntry::print_header(st);
+ st->cr();
+
+ iteration_result itres;
+ auto printer = [&](const MemStatEntry* e) {
+ e->print_brief_oneline(st);
+ };
+ iterate_sorted_filtered(printer, minsize, max_num_printed, itres);
+ print_footer(st, minsize, itres);
+ }
+
+ void print_details(outputStream* st, size_t minsize, int max_num_printed) const {
+ assert_lock_strong(NMTCompilationCostHistory_lock);
+ iteration_result itres;
+ auto printer = [&](const MemStatEntry* e) {
+ e->print_detailed(st);
+ st->cr();
+ st->print_cr("------------------------");
+ st->cr();
+ };
+ iterate_sorted_filtered(printer, minsize, max_num_printed, itres);
}
};
bool CompilationMemoryStatistic::_enabled = false;
-
-static MemStatTable* _the_table = nullptr;
+static MemStatStore* _the_store = nullptr;
void CompilationMemoryStatistic::initialize() {
- assert(_enabled == false && _the_table == nullptr, "Only once");
- _the_table = new (mtCompiler) MemStatTable;
+ assert(_enabled == false && _the_store == nullptr, "Only once");
+ _the_store = new MemStatStore;
_enabled = true;
log_info(compilation, alloc)("Compilation memory statistic enabled");
}
void CompilationMemoryStatistic::on_start_compilation(const DirectiveSet* directive) {
assert(enabled(), "Not enabled?");
+ assert(directive->should_collect_memstat(), "Don't call if not needed");
+ CompilerThread* const th = Thread::current()->as_Compiler_thread();
+ CompileTask* const task = th->task();
const size_t limit = directive->mem_limit();
- Thread::current()->as_Compiler_thread()->arena_stat()->start(limit);
+ // Create new ArenaStat object and hook it into the thread
+ assert(th->arena_stat() == nullptr, "Sanity");
+ ArenaStatCounter* const arena_stat = new ArenaStatCounter(task, limit);
+ th->set_arenastat(arena_stat);
+ // Start a "root" phase
+ PhaseInfo info;
+ info.id = phase_trc_id_none;
+ info.num = 0;
+ info.text = "(outside)";
+ arena_stat->on_phase_start(info);
}
void CompilationMemoryStatistic::on_end_compilation() {
assert(enabled(), "Not enabled?");
- ResourceMark rm;
CompilerThread* const th = Thread::current()->as_Compiler_thread();
ArenaStatCounter* const arena_stat = th->arena_stat();
+ if (arena_stat == nullptr) { // not started
+ return;
+ }
+
+ // Mark end of compilation by clearing out the arena state object in the CompilerThread.
+ // Do this before the final "phase end".
+ th->set_arenastat(nullptr);
+
+ // End final outer phase.
+ arena_stat->on_phase_end();
+
CompileTask* const task = th->task();
- const CompilerType ct = task->compiler()->type();
+ assert(task->compile_id() == arena_stat->comp_id(), "Different compilation?");
const Method* const m = th->task()->method();
- FullMethodName fmn(m);
- fmn.make_permanent();
const DirectiveSet* directive = th->task()->directive();
assert(directive->should_collect_memstat(), "Should only be called if memstat is enabled for this method");
@@ -438,7 +803,7 @@ void CompilationMemoryStatistic::on_end_compilation() {
// Store memory used in task, for later processing by JFR
task->set_arena_bytes(arena_stat->peak());
- // Store result
+ // Store result (ok, failed, oom...)
// For this to work, we must call on_end_compilation() at a point where
// Compile|Compilation already handed over the failure string to ciEnv,
// but ciEnv must still be alive.
@@ -453,24 +818,22 @@ void CompilationMemoryStatistic::on_end_compilation() {
{
MutexLocker ml(NMTCompilationCostHistory_lock, Mutex::_no_safepoint_check_flag);
- assert(_the_table != nullptr, "not initialized");
-
- _the_table->add(fmn, ct,
- arena_stat->peak(), // total
- arena_stat->peak_by_tag(),
- arena_stat->live_nodes_at_peak(),
- arena_stat->limit(),
- result);
+ assert(_the_store != nullptr, "not initialized");
+ _the_store->add(arena_stat, result);
}
+
if (print) {
- char buf[1024];
- fmn.as_C_string(buf, sizeof(buf));
- tty->print("%s Arena usage %s: ", compilertype2name(ct), buf);
- arena_stat->print_on(tty);
- tty->cr();
+ // Pre-assemble string to prevent tearing
+ stringStream ss;
+ StreamAutoIndentor sai(&ss);
+ ss.print("%s (%d) (%s) Arena usage ", compilertype2name(arena_stat->comp_type()), arena_stat->comp_id(), result);
+ arena_stat->fmn().print_on(&ss);
+ ss.print_raw(": ");
+ arena_stat->print_peak_state_on(&ss);
+ tty->print_raw(ss.base());
}
- arena_stat->end(); // reset things
+ delete arena_stat;
}
static void inform_compilation_about_oom(CompilerType ct) {
@@ -508,124 +871,178 @@ static void inform_compilation_about_oom(CompilerType ct) {
}
}
-void CompilationMemoryStatistic::on_arena_change(ssize_t diff, const Arena* arena) {
- assert(enabled(), "Not enabled?");
- CompilerThread* const th = Thread::current()->as_Compiler_thread();
+void CompilationMemoryStatistic::on_arena_chunk_allocation(size_t size, int arena_tag, uint64_t* stamp) {
+ assert(enabled(), "Not enabled?");
+ assert(arena_tag >= 0 && arena_tag < arena_tag_max, "Arena Tag OOB (%d)", arena_tag_max);
+
+ CompilerThread* const th = Thread::current()->as_Compiler_thread();
ArenaStatCounter* const arena_stat = th->arena_stat();
- if (arena_stat->limit_in_process()) {
- return; // avoid recursion on limit hit
+ if (arena_stat == nullptr || // not started
+ arena_stat->limit_in_process()) { // limit handling in process, prevent recursion
+ return;
}
- bool hit_limit_before = arena_stat->hit_limit();
+ // Compiler can be slow to bailout, so we may hit memlimit more than once
+ const bool hit_limit_before = arena_stat->hit_limit();
- if (arena_stat->is_active() && arena_stat->account(diff, (int)arena->get_tag())) { // new peak?
+ if (arena_stat->on_arena_chunk_allocation(size, arena_tag, stamp)) { // new peak?
// Limit handling
if (arena_stat->hit_limit()) {
char name[1024] = "";
- bool print = false;
- bool crash = false;
- CompilerType ct = compiler_none;
-
+ const CompilerType ct = arena_stat->comp_type();
+ const bool print = arena_stat->should_print_memstat();
+ const bool crash = arena_stat->should_crash_on_memlimit();
arena_stat->set_limit_in_process(true); // prevent recursive limit hits
+ arena_stat->fmn().as_C_string(name, sizeof(name));
- // get some more info
- const CompileTask* const task = th->task();
- if (task != nullptr) {
- ct = task->compiler()->type();
- const DirectiveSet* directive = task->directive();
- print = directive->should_print_memstat();
- crash = directive->should_crash_at_mem_limit();
- const Method* m = th->task()->method();
- if (m != nullptr) {
- FullMethodName(m).as_C_string(name, sizeof(name));
+ inform_compilation_about_oom(ct);
+
+ if (crash) {
+ // Store this ArenaStat. If other threads also run into OOMs, let them sleep.
+ // We will never return, so the global store will not contain this info. We will
+ // print the stored ArenaStat in hs-err (see print_error_report)
+ if (Atomic::cmpxchg(&_arenastat_oom_crash, (ArenaStatCounter*) nullptr, arena_stat) != nullptr) {
+ os::infinite_sleep();
}
}
- char message[1024] = "";
+ stringStream short_msg;
- // build up message if we need it later
+ // We print to tty if either print is enabled or if we are to crash on MemLimit hit.
+ // If printing/crashing are not enabled, we just quietly abort the compilation. The
+ // compilation is marked as "oom" in the compilation memory result store.
if (print || crash) {
- stringStream ss(message, sizeof(message));
- if (ct != compiler_none && name[0] != '\0') {
- ss.print("%s %s: ", compilertype2name(ct), name);
- }
- ss.print("Hit MemLimit %s(limit: %zu now: %zu)",
+ short_msg.print("%s (%d) %s: ", compilertype2name(ct), arena_stat->comp_id(), name);
+ short_msg.print("Hit MemLimit %s- limit: %zu now: %zu",
(hit_limit_before ? "again " : ""),
arena_stat->limit(), arena_stat->peak());
- }
-
- // log if needed
- if (print) {
- tty->print_raw(message);
+ tty->print_raw(short_msg.base());
tty->cr();
}
- // Crash out if needed
if (crash) {
- report_fatal(OOM_HOTSPOT_ARENA, __FILE__, __LINE__, "%s", message);
- } else {
- inform_compilation_about_oom(ct);
+ // Before crashing, if C2, end current phase. That causes its info (which is the most important) to
+ // be added to the phase timeline.
+ if (arena_stat->comp_type() == CompilerType::compiler_c2) {
+ arena_stat->on_phase_end();
+ }
+ // print extended message to tty (mirrors the one that should show up in the hs-err file, just for good measure)
+ tty->print_cr("Details:");
+ arena_stat->print_peak_state_on(tty);
+ tty->cr();
+ // abort VM
+ report_fatal(OOM_HOTSPOT_ARENA, __FILE__, __LINE__, "%s", short_msg.base());
}
arena_stat->set_limit_in_process(false);
- }
+ } // end Limit handling
}
}
-static inline ssize_t diff_entries_by_size(const MemStatEntry* e1, const MemStatEntry* e2) {
- return e1->compare_by_size(e2);
-}
-
-void CompilationMemoryStatistic::print_all_by_size(outputStream* st, bool human_readable, size_t min_size) {
-
- MutexLocker ml(NMTCompilationCostHistory_lock, Mutex::_no_safepoint_check_flag);
-
- st->cr();
- st->print_cr("Compilation memory statistics");
-
- if (!enabled()) {
- st->print_cr("(unavailable)");
+void CompilationMemoryStatistic::on_arena_chunk_deallocation(size_t size, uint64_t stamp) {
+ assert(enabled(), "Not enabled?");
+ CompilerThread* const th = Thread::current()->as_Compiler_thread();
+ ArenaStatCounter* const arena_stat = th->arena_stat();
+ if (arena_stat == nullptr) { // not started
return;
}
-
- st->cr();
-
- MemStatEntry::print_legend(st);
- st->cr();
-
- if (min_size > 0) {
- st->print_cr(" (cutoff: %zu bytes)", min_size);
+ if (arena_stat->limit_in_process()) {
+ return; // avoid recursion on limit hit
}
- st->cr();
- MemStatEntry::print_header(st);
+ arena_stat->on_arena_chunk_deallocation(size, stamp);
+}
- MemStatEntry** filtered = nullptr;
+void CompilationMemoryStatistic::on_phase_start(int phase_trc_id, const char* text) {
+ assert(enabled(), "Not enabled?");
+ assert(phase_trc_id >= 0 && phase_trc_id < phase_trc_id_max, "Phase trace id OOB (%d)", phase_trc_id);
+ CompilerThread* const th = Thread::current()->as_Compiler_thread();
+ ArenaStatCounter* const arena_stat = th->arena_stat();
+ if (arena_stat == nullptr) { // not started
+ return;
+ }
+ PhaseInfo info;
+ info.id = phase_trc_id;
+ info.num = arena_stat->advance_phase_counter();
+ info.text = text;
+ arena_stat->on_phase_start(info);
+}
- if (_the_table != nullptr) {
- // We sort with quicksort
- int num = 0;
- filtered = _the_table->calc_flat_array(num, min_size);
- if (min_size > 0) {
- st->print_cr("(%d/%d)", num, _the_table->number_of_entries());
- }
- if (num > 0) {
- QuickSort::sort(filtered, num, diff_entries_by_size);
- // Now print. Has to happen under lock protection too, since entries may be changed.
- for (int i = 0; i < num; i ++) {
- filtered[i]->print_on(st, human_readable);
- }
- } else {
- st->print_cr("No entries.");
- }
+void CompilationMemoryStatistic::on_phase_end() {
+ assert(enabled(), "Not enabled?");
+ CompilerThread* const th = Thread::current()->as_Compiler_thread();
+ ArenaStatCounter* const arena_stat = th->arena_stat();
+ if (arena_stat == nullptr) { // not started
+ return;
+ }
+ arena_stat->on_phase_end();
+}
+
+static bool check_before_reporting(outputStream* st) {
+ if (!CompilationMemoryStatistic::enabled()) {
+ st->print_cr("Compilation memory statistics disabled.");
+ return false;
+ }
+ if (_the_store == nullptr) {
+ st->print_cr("Compilation memory statistics not yet initialized. ");
+ return false;
+ }
+ return true;
+}
+
+bool CompilationMemoryStatistic::in_oom_crash() {
+ return Atomic::load(&_arenastat_oom_crash) != nullptr;
+}
+
+void CompilationMemoryStatistic::print_error_report(outputStream* st) {
+ if (!check_before_reporting(st)) {
+ return;
+ }
+ StreamAutoIndentor sai(tty);
+ streamIndentor si(tty, 4);
+ const ArenaStatCounter* const oom_stats = Atomic::load(&_arenastat_oom_crash);
+ if (oom_stats != nullptr) {
+ // we crashed due to a compiler limit hit. Lead with a printout of the offending stats
+ // in detail.
+ st->print_cr("Compiler Memory Statistic, hit OOM limit; offending compilation:");
+ oom_stats->fmn().print_on(st);
+ st->cr();
+ oom_stats->print_peak_state_on(st);
+ st->cr();
+ }
+ st->print_cr("Compiler Memory Statistic, 10 most expensive compilations:");
+ print_all_by_size(st, false, false, 0, 10);
+}
+
+void CompilationMemoryStatistic::print_final_report(outputStream* st) {
+ if (!check_before_reporting(st)) {
+ return;
+ }
+ st->print_cr("Compiler Memory Statistic, 10 most expensive compilations:");
+ StreamAutoIndentor sai(st);
+ streamIndentor si(st, 4);
+ print_all_by_size(st, false, false, 0, 10);
+}
+
+void CompilationMemoryStatistic::print_jcmd_report(outputStream* st, bool verbose, bool legend, size_t minsize) {
+ if (!check_before_reporting(st)) {
+ return;
+ }
+ st->print_cr("Compiler Memory Statistic");
+ StreamAutoIndentor sai(st);
+ streamIndentor si(st, 4);
+ print_all_by_size(st, verbose, legend, minsize, -1);
+}
+
+void CompilationMemoryStatistic::print_all_by_size(outputStream* st, bool verbose, bool legend, size_t minsize, int max_num_printed) {
+ MutexLocker ml(NMTCompilationCostHistory_lock, Mutex::_no_safepoint_check_flag);
+ if (verbose) {
+ _the_store->print_details(st, minsize, max_num_printed);
} else {
- st->print_cr("Not initialized.");
+ _the_store->print_table(st, legend, minsize, max_num_printed);
}
- st->cr();
-
- FREE_C_HEAP_ARRAY(Entry, filtered);
}
const char* CompilationMemoryStatistic::failure_reason_memlimit() {
@@ -633,12 +1050,50 @@ const char* CompilationMemoryStatistic::failure_reason_memlimit() {
return s;
}
+#ifdef ASSERT
+void CompilationMemoryStatistic::do_test_allocations() {
+
+ CompilerThread* const th = Thread::current()->as_Compiler_thread();
+ ArenaStatCounter* const arena_stat = th->arena_stat();
+ if (arena_stat == nullptr || !arena_stat->is_test_class()) { // not started or not the JTREG test
+ return;
+ }
+ const CompilerType ctyp = th->task()->compiler()->type();
+
+ // Note: all allocations in here need to be tagged as mtCompiler to be recognized
+ // by the compilation memstat. ResourceArea in a CompilerThread is already tagged
+ // mtCompiler (special handling for compiler threads).
+
+#ifdef COMPILER2
+ if (ctyp == CompilerType::compiler_c2) {
+ {
+ Compile::TracePhase tp(Phase::_t_testPhase1);
+ Arena ar(MemTag::mtCompiler, Arena::Tag::tag_reglive);
+ ar.Amalloc(2 * M); // phase-local peak
+ }
+ {
+ Compile::TracePhase tp(Phase::_t_testPhase2);
+ NEW_RESOURCE_ARRAY(char, 32 * M); // leaked (until compilation end)
+ }
+ } // C2
+#endif // COMPILER2
+
+#ifdef COMPILER1
+ if (ctyp == CompilerType::compiler_c1) {
+ NEW_RESOURCE_ARRAY(char, 32 * M); // leaked (until compilation end)
+ }
+#endif // COMPILER2
+
+}
+#endif // ASSERT
+
CompilationMemoryStatisticMark::CompilationMemoryStatisticMark(const DirectiveSet* directive)
: _active(directive->should_collect_memstat()) {
if (_active) {
CompilationMemoryStatistic::on_start_compilation(directive);
}
}
+
CompilationMemoryStatisticMark::~CompilationMemoryStatisticMark() {
if (_active) {
CompilationMemoryStatistic::on_end_compilation();
diff --git a/src/hotspot/share/compiler/compilationMemoryStatistic.hpp b/src/hotspot/share/compiler/compilationMemoryStatistic.hpp
index d9cbfabde1b..2de5b0e9abb 100644
--- a/src/hotspot/share/compiler/compilationMemoryStatistic.hpp
+++ b/src/hotspot/share/compiler/compilationMemoryStatistic.hpp
@@ -1,6 +1,6 @@
/*
- * Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2023, Red Hat, Inc. and/or its affiliates.
+ * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2025, Red Hat, Inc. and/or its affiliates.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -26,114 +26,49 @@
#ifndef SHARE_COMPILER_COMPILATIONMEMORYSTATISTIC_HPP
#define SHARE_COMPILER_COMPILATIONMEMORYSTATISTIC_HPP
-#include "compiler/compilerDefinitions.hpp"
#include "memory/allocation.hpp"
#include "memory/allStatic.hpp"
-#include "memory/arena.hpp"
#include "utilities/globalDefinitions.hpp"
-class outputStream;
-class Symbol;
class DirectiveSet;
-
-// Helper class to wrap the array of arena tags for easier processing
-class ArenaCountersByTag {
-private:
- size_t _counter[Arena::tag_count()];
-
-public:
- int element_count() const { return Arena::tag_count(); }
- const char* tag_name(int tag) const { return Arena::tag_name[tag]; }
-
- size_t counter(int tag) const {
- assert(tag < element_count(), "invalid tag %d", tag);
- return _counter[tag];
- }
-
- void add(int tag, size_t value) {
- assert(tag < element_count(), "invalid tag %d", tag);
- _counter[tag] += value;
- }
-
- void clear() {
- memset(_counter, 0, sizeof(size_t) * element_count());
- }
-};
-
-// Counters for allocations from arenas during compilation
-class ArenaStatCounter : public CHeapObj